From 12b03f34bb36a2deeaa627924b7af6f9e1e70d34 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 28 Mar 2021 22:43:46 +0530 Subject: [PATCH 01/15] Misc --- v2/Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 v2/Makefile diff --git a/v2/Makefile b/v2/Makefile new file mode 100644 index 000000000..247e7de43 --- /dev/null +++ b/v2/Makefile @@ -0,0 +1,14 @@ +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GOMOD=$(GOCMD) mod +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get + +all: build +build: + $(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go +test: + $(GOTEST) -v ./... +tidy: + $(GOMOD) tidy \ No newline at end of file From 718e4505a6b03530b7c5838f2a0c9a4c3a561296 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 31 Mar 2021 21:27:40 +0530 Subject: [PATCH 02/15] Fixed nuclei ignore issues + made random agent default --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/config.go | 7 +------ v2/internal/runner/runner.go | 1 + v2/internal/runner/update.go | 3 ++- v2/pkg/catalog/catalogue.go | 6 +++++- v2/pkg/catalog/ignore.go | 26 -------------------------- 6 files changed, 10 insertions(+), 35 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 9e961f07e..67543443c 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -52,7 +52,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output") set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") - set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value") + set.BoolVarP(&options.RandomAgent, "random-agent", "ra", true, "Use randomly selected HTTP User-Agent header value") set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses") set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request") diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index c7634388e..c68b7d594 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -114,17 +114,12 @@ func (r *Runner) getIgnoreFilePath() string { _ = os.MkdirAll(configDir, os.ModePerm) defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile) + return defIgnoreFilePath } - cwd, err := os.Getwd() if err != nil { return defIgnoreFilePath } cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile) - - cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath) - if os.IsNotExist(err) || cwdIfpInfo.IsDir() { - return defIgnoreFilePath - } return cwdIgnoreFilePath } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c8238664d..45d36be6c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -67,6 +67,7 @@ func New(options *types.Options) (*Runner, error) { runner.readNucleiIgnoreFile() } runner.catalog = catalog.New(runner.options.TemplatesDirectory) + runner.catalog.AppendIgnore(runner.templatesConfig.IgnorePaths) var reportingOptions *reporting.Options if options.ReportingConfig != "" { diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 0da92434d..4c7c38188 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -44,7 +44,7 @@ func (r *Runner) updateTemplates() error { configDir := path.Join(home, "/.config", "/nuclei") _ = os.MkdirAll(configDir, os.ModePerm) - templatesConfigFile := path.Join(home, nucleiConfigFilename) + templatesConfigFile := path.Join(configDir, nucleiConfigFilename) if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { config, readErr := readConfiguration() if err != nil { @@ -65,6 +65,7 @@ func (r *Runner) updateTemplates() error { } r.templatesConfig = currentConfig } + // Check if last checked for nuclei-ignore is more than 1 hours. // and if true, run the check. if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates { diff --git a/v2/pkg/catalog/catalogue.go b/v2/pkg/catalog/catalogue.go index cab2857ff..c085e5c2d 100644 --- a/v2/pkg/catalog/catalogue.go +++ b/v2/pkg/catalog/catalogue.go @@ -9,6 +9,10 @@ type Catalog struct { // New creates a new Catalog structure using provided input items func New(directory string) *Catalog { catalog := &Catalog{templatesDirectory: directory} - catalog.readNucleiIgnoreFile() return catalog } + +// AppendIgnore appends to the catalog store ignore list. +func (c *Catalog) AppendIgnore(list []string) { + c.ignoreFiles = append(c.ignoreFiles, list...) +} diff --git a/v2/pkg/catalog/ignore.go b/v2/pkg/catalog/ignore.go index b727ecd80..77e94525d 100644 --- a/v2/pkg/catalog/ignore.go +++ b/v2/pkg/catalog/ignore.go @@ -1,37 +1,11 @@ package catalog import ( - "bufio" - "os" - "path" "strings" "github.com/projectdiscovery/gologger" ) -const nucleiIgnoreFile = ".nuclei-ignore" - -// readNucleiIgnoreFile reads the nuclei ignore file marking it in map -func (c *Catalog) readNucleiIgnoreFile() { - file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile)) - if err != nil { - return - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if strings.HasPrefix(text, "#") { - continue - } - c.ignoreFiles = append(c.ignoreFiles, text) - } -} - // checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules. func (c *Catalog) checkIfInNucleiIgnore(item string) bool { if c.templatesDirectory == "" { From 64c8c33a4fca72d659fd3b5641b0a6970b298c9c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Apr 2021 01:35:32 +0530 Subject: [PATCH 03/15] Fixed some bugs + honor UA --- v2/cmd/nuclei/main.go | 1 - v2/internal/runner/runner.go | 6 ++-- v2/internal/testutils/testutils.go | 1 - v2/pkg/catalog/find.go | 10 +++--- v2/pkg/protocols/common/protocolinit/init.go | 38 ++++++++++++++++++++ v2/pkg/protocols/headless/engine/engine.go | 4 +-- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/types/types.go | 2 -- 8 files changed, 48 insertions(+), 16 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 67543443c..a26d41c8a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -52,7 +52,6 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output") set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") - set.BoolVarP(&options.RandomAgent, "random-agent", "ra", true, "Use randomly selected HTTP User-Agent header value") set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses") set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 45d36be6c..18d2ba117 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -236,8 +236,8 @@ func (r *Runner) RunEnumeration() { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates) - excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates) + includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false) + excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true) // defaults to all templates allTemplates := includedTemplates @@ -261,7 +261,7 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false) availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 9f448a94a..93c8b2fd9 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -19,7 +19,6 @@ func Init(options *types.Options) { // DefaultOptions is the default options structure for nuclei during mocking. var DefaultOptions = &types.Options{ - RandomAgent: false, Metrics: false, Debug: false, DebugRequests: false, diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 2575b0cb3..2a9922bf6 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -12,7 +12,7 @@ import ( ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string) []string { +func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -23,6 +23,10 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string { gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) } for _, path := range paths { + if !noCheckIgnore && c.checkIfInNucleiIgnore(path) { + gologger.Error().Msgf("Could not find template '%s': %s (nuclei-ignore)\n", path, err) + continue + } if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) @@ -139,10 +143,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru }, Callback: func(path string, d *godirwalk.Dirent) error { if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - if c.checkIfInNucleiIgnore(path) { - return nil - } - if _, ok := processed[path]; !ok { results = append(results, path) processed[path] = struct{}{} diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index db024ceaa..5f58cbf6b 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -1,6 +1,7 @@ package protocolinit import ( + "github.com/corpix/uarand" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" @@ -9,6 +10,8 @@ import ( // Init initializes the client pools for the protocols func Init(options *types.Options) error { + uarand.Default = uarand.NewWithCustomList(userAgents) + if err := dnsclientpool.Init(options); err != nil { return err } @@ -20,3 +23,38 @@ func Init(options *types.Options) error { } return nil } + +var userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2762.73 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2656.18 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", +} diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 646273fdf..8151889db 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -73,9 +73,7 @@ func New(options *types.Options) (*Browser, error) { customAgent = parts[1] } } - if options.RandomAgent { - customAgent = uarand.GetRandom() - } + customAgent = uarand.GetRandom() httpclient, err := newhttpClient(options) if err != nil { return nil, err diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 1d3fa7b7e..58b7c7395 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -103,7 +103,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if r.options.Options.RandomAgent { + if _, ok := r.customHeaders["User-Agent"]; !ok { r.customHeaders["User-Agent"] = uarand.GetRandom() } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 53464684b..08f7bebdb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -73,8 +73,6 @@ type Options struct { ShowBrowser bool // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool - // RandomAgent generates random User-Agent - RandomAgent bool // Metrics enables display of metrics via an http endpoint Metrics bool // Debug mode allows debugging request/responses for the engine From 8d3163a842da3ab57b4e3ce026b1a1c32e6402e5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Apr 2021 01:37:40 +0530 Subject: [PATCH 04/15] Fixed a small bug --- v2/internal/runner/runner.go | 6 +++--- v2/pkg/catalog/find.go | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 18d2ba117..45d36be6c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -236,8 +236,8 @@ func (r *Runner) RunEnumeration() { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false) - excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true) + includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates) + excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates) // defaults to all templates allTemplates := includedTemplates @@ -261,7 +261,7 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false) + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 2a9922bf6..f0a54b637 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -12,7 +12,7 @@ import ( ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string { +func (c *Catalog) GetTemplatesPath(definitions []string) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -23,10 +23,6 @@ func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []s gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) } for _, path := range paths { - if !noCheckIgnore && c.checkIfInNucleiIgnore(path) { - gologger.Error().Msgf("Could not find template '%s': %s (nuclei-ignore)\n", path, err) - continue - } if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) From 973e0be3e196333feee42cefbb3041d54c33a49d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Apr 2021 01:39:25 +0530 Subject: [PATCH 05/15] MIsc --- v2/internal/runner/runner.go | 6 +++--- v2/pkg/catalog/find.go | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 45d36be6c..18d2ba117 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -236,8 +236,8 @@ func (r *Runner) RunEnumeration() { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates) - excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates) + includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false) + excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true) // defaults to all templates allTemplates := includedTemplates @@ -261,7 +261,7 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false) availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index f0a54b637..4529d2e50 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -12,7 +12,7 @@ import ( ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string) []string { +func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -23,6 +23,9 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string { gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) } for _, path := range paths { + if !noCheckIgnore && c.checkIfInNucleiIgnore(path) { + continue + } if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) From 8233efe9213e9817e646c15c484fa6a4fb8cb2ee Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Apr 2021 01:43:48 +0530 Subject: [PATCH 06/15] Fixed custom user agent issue in headless --- v2/pkg/protocols/headless/engine/engine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 8151889db..8da632688 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -73,7 +73,9 @@ func New(options *types.Options) (*Browser, error) { customAgent = parts[1] } } - customAgent = uarand.GetRandom() + if customAgent == "" { + customAgent = uarand.GetRandom() + } httpclient, err := newhttpClient(options) if err != nil { return nil, err From 5ea05b385a27ca9898562ccd7a5b09ec9237650c Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 1 Apr 2021 01:46:40 +0530 Subject: [PATCH 07/15] version update --- v2/internal/runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 6e5e0460a..45fb24067 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,11 +7,11 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.3.2 + /_/ /_/\__,_/\___/_/\___/_/ v2.3.3 ` // Version is the current version of nuclei -const Version = `2.3.2` +const Version = `2.3.3` // showBanner is used to show the banner to the user func showBanner() { From 6804bd79e8f6f410ba504282a5198a0934bad4a2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 2 Apr 2021 18:40:58 +0530 Subject: [PATCH 08/15] New ignore functionality + error to warning --- v2/internal/runner/config.go | 28 ++++++++++++++++------------ v2/internal/runner/options.go | 1 - v2/pkg/catalog/ignore.go | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index c68b7d594..ac5109685 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -1,14 +1,14 @@ package runner import ( - "bufio" "os" "path" "regexp" - "strings" "time" jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v2" ) // nucleiConfig contains some configuration options for nuclei @@ -83,25 +83,29 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error { const nucleiIgnoreFile = ".nuclei-ignore" +type ignoreFile struct { + Tags []string `yaml:"tags"` + Files []string `yaml:"files"` +} + // readNucleiIgnoreFile reads the nuclei ignore file marking it in map func (r *Runner) readNucleiIgnoreFile() { file, err := os.Open(r.getIgnoreFilePath()) if err != nil { + gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) return } defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if strings.HasPrefix(text, "#") { - continue - } - r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text) + ignore := &ignoreFile{} + if err := yaml.NewDecoder(file).Decode(ignore); err != nil { + gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) + return } + for _, file := range ignore.Files { + r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, file) + } + r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...) } // getIgnoreFilePath returns the ignore file path for the runner diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f47c4f111..61864cbfc 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -25,7 +25,6 @@ func ParseOptions(options *types.Options) { // Show the user the banner showBanner() - options.ExcludeTags = append(options.ExcludeTags, "dos") if options.Version { gologger.Info().Msgf("Current Version: %s\n", Version) os.Exit(0) diff --git a/v2/pkg/catalog/ignore.go b/v2/pkg/catalog/ignore.go index 77e94525d..c3ecb9910 100644 --- a/v2/pkg/catalog/ignore.go +++ b/v2/pkg/catalog/ignore.go @@ -25,7 +25,7 @@ func (c *Catalog) checkIfInNucleiIgnore(item string) bool { } } if matched { - gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item) + gologger.Warning().Msgf("Excluding %s due to nuclei-ignore filter", item) return true } return false From 8da933af53b1a77a47969437d032ac355a92732b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 2 Apr 2021 18:44:28 +0530 Subject: [PATCH 09/15] Misc --- v2/internal/runner/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 4c7c38188..7e09cf73c 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -317,7 +317,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults paths := strings.Split(directory, "/") finalPath := strings.Join(paths[1:], "/") - if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { + if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { continue } results.totalCount++ From 459fe31e0bf47ef2087d90059fb4604279562d76 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 2 Apr 2021 21:32:56 +0530 Subject: [PATCH 10/15] Lint error fix --- v2/internal/runner/config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index ac5109685..38643b369 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -102,10 +102,8 @@ func (r *Runner) readNucleiIgnoreFile() { gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) return } - for _, file := range ignore.Files { - r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, file) - } r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...) + r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, ignore.Files...) } // getIgnoreFilePath returns the ignore file path for the runner From dbaa573b976d68961402cdbbce366a920ae6c1b0 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 2 Apr 2021 22:06:59 +0530 Subject: [PATCH 11/15] version update --- v2/internal/runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 45fb24067..b440ac342 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,11 +7,11 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.3.3 + /_/ /_/\__,_/\___/_/\___/_/ v2.3.4 ` // Version is the current version of nuclei -const Version = `2.3.3` +const Version = `2.3.4` // showBanner is used to show the banner to the user func showBanner() { From 954aec0907dbf47a74361b25b2dd7d4b56f23033 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Apr 2021 07:45:55 +0000 Subject: [PATCH 12/15] chore(deps): bump golangci/golangci-lint-action from v2 to v2.5.2 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from v2 to v2.5.2. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v2...5c56cd6c9dc07901af25baab6f2b0d9f3b7c3018) Signed-off-by: dependabot[bot] --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7dc47bdcc..2020dd10a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,7 +13,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v2.5.2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. version: v1.33 From 781a71e3d5085e1ffd6e1db7b4f7a1479cb39d42 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 8 Apr 2021 23:45:22 +0530 Subject: [PATCH 13/15] FAQ update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0cc3b6afd..49cfb9fda 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ For Developers • Documentation • Credits • - License • - Join Discord + FAQs • + Join Discord

--- From 7f82270ea71cda77df46876a5998e8c88cc23c66 Mon Sep 17 00:00:00 2001 From: "Gia. Bui Dai" Date: Tue, 13 Apr 2021 13:27:36 +0700 Subject: [PATCH 14/15] Fix can't set user agent in templates --- v2/pkg/protocols/http/build_request.go | 3 ++- v2/pkg/protocols/http/http.go | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 0bdcabcaf..8acc6d3d1 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" @@ -182,7 +183,7 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte if r.request.Body != "" { req.Body = ioutil.NopCloser(strings.NewReader(r.request.Body)) } - setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)") + setHeader(req, "User-Agent", uarand.GetRandom()) // Only set these headers on non raw requests if len(r.request.Raw) == 0 { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 58b7c7395..2f8c4212c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -3,7 +3,6 @@ package http import ( "strings" - "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -102,10 +101,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } - // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if _, ok := r.customHeaders["User-Agent"]; !ok { - r.customHeaders["User-Agent"] = uarand.GetRandom() - } if r.Body != "" && !strings.Contains(r.Body, "\r\n") { r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n") From e229a3eccc2e7bc0ca575fbd94d590da8bf765b3 Mon Sep 17 00:00:00 2001 From: "Gia. Bui Dai" Date: Tue, 13 Apr 2021 13:28:29 +0700 Subject: [PATCH 15/15] fix typos --- v2/pkg/protocols/http/build_request.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8acc6d3d1..c372a6fea 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -114,11 +114,11 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st // makeHTTPRequestFromRaw creates a *http.Request from a raw request func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { - return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) + return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads) } -// handleRawWithPaylods handles raw requests along with paylaods -func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { +// handleRawWithPayloads handles raw requests along with payloads +func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { // Combine the template payloads along with base // request values. finalValues := generators.MergeMaps(generatorValues, values)