diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 67543443c..a26d41c8a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -52,7 +52,6 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output") set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") - set.BoolVarP(&options.RandomAgent, "random-agent", "ra", true, "Use randomly selected HTTP User-Agent header value") set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses") set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 45d36be6c..18d2ba117 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -236,8 +236,8 @@ func (r *Runner) RunEnumeration() { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates) - excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates) + includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false) + excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true) // defaults to all templates allTemplates := includedTemplates @@ -261,7 +261,7 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false) availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 9f448a94a..93c8b2fd9 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -19,7 +19,6 @@ func Init(options *types.Options) { // DefaultOptions is the default options structure for nuclei during mocking. var DefaultOptions = &types.Options{ - RandomAgent: false, Metrics: false, Debug: false, DebugRequests: false, diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 2575b0cb3..2a9922bf6 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -12,7 +12,7 @@ import ( ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string) []string { +func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -23,6 +23,10 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string { gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) } for _, path := range paths { + if !noCheckIgnore && c.checkIfInNucleiIgnore(path) { + gologger.Error().Msgf("Could not find template '%s': %s (nuclei-ignore)\n", path, err) + continue + } if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) @@ -139,10 +143,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru }, Callback: func(path string, d *godirwalk.Dirent) error { if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - if c.checkIfInNucleiIgnore(path) { - return nil - } - if _, ok := processed[path]; !ok { results = append(results, path) processed[path] = struct{}{} diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index db024ceaa..5f58cbf6b 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -1,6 +1,7 @@ package protocolinit import ( + "github.com/corpix/uarand" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" @@ -9,6 +10,8 @@ import ( // Init initializes the client pools for the protocols func Init(options *types.Options) error { + uarand.Default = uarand.NewWithCustomList(userAgents) + if err := dnsclientpool.Init(options); err != nil { return err } @@ -20,3 +23,38 @@ func Init(options *types.Options) error { } return nil } + +var userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2762.73 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2656.18 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", +} diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 646273fdf..8151889db 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -73,9 +73,7 @@ func New(options *types.Options) (*Browser, error) { customAgent = parts[1] } } - if options.RandomAgent { - customAgent = uarand.GetRandom() - } + customAgent = uarand.GetRandom() httpclient, err := newhttpClient(options) if err != nil { return nil, err diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 1d3fa7b7e..58b7c7395 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -103,7 +103,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if r.options.Options.RandomAgent { + if _, ok := r.customHeaders["User-Agent"]; !ok { r.customHeaders["User-Agent"] = uarand.GetRandom() } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 53464684b..08f7bebdb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -73,8 +73,6 @@ type Options struct { ShowBrowser bool // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool - // RandomAgent generates random User-Agent - RandomAgent bool // Metrics enables display of metrics via an http endpoint Metrics bool // Debug mode allows debugging request/responses for the engine