diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7dc47bdcc..2020dd10a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,7 +13,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v2.5.2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. version: v1.33 diff --git a/README.md b/README.md index 0cc3b6afd..49cfb9fda 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ For DevelopersDocumentationCredits • - License • - Join Discord + FAQs • + Join Discord

--- diff --git a/v2/Makefile b/v2/Makefile new file mode 100644 index 000000000..247e7de43 --- /dev/null +++ b/v2/Makefile @@ -0,0 +1,14 @@ +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GOMOD=$(GOCMD) mod +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get + +all: build +build: + $(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go +test: + $(GOTEST) -v ./... +tidy: + $(GOMOD) tidy \ No newline at end of file diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 9e961f07e..a26d41c8a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -52,7 +52,6 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output") set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") - set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value") set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses") set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request") diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 6e5e0460a..b440ac342 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,11 +7,11 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.3.2 + /_/ /_/\__,_/\___/_/\___/_/ v2.3.4 ` // Version is the current version of nuclei -const Version = `2.3.2` +const Version = `2.3.4` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index c7634388e..38643b369 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -1,14 +1,14 @@ package runner import ( - "bufio" "os" "path" "regexp" - "strings" "time" jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v2" ) // nucleiConfig contains some configuration options for nuclei @@ -83,25 +83,27 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error { const nucleiIgnoreFile = ".nuclei-ignore" +type ignoreFile struct { + Tags []string `yaml:"tags"` + Files []string `yaml:"files"` +} + // readNucleiIgnoreFile reads the nuclei ignore file marking it in map func (r *Runner) readNucleiIgnoreFile() { file, err := os.Open(r.getIgnoreFilePath()) if err != nil { + gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) return } defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if strings.HasPrefix(text, "#") { - continue - } - r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text) + ignore := &ignoreFile{} + if err := yaml.NewDecoder(file).Decode(ignore); err != nil { + gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) + return } + r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...) + r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, ignore.Files...) } // getIgnoreFilePath returns the ignore file path for the runner @@ -114,17 +116,12 @@ func (r *Runner) getIgnoreFilePath() string { _ = os.MkdirAll(configDir, os.ModePerm) defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile) + return defIgnoreFilePath } - cwd, err := os.Getwd() if err != nil { return defIgnoreFilePath } cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile) - - cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath) - if os.IsNotExist(err) || cwdIfpInfo.IsDir() { - return defIgnoreFilePath - } return cwdIgnoreFilePath } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f47c4f111..61864cbfc 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -25,7 +25,6 @@ func ParseOptions(options *types.Options) { // Show the user the banner showBanner() - options.ExcludeTags = append(options.ExcludeTags, "dos") if options.Version { gologger.Info().Msgf("Current Version: %s\n", Version) os.Exit(0) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c8238664d..18d2ba117 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -67,6 +67,7 @@ func New(options *types.Options) (*Runner, error) { runner.readNucleiIgnoreFile() } runner.catalog = catalog.New(runner.options.TemplatesDirectory) + runner.catalog.AppendIgnore(runner.templatesConfig.IgnorePaths) var reportingOptions *reporting.Options if options.ReportingConfig != "" { @@ -235,8 +236,8 @@ func (r *Runner) RunEnumeration() { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates) - excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates) + includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false) + excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true) // defaults to all templates allTemplates := includedTemplates @@ -260,7 +261,7 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false) availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 0da92434d..7e09cf73c 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -44,7 +44,7 @@ func (r *Runner) updateTemplates() error { configDir := path.Join(home, "/.config", "/nuclei") _ = os.MkdirAll(configDir, os.ModePerm) - templatesConfigFile := path.Join(home, nucleiConfigFilename) + templatesConfigFile := path.Join(configDir, nucleiConfigFilename) if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { config, readErr := readConfiguration() if err != nil { @@ -65,6 +65,7 @@ func (r *Runner) updateTemplates() error { } r.templatesConfig = currentConfig } + // Check if last checked for nuclei-ignore is more than 1 hours. // and if true, run the check. if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates { @@ -316,7 +317,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults paths := strings.Split(directory, "/") finalPath := strings.Join(paths[1:], "/") - if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { + if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { continue } results.totalCount++ diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 9f448a94a..93c8b2fd9 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -19,7 +19,6 @@ func Init(options *types.Options) { // DefaultOptions is the default options structure for nuclei during mocking. var DefaultOptions = &types.Options{ - RandomAgent: false, Metrics: false, Debug: false, DebugRequests: false, diff --git a/v2/pkg/catalog/catalogue.go b/v2/pkg/catalog/catalogue.go index cab2857ff..c085e5c2d 100644 --- a/v2/pkg/catalog/catalogue.go +++ b/v2/pkg/catalog/catalogue.go @@ -9,6 +9,10 @@ type Catalog struct { // New creates a new Catalog structure using provided input items func New(directory string) *Catalog { catalog := &Catalog{templatesDirectory: directory} - catalog.readNucleiIgnoreFile() return catalog } + +// AppendIgnore appends to the catalog store ignore list. +func (c *Catalog) AppendIgnore(list []string) { + c.ignoreFiles = append(c.ignoreFiles, list...) +} diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 2575b0cb3..4529d2e50 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -12,7 +12,7 @@ import ( ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string) []string { +func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -23,6 +23,9 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string { gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) } for _, path := range paths { + if !noCheckIgnore && c.checkIfInNucleiIgnore(path) { + continue + } if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) @@ -139,10 +142,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru }, Callback: func(path string, d *godirwalk.Dirent) error { if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - if c.checkIfInNucleiIgnore(path) { - return nil - } - if _, ok := processed[path]; !ok { results = append(results, path) processed[path] = struct{}{} diff --git a/v2/pkg/catalog/ignore.go b/v2/pkg/catalog/ignore.go index b727ecd80..c3ecb9910 100644 --- a/v2/pkg/catalog/ignore.go +++ b/v2/pkg/catalog/ignore.go @@ -1,37 +1,11 @@ package catalog import ( - "bufio" - "os" - "path" "strings" "github.com/projectdiscovery/gologger" ) -const nucleiIgnoreFile = ".nuclei-ignore" - -// readNucleiIgnoreFile reads the nuclei ignore file marking it in map -func (c *Catalog) readNucleiIgnoreFile() { - file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile)) - if err != nil { - return - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if strings.HasPrefix(text, "#") { - continue - } - c.ignoreFiles = append(c.ignoreFiles, text) - } -} - // checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules. func (c *Catalog) checkIfInNucleiIgnore(item string) bool { if c.templatesDirectory == "" { @@ -51,7 +25,7 @@ func (c *Catalog) checkIfInNucleiIgnore(item string) bool { } } if matched { - gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item) + gologger.Warning().Msgf("Excluding %s due to nuclei-ignore filter", item) return true } return false diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index db024ceaa..5f58cbf6b 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -1,6 +1,7 @@ package protocolinit import ( + "github.com/corpix/uarand" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" @@ -9,6 +10,8 @@ import ( // Init initializes the client pools for the protocols func Init(options *types.Options) error { + uarand.Default = uarand.NewWithCustomList(userAgents) + if err := dnsclientpool.Init(options); err != nil { return err } @@ -20,3 +23,38 @@ func Init(options *types.Options) error { } return nil } + +var userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2762.73 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2656.18 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", +} diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 646273fdf..8da632688 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -73,7 +73,7 @@ func New(options *types.Options) (*Browser, error) { customAgent = parts[1] } } - if options.RandomAgent { + if customAgent == "" { customAgent = uarand.GetRandom() } httpclient, err := newhttpClient(options) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 0bdcabcaf..c372a6fea 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" @@ -113,11 +114,11 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st // makeHTTPRequestFromRaw creates a *http.Request from a raw request func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { - return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) + return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads) } -// handleRawWithPaylods handles raw requests along with paylaods -func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { +// handleRawWithPayloads handles raw requests along with payloads +func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { // Combine the template payloads along with base // request values. finalValues := generators.MergeMaps(generatorValues, values) @@ -182,7 +183,7 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte if r.request.Body != "" { req.Body = ioutil.NopCloser(strings.NewReader(r.request.Body)) } - setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)") + setHeader(req, "User-Agent", uarand.GetRandom()) // Only set these headers on non raw requests if len(r.request.Raw) == 0 { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 1d3fa7b7e..2f8c4212c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -3,7 +3,6 @@ package http import ( "strings" - "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -102,10 +101,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } - // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if r.options.Options.RandomAgent { - r.customHeaders["User-Agent"] = uarand.GetRandom() - } if r.Body != "" && !strings.Contains(r.Body, "\r\n") { r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n") diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 53464684b..08f7bebdb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -73,8 +73,6 @@ type Options struct { ShowBrowser bool // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool - // RandomAgent generates random User-Agent - RandomAgent bool // Metrics enables display of metrics via an http endpoint Metrics bool // Debug mode allows debugging request/responses for the engine