From 31ce4b12cdce9bd83dac12de1fd1bbbf8f85ba17 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Mar 2021 12:08:31 +0530 Subject: [PATCH] Added workflows flag + new templates flag to run newly added ones --- v2/cmd/integration-test/workflow.go | 8 +++--- v2/cmd/nuclei/main.go | 3 +- v2/internal/runner/options.go | 2 +- v2/internal/runner/processor.go | 3 -- v2/internal/runner/runner.go | 41 ++++++++++++++++++++++++++-- v2/internal/runner/templates.go | 7 +++-- v2/internal/runner/update.go | 17 +++++++++++- v2/internal/testutils/integration.go | 23 ++++++++++++++++ v2/pkg/types/types.go | 6 ++-- 9 files changed, 93 insertions(+), 17 deletions(-) diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index f260ceb1e..35a2a786c 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -28,7 +28,7 @@ func (h *workflowBasic) Execute(filePath string) error { ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -50,7 +50,7 @@ func (h *workflowConditionMatched) Execute(filePath string) error { ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -72,7 +72,7 @@ func (h *workflowConditionUnmatch) Execute(filePath string) error { ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -94,7 +94,7 @@ func (h *workflowMatcherName) Execute(filePath string) error { ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug) if err != nil { return err } diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index fb90aec1e..88da079c0 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -41,6 +41,7 @@ based on templates offering massive extensibility and ease of use.`) set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei") set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.") + set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei") set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.") set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.") set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on") @@ -81,10 +82,10 @@ based on templates offering massive extensibility and ease of use.`) set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei") set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support") set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen") - set.BoolVarP(&options.Workflows, "workflows", "w", false, "Only run workflow templates with nuclei") set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line") set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") + set.BoolVar(&options.NewTemplates, "new-templates", false, "Only run newly added templates") _ = set.Parse() if cfgFile != "" { diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 368c63fd5..f901e9e05 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -82,7 +82,7 @@ func validateOptions(options *types.Options) error { if !options.TemplateList { // Check if a list of templates was provided and it exists - if len(options.Templates) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates { + if len(options.Templates) == 0 && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates { return errors.New("no template/templates provided") } } diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 5e5f55718..a00993b88 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -9,9 +9,6 @@ import ( // processTemplateWithList process a template on the URL list func (r *Runner) processTemplateWithList(template *templates.Template) bool { - if r.options.Workflows { - return false - } results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) r.hostMap.Scan(func(k, _ []byte) error { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f5900edef..7daaa7bc0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "path" "strings" "github.com/logrusorgru/aurora" @@ -82,7 +83,7 @@ func New(options *types.Options) (*Runner, error) { os.Exit(0) } - if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates { + if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates { os.Exit(0) } if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil { @@ -200,6 +201,13 @@ func (r *Runner) RunEnumeration() { if len(r.options.Templates) == 0 && len(r.options.Tags) > 0 { r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory) } + if r.options.NewTemplates { + templates, err := r.readNewTemplatesFile() + if err != nil { + gologger.Warning().Msgf("Could not get newly added templates: %s\n", err) + } + r.options.Templates = append(r.options.Templates, templates...) + } includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates) excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates) // defaults to all templates @@ -224,7 +232,10 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity) + + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) + availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) + availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) var unclusteredRequests int64 = 0 for _, template := range availableTemplates { @@ -264,6 +275,9 @@ func (r *Runner) RunEnumeration() { finalTemplates = append(finalTemplates, cluster...) } } + for _, workflows := range availableWorkflows { + finalTemplates = append(finalTemplates, workflows) + } var totalRequests int64 = 0 for _, t := range finalTemplates { @@ -275,7 +289,7 @@ func (r *Runner) RunEnumeration() { if totalRequests < unclusteredRequests { gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount) } - templateCount := originalTemplatesCount + templateCount := originalTemplatesCount + len(availableWorkflows) // 0 matches means no templates were found in directory if templateCount == 0 { @@ -325,3 +339,24 @@ func (r *Runner) RunEnumeration() { r.browser.Close() } } + +// readNewTemplatesFile reads newly added templates from directory if it exists +func (r *Runner) readNewTemplatesFile() ([]string, error) { + additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions") + file, err := os.Open(additionsFile) + if err != nil { + return nil, err + } + defer file.Close() + + templates := []string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + templates = append(templates, text) + } + return templates, nil +} diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 19e75a822..43d1326dc 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -14,7 +14,7 @@ import ( // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered // by severity, along with a flag indicating if workflows are present. -func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string) (parsedTemplates map[string]*templates.Template, workflowCount int) { +func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) { filterBySeverity := len(severities) > 0 gologger.Info().Msgf("Loading templates...") @@ -26,9 +26,12 @@ func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string) (pars gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err) continue } - if len(t.Workflows) == 0 && r.options.Workflows { + if len(t.Workflows) == 0 && workflows { continue // don't print if user only wants to run workflows } + if len(t.Workflows) > 0 && !workflows { + continue // don't print workflow if user only wants to run templates + } if len(t.Workflows) > 0 { workflowCount++ } diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index d98782a6b..622784ca3 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -7,7 +7,6 @@ import ( "context" "crypto/md5" "encoding/hex" - "errors" "fmt" "io" "io/ioutil" @@ -22,6 +21,7 @@ import ( "github.com/blang/semver" "github.com/google/go-github/v32/github" "github.com/olekukonko/tablewriter" + "github.com/pkg/errors" "github.com/projectdiscovery/gologger" ) @@ -221,6 +221,21 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU r.printUpdateChangelog(results, version) checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum") err = writeTemplatesChecksum(checksumFile, results.checksums) + if err != nil { + return nil, errors.Wrap(err, "could not write checksum") + } + + // Write the additions to a cached file for new runs. + additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions") + buffer := &bytes.Buffer{} + for _, addition := range results.additions { + buffer.WriteString(addition) + buffer.WriteString("\n") + } + err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm) + if err != nil { + return nil, errors.Wrap(err, "could not write new additions file") + } return results, err } diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 74deecb5c..47d7e477b 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -30,6 +30,29 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( return parts, nil } +// RunNucleiWorkflowAndGetResults returns a list of results for a workflow +func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei", "-w", template, "-target", url) + if debug { + cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug") + cmd.Stderr = os.Stderr + } + cmd.Args = append(cmd.Args, extra...) + + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} + // TestCase is a single integration test case type TestCase interface { // Execute executes a test case and returns any errors if occurred diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index e9820c597..507d4bec3 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -8,6 +8,8 @@ type Options struct { // can be specified with -l flag and -tags can be used in combination with // the -l flag. Tags goflags.StringSlice + // Workflows specifies any workflows to run by nuclei + Workflows goflags.StringSlice // Templates specifies the template/templates to use Templates goflags.StringSlice // ExcludedTemplates specifies the template/templates to exclude @@ -65,8 +67,6 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool - // Workflows specifies if only to execute workflows (no normal templates will be run) - Workflows bool // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool // RandomAgent generates random User-Agent @@ -107,4 +107,6 @@ type Options struct { NoMeta bool // Project is used to avoid sending same HTTP request multiple times Project bool + // NewTemplates only runs newly added templates from the repository + NewTemplates bool }