From 54ada7735db98373420241162440e359f8feaa74 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Tue, 14 Jul 2020 00:01:46 +0200 Subject: [PATCH 1/7] Add generic multiStringFlag option flag, update usage string --- v2/internal/runner/options.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index baab72b9e..5f252ca6b 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -12,7 +12,7 @@ import ( // the template requesting process. type Options struct { Debug bool // Debug mode allows debugging request/responses for the engine - Templates string // Signature specifies the template/templates to use + Templates multiStringFlag // Signature specifies the template/templates to use Target string // Target is a single URL/Domain to scan usng a template Targets string // Targets specifies the targets to scan using templates. Threads int // Thread controls the number of concurrent requests to make. @@ -33,12 +33,23 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process } +type multiStringFlag []string + +func (m *multiStringFlag) String() string { + return "" +} + +func (m *multiStringFlag) Set(value string) error { + *m = append(*m, value) + return nil +} + // ParseOptions parses the command line flags provided by a user func ParseOptions() *Options { options := &Options{} flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") - flag.StringVar(&options.Templates, "t", "", "Template input file/files to run on host") + flag.Var(&options.Templates, "t","Template input file/files to run on host. Can be used multiple times.") flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on") flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server") From 9d7303549b1e8646e1b4741de79aa2f146595216 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Tue, 14 Jul 2020 00:04:19 +0200 Subject: [PATCH 2/7] Refactor enumeration driver and streamline input processing --- v2/internal/runner/runner.go | 153 +++++++++++++++++++-------------- v2/internal/runner/validate.go | 2 +- 2 files changed, 91 insertions(+), 64 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 84f0d041f..705651ae4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -42,7 +42,7 @@ func New(options *Options) (*Runner, error) { if err := runner.updateTemplates(); err != nil { gologger.Warningf("Could not update templates: %s\n", err) } - if (options.Templates == "" || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates { + if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates { os.Exit(0) } @@ -86,84 +86,111 @@ func (r *Runner) Close() { os.Remove(r.tempFile) } +func isFilePath(path string) (bool, error) { + info, err := os.Stat(path) + if err != nil { + return false, err + } + return info.Mode().IsRegular(), nil +} + +func (r *Runner) translateToAbsolutePath(path string) (string, error) { + if r.isRelative(path) { + newPath, err := r.resolvePath(path) + if err != nil { + return "", err + } + return newPath, nil + } + return path, nil +} + +func isNewPath(path string, pathMap map[string]bool) bool { + if _, already := pathMap[path]; already { + gologger.Warningf("Skipping already specified path '%s'", path) + return false + } + return true +} + // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { - var err error + // keeps track of processed dirs and files + processed := make(map[string]bool) + allTemplates := []string{} - // Check if the template is an absolute path or relative path. - // If the path is absolute, use it. Otherwise, - if r.isRelative(r.options.Templates) { - newPath, err := r.resolvePath(r.options.Templates) + // parses user input, handle file/directory cases and produce a list of unique templates + for _, t := range r.options.Templates { + // determine file/directory + isFile, err := isFilePath(t) if err != nil { - gologger.Errorf("Could not find template file '%s': %s\n", r.options.Templates, err) - return + gologger.Errorf("Could not stat '%s': %s\n", t, err) + continue } - r.options.Templates = newPath - } - // Single yaml provided - if strings.HasSuffix(r.options.Templates, ".yaml") { - t, err := r.parse(r.options.Templates) - switch t.(type) { - case *templates.Template: - var results bool - template := t.(*templates.Template) - // process http requests - for _, request := range template.RequestsHTTP { - results = r.processTemplateRequest(template, request) - } - // process dns requests - for _, request := range template.RequestsDNS { - dnsResults := r.processTemplateRequest(template, request) - if !results { - results = dnsResults - } - } - - if !results { - if r.output != nil { - outputFile := r.output.Name() - r.output.Close() - os.Remove(outputFile) - } - gologger.Infof("No results found for the template. Happy hacking!") - } - case *workflows.Workflow: - workflow := t.(*workflows.Workflow) - r.ProcessWorkflowWithList(workflow) - default: - gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err) + // convert relative to absolute path + absPath, err := r.translateToAbsolutePath(t) + if err != nil { + gologger.Errorf("Could not find template file '%s': %s\n", t, err) + continue } - return - } - // If the template passed is a directory - matches := []string{} + // test for uniqueness + if !isNewPath(absPath, processed) { + continue + } - // Recursively walk down the Templates directory and run all the template file checks - err = godirwalk.Walk(r.options.Templates, &godirwalk.Options{ - Callback: func(path string, d *godirwalk.Dirent) error { - if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - matches = append(matches, path) + // mark this absolute path as processed + // - if it's a file, we'll never process it again + // - if it's a dir, we'll never walk it again + processed[absPath] = true + + if isFile { + allTemplates = append(allTemplates, absPath) + } else { + matches := []string{} + + // Recursively walk down the Templates directory and run all the template file checks + err = godirwalk.Walk(absPath, &godirwalk.Options{ + Callback: func(path string, d *godirwalk.Dirent) error { + if !d.IsDir() && strings.HasSuffix(path, ".yaml") { + if isNewPath(path, processed) { + matches = append(matches, path) + processed[path] = true + } + } + return nil + }, + ErrorCallback: func(path string, err error) godirwalk.ErrorAction { + return godirwalk.SkipNode + }, + Unsorted: true, + }) + + // directory couldn't be walked + if err != nil { + gologger.Warningf("Could not find templates in directory '%s': %s\n", absPath, err) + continue } - return nil - }, - ErrorCallback: func(path string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Unsorted: true, - }) - if err != nil { - gologger.Fatalf("Could not find templates in directory '%s': %s\n", r.options.Templates, err) + + // couldn't find templates in directory + if len(matches) == 0 { + gologger.Warningf("Error, no templates were found in '%s'.\n", absPath) + } + + allTemplates = append(allTemplates, matches...) + } } + // 0 matches means no templates were found in directory - if len(matches) == 0 { - gologger.Fatalf("Error, no templates found in directory: '%s'\n", r.options.Templates) + if len(allTemplates) == 0 { + gologger.Fatalf("Error, no templates were found.\n") } + // run with the specified templates var results bool - for _, match := range matches { + for _, match := range allTemplates { t, err := r.parse(match) switch t.(type) { case *templates.Template: diff --git a/v2/internal/runner/validate.go b/v2/internal/runner/validate.go index 67233438d..2fb0e00c0 100644 --- a/v2/internal/runner/validate.go +++ b/v2/internal/runner/validate.go @@ -15,7 +15,7 @@ func (options *Options) validateOptions() error { } // Check if a list of templates was provided and it exists - if options.Templates == "" && !options.UpdateTemplates { + if len(options.Templates) == 0 && !options.UpdateTemplates { return errors.New("no template/templates provided") } From f376a7d9cda057e70dc9cc3c836ebbcb163469a3 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Tue, 14 Jul 2020 00:10:08 +0200 Subject: [PATCH 3/7] Move no-results logic as Workflows will not return any --- v2/internal/runner/runner.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 705651ae4..2088fc7cd 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -207,21 +207,22 @@ func (r *Runner) RunEnumeration() { results = httpResults } } + + if !results { + if r.output != nil { + outputFile := r.output.Name() + r.output.Close() + os.Remove(outputFile) + } + gologger.Infof("No results found for [%s]. Happy hacking!", template.ID) + } case *workflows.Workflow: workflow := t.(*workflows.Workflow) r.ProcessWorkflowWithList(workflow) default: - gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err) + gologger.Errorf("Could not parse file '%s': %s\n", match, err) } } - if !results { - if r.output != nil { - outputFile := r.output.Name() - r.output.Close() - os.Remove(outputFile) - } - gologger.Infof("No results found for the template. Happy hacking!") - } return } From 0833e21a179a7cb78e26556f4f7995bbccb03a8b Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 18 Jul 2020 18:55:52 +0200 Subject: [PATCH 4/7] Avoid adding empty array, ensure warning is shown to the user. --- v2/internal/runner/runner.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 3686faa60..0f06f4c32 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -171,13 +171,14 @@ func (r *Runner) RunEnumeration() { // directory couldn't be walked if err != nil { - gologger.Warningf("Could not find templates in directory '%s': %s\n", absPath, err) + gologger.Labelf("Could not find templates in directory '%s': %s\n", absPath, err) continue } // couldn't find templates in directory if len(matches) == 0 { - gologger.Warningf("Error, no templates were found in '%s'.\n", absPath) + gologger.Labelf("Error, no templates were found in '%s'.\n", absPath) + continue } allTemplates = append(allTemplates, matches...) From c161a385b07f2dcc197adeaef0171c7668cde018 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 18 Jul 2020 20:32:00 +0200 Subject: [PATCH 5/7] Do not report per-template empty results --- v2/internal/runner/runner.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0f06f4c32..892194fad 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -209,15 +209,6 @@ func (r *Runner) RunEnumeration() { results = httpResults } } - - if !results { - if r.output != nil { - outputFile := r.output.Name() - r.output.Close() - os.Remove(outputFile) - } - gologger.Infof("No results found for template [%s]. Happy hacking!", template.ID) - } case *workflows.Workflow: workflow := t.(*workflows.Workflow) r.ProcessWorkflowWithList(workflow) @@ -225,6 +216,14 @@ func (r *Runner) RunEnumeration() { gologger.Errorf("Could not parse file '%s': %s\n", match, err) } } + if !results { + if r.output != nil { + outputFile := r.output.Name() + r.output.Close() + os.Remove(outputFile) + } + gologger.Infof("No results found. Happy hacking!") + } return } From 0983e8b9fab2790d35704309f661fa34da96a44c Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 19 Jul 2020 14:04:49 +0200 Subject: [PATCH 6/7] Ensure path is resolved before using it --- v2/internal/runner/runner.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9defc8d32..3b4203d88 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -124,20 +124,20 @@ func (r *Runner) RunEnumeration() { // parses user input, handle file/directory cases and produce a list of unique templates for _, t := range r.options.Templates { - // determine file/directory - isFile, err := isFilePath(t) - if err != nil { - gologger.Errorf("Could not stat '%s': %s\n", t, err) - continue - } - - // convert relative to absolute path + // resolve and convert relative to absolute path absPath, err := r.translateToAbsolutePath(t) if err != nil { gologger.Errorf("Could not find template file '%s': %s\n", t, err) continue } + // determine file/directory + isFile, err := isFilePath(absPath) + if err != nil { + gologger.Errorf("Could not stat '%s': %s\n", absPath, err) + continue + } + // test for uniqueness if !isNewPath(absPath, processed) { continue From 838f8448372aa255a4831ad35c69625366561b89 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 19 Jul 2020 14:24:43 +0200 Subject: [PATCH 7/7] Better naming --- v2/internal/runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 3b4203d88..3b934079f 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -96,7 +96,7 @@ func isFilePath(path string) (bool, error) { return info.Mode().IsRegular(), nil } -func (r *Runner) translateToAbsolutePath(path string) (string, error) { +func (r *Runner) resolvePathIfRelative(path string) (string, error) { if r.isRelative(path) { newPath, err := r.resolvePath(path) if err != nil { @@ -125,7 +125,7 @@ func (r *Runner) RunEnumeration() { // parses user input, handle file/directory cases and produce a list of unique templates for _, t := range r.options.Templates { // resolve and convert relative to absolute path - absPath, err := r.translateToAbsolutePath(t) + absPath, err := r.resolvePathIfRelative(t) if err != nil { gologger.Errorf("Could not find template file '%s': %s\n", t, err) continue