From 448388a825dffe23ff7e60ff9567413384fa2fca Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:21:18 +0300 Subject: [PATCH 1/7] Added TODOs next to un-used parameters --- v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/file/request.go | 2 +- .../protocols/headless/engine/page_actions.go | 34 +++++++++---------- v2/pkg/protocols/headless/request.go | 2 +- .../network/networkclientpool/clientpool.go | 4 +-- v2/pkg/protocols/network/request.go | 2 +- v2/pkg/protocols/offlinehttp/request.go | 2 +- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 71ac0f5f4..0a9f84d6f 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,7 +12,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string if isURL(input) { diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index e775c57aa..d272e256b 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -15,7 +15,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(r.options.Options.BulkSize) err := r.getInputPaths(input, func(data string) { diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 36c532e21..f5b4b933e 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -84,7 +84,7 @@ type requestRule struct { } // ActionAddHeader executes a AddHeader action. -func (p *Page) ActionAddHeader(act *Action, out map[string]string) error { +func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") args := make(map[string]string) @@ -100,7 +100,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string) error { } // ActionSetHeader executes a SetHeader action. -func (p *Page) ActionSetHeader(act *Action, out map[string]string) error { +func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") args := make(map[string]string) @@ -116,7 +116,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string) error { } // ActionDeleteHeader executes a DeleteHeader action. -func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error { +func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") args := make(map[string]string) @@ -131,7 +131,7 @@ func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error { } // ActionSetBody executes a SetBody action. -func (p *Page) ActionSetBody(act *Action, out map[string]string) error { +func (p *Page) ActionSetBody(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") args := make(map[string]string) @@ -146,7 +146,7 @@ func (p *Page) ActionSetBody(act *Action, out map[string]string) error { } // ActionSetMethod executes an SetMethod action. -func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { +func (p *Page) ActionSetMethod(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") args := make(map[string]string) @@ -161,7 +161,7 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { } // NavigateURL executes an ActionLoadURL actions loading a URL for the page. -func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error { +func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL /*TODO review unused parameter*/) error { URL := action.GetArg("url") if URL == "" { return errors.New("invalid arguments provided") @@ -205,7 +205,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error { } // ClickElement executes click actions for an element. -func (p *Page) ClickElement(act *Action, out map[string]string) error { +func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, "could not get element") @@ -220,12 +220,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string) error { } // KeyboardAction executes a keyboard action on the page. -func (p *Page) KeyboardAction(act *Action, out map[string]string) error { +func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { return p.page.Keyboard.Press([]rune(act.GetArg("keys"))...) } // RightClickElement executes right click actions for an element. -func (p *Page) RightClickElement(act *Action, out map[string]string) error { +func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, "could not get element") @@ -266,7 +266,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error { } // InputElement executes input element actions for an element. -func (p *Page) InputElement(act *Action, out map[string]string) error { +func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { value := act.GetArg("value") if value == "" { return errors.New("invalid arguments provided") @@ -285,7 +285,7 @@ func (p *Page) InputElement(act *Action, out map[string]string) error { } // TimeInputElement executes time input on an element -func (p *Page) TimeInputElement(act *Action, out map[string]string) error { +func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { value := act.GetArg("value") if value == "" { return errors.New("invalid arguments provided") @@ -308,7 +308,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string) error { } // SelectInputElement executes select input statement action on a element -func (p *Page) SelectInputElement(act *Action, out map[string]string) error { +func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { value := act.GetArg("value") if value == "" { return errors.New("invalid arguments provided") @@ -333,7 +333,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string) error { } // WaitLoad waits for the page to load -func (p *Page) WaitLoad(act *Action, out map[string]string) error { +func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error { p.page.Timeout(1 * time.Second).WaitNavigation(proto.PageLifecycleEventNameDOMContentLoaded)() // Wait for the window.onload event and also wait for the network requests @@ -363,7 +363,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error { } // FilesInput acts with a file input element on page -func (p *Page) FilesInput(act *Action, out map[string]string) error { +func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, "could not get element") @@ -423,7 +423,7 @@ func (p *protoEvent) ProtoEvent() string { } // WaitEvent waits for an event to happen on the page. -func (p *Page) WaitEvent(act *Action, out map[string]string) error { +func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) error { event := act.GetArg("event") if event == "" { return errors.New("event not recognized") @@ -480,14 +480,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) { } // DebugAction enables debug action on a page. -func (p *Page) DebugAction(act *Action, out map[string]string) error { +func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { p.instance.browser.engine.SlowMotion(5 * time.Second) p.instance.browser.engine.Trace(true) return nil } // SleepAction sleeps on the page for a specified duration -func (p *Page) SleepAction(act *Action, out map[string]string) error { +func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { seconds := act.Data["duration"] if seconds == "" { seconds = "5" diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 7010fe027..54345d468 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -14,7 +14,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := r.options.Browser.NewInstance() if err != nil { r.options.Output.Request(r.options.TemplateID, input, "headless", err) diff --git a/v2/pkg/protocols/network/networkclientpool/clientpool.go b/v2/pkg/protocols/network/networkclientpool/clientpool.go index 127282d24..d35d4eb61 100644 --- a/v2/pkg/protocols/network/networkclientpool/clientpool.go +++ b/v2/pkg/protocols/network/networkclientpool/clientpool.go @@ -11,7 +11,7 @@ var ( ) // Init initializes the clientpool implementation -func Init(options *types.Options) error { +func Init(options *types.Options /*TODO review unused parameter*/) error { // Don't create clients if already created in past. if normalClient != nil { return nil @@ -29,6 +29,6 @@ func (c *Configuration) Hash() string { } // Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration) (*fastdialer.Dialer, error) { +func Get(options *types.Options, configuration *Configuration /*TODO review unused parameters*/) (*fastdialer.Dialer, error) { return normalClient, nil } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 98f9e744d..8c9362d44 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -21,7 +21,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "network", err) diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index f00acf9e0..27b10e9db 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -20,7 +20,7 @@ var _ protocols.Request = &Request{} const maxSize = 5 * 1024 * 1024 // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(r.options.Options.BulkSize) err := r.getInputPaths(input, func(data string) { From 40d88d230450a5b6bc941df4c477481d459363a6 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 31 Aug 2021 12:55:52 +0300 Subject: [PATCH 2/7] In-lined error checks, reduced scope of error variables, introduced new error variables instead of re-using them --- v2/cmd/integration-test/integration-test.go | 3 +- v2/cmd/integration-test/workflow.go | 18 +++++------ v2/internal/runner/options.go | 6 ++-- v2/internal/runner/update.go | 32 +++++++------------ v2/pkg/catalog/config/config.go | 3 +- v2/pkg/operators/common/dsl/dsl.go | 8 ++--- v2/pkg/operators/extractors/extract.go | 4 +-- v2/pkg/output/file_output_writer.go | 5 ++- v2/pkg/parsers/parser.go | 3 +- v2/pkg/projectfile/httputil.go | 9 ++---- v2/pkg/projectfile/project.go | 3 +- v2/pkg/protocols/common/executer/executer.go | 3 +- v2/pkg/protocols/dns/request.go | 3 +- v2/pkg/protocols/file/find.go | 6 ++-- v2/pkg/protocols/headless/engine/page.go | 10 +++--- .../protocols/headless/engine/page_actions.go | 3 +- v2/pkg/protocols/http/request.go | 3 +- v2/pkg/protocols/network/request.go | 6 ++-- v2/pkg/protocols/offlinehttp/find.go | 6 ++-- v2/pkg/reporting/exporters/disk/disk.go | 3 +- v2/pkg/templates/compile.go | 6 ++-- v2/pkg/workflows/execute.go | 6 ++-- 22 files changed, 55 insertions(+), 94 deletions(-) diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index ad416b5c4..baa743e8d 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -35,8 +35,7 @@ func main() { if customTest != "" && !strings.Contains(file, customTest) { continue // only run tests user asked } - err := test.Execute(file) - if err != nil { + if err := test.Execute(file); err != nil { fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err) errored = true } else { diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 35a2a786c..a8548e9b6 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -21,10 +21,10 @@ type workflowBasic struct{} // Executes executes a test case and returns an error if occurred func (h *workflowBasic) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() @@ -43,10 +43,10 @@ type workflowConditionMatched struct{} // Executes executes a test case and returns an error if occurred func (h *workflowConditionMatched) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() @@ -65,10 +65,10 @@ type workflowConditionUnmatch struct{} // Executes executes a test case and returns an error if occurred func (h *workflowConditionUnmatch) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() @@ -84,13 +84,13 @@ func (h *workflowConditionUnmatch) Execute(filePath string) error { type workflowMatcherName struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *workflowMatcherName) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index b72e1a7f1..0cd005463 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -83,13 +83,11 @@ func validateOptions(options *types.Options) error { } // Validate proxy options if provided - err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)") - if err != nil { + if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { return err } - err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)") - if err != nil { + if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { return err } diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index a0aa2bd3b..2243398f9 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -108,14 +108,12 @@ func (r *Runner) updateTemplates() error { gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) r.fetchLatestVersionsFromGithub() // also fetch latest versions - _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) - if err != nil { + if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { return err } r.templatesConfig.CurrentVersion = version.String() - err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore) - if err != nil { + if err := config.WriteConfiguration(r.templatesConfig, true, checkedIgnore); err != nil { return err } gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String()) @@ -163,12 +161,11 @@ func (r *Runner) updateTemplates() error { gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) r.fetchLatestVersionsFromGithub() - _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) - if err != nil { + if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { return err } - err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore) - if err != nil { + + if err := config.WriteConfiguration(r.templatesConfig, true, checkedIgnore); err != nil { return err } gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String()) @@ -294,8 +291,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU } // Create the template folder if it doesn't exists - err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm) - if err != nil { + if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm); err != nil { return nil, fmt.Errorf("failed to create template base folder: %s", err) } @@ -308,8 +304,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU r.printUpdateChangelog(results, version) } checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum") - err = writeTemplatesChecksum(checksumFile, results.checksums) - if err != nil { + if err := writeTemplatesChecksum(checksumFile, results.checksums); err != nil { return nil, errors.Wrap(err, "could not write checksum") } @@ -320,8 +315,8 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU buffer.WriteString(addition) buffer.WriteString("\n") } - err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm) - if err != nil { + + if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm); err != nil { return nil, errors.Wrap(err, "could not write new additions file") } return results, err @@ -362,8 +357,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults } results.totalCount++ templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath) - err := os.MkdirAll(templateDirectory, os.ModePerm) - if err != nil { + if err := os.MkdirAll(templateDirectory, os.ModePerm); err != nil { return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) } @@ -387,8 +381,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults hasher := md5.New() // Save file and also read into hasher for md5 - _, err = io.Copy(f, io.TeeReader(reader, hasher)) - if err != nil { + if _, err := io.Copy(f, io.TeeReader(reader, hasher)); err != nil { f.Close() return nil, fmt.Errorf("could not write template file: %s", err) } @@ -545,8 +538,7 @@ func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { } var tags []githubTagData - err = json.Unmarshal(body, &tags) - if err != nil { + if err := json.Unmarshal(body, &tags); err != nil { return "", err } if len(tags) == 0 { diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index e1d7924dc..0d18cac57 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -55,8 +55,7 @@ func ReadConfiguration() (*Config, error) { defer file.Close() config := &Config{} - err = jsoniter.NewDecoder(file).Decode(config) - if err != nil { + if err := jsoniter.NewDecoder(file).Decode(config); err != nil { return nil, err } return config, nil diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index b33d6b733..f59fa9f30 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -113,18 +113,14 @@ var functions = map[string]govaluate.ExpressionFunction{ }, "sha256": func(args ...interface{}) (interface{}, error) { h := sha256.New() - _, err := h.Write([]byte(types.ToString(args[0]))) - - if err != nil { + if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { return nil, err } return hex.EncodeToString(h.Sum(nil)), nil }, "sha1": func(args ...interface{}) (interface{}, error) { h := sha1.New() - _, err := h.Write([]byte(types.ToString(args[0]))) - - if err != nil { + if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { return nil, err } return hex.EncodeToString(h.Sum(nil)), nil diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 913fd8284..72440c6ec 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -84,9 +84,7 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { var jsonObj interface{} - err := json.Unmarshal([]byte(corpus), &jsonObj) - - if err != nil { + if err := json.Unmarshal([]byte(corpus), &jsonObj); err != nil { return results } diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go index a72e600ee..94765ae23 100644 --- a/v2/pkg/output/file_output_writer.go +++ b/v2/pkg/output/file_output_writer.go @@ -20,11 +20,10 @@ func newFileOutputWriter(file string) (*fileWriter, error) { // WriteString writes an output to the underlying file func (w *fileWriter) Write(data []byte) error { - _, err := w.file.Write(data) - if err != nil { + if _, err := w.file.Write(data); err != nil { return err } - _, err = w.file.Write([]byte("\n")) + _, err := w.file.Write([]byte("\n")) return err } diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 6db6b97fe..4fe823492 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -107,8 +107,7 @@ func ParseTemplate(templatePath string) (*templates.Template, error) { } template := &templates.Template{} - err = yaml.Unmarshal(data, template) - if err != nil { + if err := yaml.Unmarshal(data, template); err != nil { return nil, err } parsedTemplatesCache.Store(templatePath, template, nil) diff --git a/v2/pkg/projectfile/httputil.go b/v2/pkg/projectfile/httputil.go index d077a5a5b..6479fd20b 100644 --- a/v2/pkg/projectfile/httputil.go +++ b/v2/pkg/projectfile/httputil.go @@ -18,8 +18,7 @@ func hash(v interface{}) (string, error) { sh := sha256.New() - _, err = io.WriteString(sh, string(data)) - if err != nil { + if _, err = io.WriteString(sh, string(data)); err != nil { return "", err } return hex.EncodeToString(sh.Sum(nil)), nil @@ -28,8 +27,7 @@ func hash(v interface{}) (string, error) { func marshal(data interface{}) ([]byte, error) { var b bytes.Buffer enc := gob.NewEncoder(&b) - err := enc.Encode(data) - if err != nil { + if err := enc.Encode(data); err != nil { return nil, err } @@ -38,8 +36,7 @@ func marshal(data interface{}) ([]byte, error) { func unmarshal(data []byte, obj interface{}) error { dec := gob.NewDecoder(bytes.NewBuffer(data)) - err := dec.Decode(obj) - if err != nil { + if err := dec.Decode(obj); err != nil { return err } diff --git a/v2/pkg/projectfile/project.go b/v2/pkg/projectfile/project.go index cc3d51de3..71ad9e365 100644 --- a/v2/pkg/projectfile/project.go +++ b/v2/pkg/projectfile/project.go @@ -44,8 +44,7 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) { var httprecord HTTPRecord httprecord.Response = newInternalResponse() - err = unmarshal(data, &httprecord) - if err != nil { + if err := unmarshal(data, &httprecord); err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 65ea5eb39..d9a285948 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -24,8 +24,7 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption // Compile compiles the execution generators preparing any requests possible. func (e *Executer) Compile() error { for _, request := range e.requests { - err := request.Compile(e.options) - if err != nil { + if err := request.Compile(e.options); err != nil { return err } } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 0a9f84d6f..ae3041f33 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -71,8 +71,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { - _, err := url.ParseRequestURI(toTest) - if err != nil { + if _, err := url.ParseRequestURI(toTest); err != nil { return false } u, err := url.Parse(toTest) diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index 78d9d1fe1..ec1cc2585 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -18,8 +18,7 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { // Template input includes a wildcard if strings.Contains(target, "*") && !r.NoRecursive { - err := r.findGlobPathMatches(target, processed, callback) - if err != nil { + if err := r.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil @@ -38,8 +37,7 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { } // Recursively walk down the Templates directory and run all // the template file checks - err = r.findDirectoryMatches(target, processed, callback) - if err != nil { + if err := r.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 16eae572f..0ae5f0ac7 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -37,18 +37,18 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio } createdPage.router = router - err = page.SetViewport(&proto.EmulationSetDeviceMetricsOverride{Viewport: &proto.PageViewport{ + if err := page.SetViewport(&proto.EmulationSetDeviceMetricsOverride{Viewport: &proto.PageViewport{ Scale: 1, Width: float64(1920), Height: float64(1080), - }}) - if err != nil { + }}); err != nil { return nil, nil, err } - _, err = page.SetExtraHeaders([]string{"Accept-Language", "en, en-GB, en-us;"}) - if err != nil { + + if _, err := page.SetExtraHeaders([]string{"Accept-Language", "en, en-GB, en-us;"}); err != nil { return nil, nil, err } + go router.Run() data, err := createdPage.ExecuteActions(baseURL, actions) if err != nil { diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index f5b4b933e..b7a21d88c 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -176,8 +176,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.UR values["BaseURL"] = parsedString final := fasttemplate.ExecuteStringStd(URL, "{{", "}}", values) - err := p.page.Navigate(final) - if err != nil { + if err := p.page.Navigate(final); err != nil { return errors.Wrap(err, "could not navigate") } return nil diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 4a884b917..dc90cba74 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -418,8 +418,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // if nuclei-project is enabled store the response if not previously done if r.options.ProjectFile != nil && !fromcache { - err := r.options.ProjectFile.Set(dumpedRequest, resp, data) - if err != nil { + if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { return errors.Wrap(err, "could not store in project file") } } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 8c9362d44..9d03a7cfb 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -38,8 +38,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused actualAddress = net.JoinHostPort(actualAddress, kv.port) } - err = r.executeAddress(actualAddress, address, input, kv.tls, previous, callback) - if err != nil { + if err := r.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -138,8 +137,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } reqBuilder.Write(finalData) - _, err = conn.Write(finalData) - if err != nil { + if _, err := conn.Write(finalData); err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") diff --git a/v2/pkg/protocols/offlinehttp/find.go b/v2/pkg/protocols/offlinehttp/find.go index 2b724847b..ee918ad57 100644 --- a/v2/pkg/protocols/offlinehttp/find.go +++ b/v2/pkg/protocols/offlinehttp/find.go @@ -17,8 +17,7 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { // Template input includes a wildcard if strings.Contains(target, "*") { - err := r.findGlobPathMatches(target, processed, callback) - if err != nil { + if err := r.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil @@ -35,8 +34,7 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { // Recursively walk down the Templates directory and run all // the template file checks - err = r.findDirectoryMatches(target, processed, callback) - if err != nil { + if err := r.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index 603515644..6ce86345d 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -55,8 +55,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { dataBuilder.WriteString(description) data := dataBuilder.Bytes() - err := ioutil.WriteFile(filepath.Join(i.directory, finalFilename), data, 0644) - return err + return ioutil.WriteFile(filepath.Join(i.directory, finalFilename), data, 0644) } // Close closes the exporter after operation diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 6532cc06a..1622b1f0d 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -56,8 +56,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute data = preprocessor.Process(data) } - err = yaml.UnmarshalStrict(data, template) - if err != nil { + if err := yaml.UnmarshalStrict(data, template); err != nil { if !fieldErrorRegexp.MatchString(err.Error()) { return nil, err } @@ -141,8 +140,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.Executer = executer.NewExecuter(requests, &options) } if template.Executer != nil { - err := template.Executer.Compile() - if err != nil { + if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") } template.TotalRequests += template.Executer.Requests() diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index f4b294895..c0710c6ad 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -15,8 +15,7 @@ func (w *Workflow) RunWorkflow(input string) bool { for _, template := range w.Workflows { swg.Add() func(template *WorkflowTemplate) { - err := w.runWorkflowStep(template, input, results, &swg) - if err != nil { + if err := w.runWorkflowStep(template, input, results, &swg); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() @@ -116,8 +115,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res swg.Add() go func(template *WorkflowTemplate) { - err := w.runWorkflowStep(template, input, results, swg) - if err != nil { + if err := w.runWorkflowStep(template, input, results, swg); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() From f36ed8df64467a154030abe143296c4ea5451cc0 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 31 Aug 2021 13:21:15 +0300 Subject: [PATCH 3/7] Removed unused parameter --- v2/pkg/catalog/loader/loader.go | 4 ++-- v2/pkg/operators/common/dsl/dsl.go | 2 +- v2/pkg/parsers/parser.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index e6f13ce7f..8d54f4af5 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -108,7 +108,7 @@ func (store *Store) ValidateTemplates(templatesList, workflowsList []string) err func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool { return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { - return parsers.LoadWorkflow(templatePath, store.tagFilter) + return parsers.LoadWorkflow(templatePath) }) } @@ -178,7 +178,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap)) for workflowPath := range workflowPathMap { - loaded, err := parsers.LoadWorkflow(workflowPath, store.tagFilter) + loaded, err := parsers.LoadWorkflow(workflowPath) if err != nil { gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err) } diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index f59fa9f30..f54b18758 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -248,7 +248,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { return functions } -// AddHelperFunction allows creation of additiona helper functions to be supported with templates +// AddHelperFunction allows creation of additional helper functions to be supported with templates func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error { if _, ok := functions[key]; !ok { functions[key] = value diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 4fe823492..90033fdbe 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -36,7 +36,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. -func LoadWorkflow(templatePath string, tagFilter *filter.TagFilter) (bool, error) { +func LoadWorkflow(templatePath string) (bool, error) { template, templateParseError := ParseTemplate(templatePath) if templateParseError != nil { return false, templateParseError From f9eb8ba8acb24dbc24734ea99b0b1764ea80dd5d Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:34:51 +0300 Subject: [PATCH 4/7] Changed/removed some documentation/comments --- v2/internal/runner/options.go | 5 +--- v2/internal/runner/processor.go | 2 +- v2/internal/runner/runner.go | 6 ++-- v2/internal/runner/templates.go | 2 +- v2/internal/runner/update.go | 32 ++++++++++------------ v2/internal/severity/misc.go | 2 +- v2/pkg/catalog/loader/filter/tag_filter.go | 2 +- 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 0cd005463..e9339cb3b 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -77,12 +77,10 @@ func hasStdin() bool { // validateOptions validates the configuration options passed func validateOptions(options *types.Options) error { - // Both verbose and silent flags were used if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - // Validate proxy options if provided if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { return err } @@ -111,9 +109,8 @@ func isValidURL(urlString string) bool { return err == nil } -// configureOutput configures the output on the screen +// configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { - // If the user desires verbose output, show verbose output if options.Verbose { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 6006823ec..f24055ccf 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -7,7 +7,7 @@ import ( "go.uber.org/atomic" ) -// processTemplateWithList process a template on the URL list +// processTemplateWithList execute a template against the list of user provided targets func (r *Runner) processTemplateWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index bdd21faf4..7f286da45 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -229,7 +229,7 @@ func New(options *types.Options) (*Runner, error) { return nil, progressErr } - // create project file if requested or load existing one + // create project file if requested or load the existing one if options.Project { var projectFileErr error runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: utils.IsBlank(options.ProjectPath)}) @@ -284,7 +284,7 @@ func (r *Runner) Close() { func (r *Runner) RunEnumeration() error { defer r.Close() - // If user asked for new templates to be executed, collect the list from template directory. + // If user asked for new templates to be executed, collect the list from the templates' directory. if r.options.NewTemplates { templatesLoaded, err := r.readNewTemplatesFile() if err != nil { @@ -536,7 +536,7 @@ func (r *Runner) readNewTemplatesFile() ([]string, error) { return templatesList, nil } -// readNewTemplatesFile reads newly added templates from directory if it exists +// countNewTemplates returns the number of newly added templates func (r *Runner) countNewTemplates() int { if r.templatesConfig == nil { return 0 diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index f703adc28..a63694c76 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -57,7 +57,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) { } } -// ListAvailableTemplates prints available templates to stdout +// listAvailableTemplates prints available templates to stdout func (r *Runner) listAvailableTemplates() { if r.templatesConfig == nil { return diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 2243398f9..270776713 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -44,11 +44,11 @@ const ( var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) // updateTemplates checks if the default list of nuclei-templates -// exist in the users home directory, if not the latest revision -// is downloaded from github. +// exist in the user's home directory, if not the latest revision +// is downloaded from GitHub. // -// If the path exists but is not latest, the new version is downloaded -// from github and replaced with the templates directory. +// If the path exists but does not contain the latest version of public templates, +// the new version is downloaded from GitHub to the templates' directory, overwriting the old content. func (r *Runner) updateTemplates() error { home, err := os.UserHomeDir() if err != nil { @@ -61,7 +61,7 @@ func (r *Runner) updateTemplates() error { return errors.Wrap(err, "could not read configuration file") } - // If the config doesn't exist, write it now. + // If the config doesn't exist, create it now. if r.templatesConfig == nil { currentConfig := &config.Config{ TemplatesDirectory: filepath.Join(home, "nuclei-templates"), @@ -78,11 +78,8 @@ func (r *Runner) updateTemplates() error { return nil } - // Check if last checked for nuclei-ignore is more than 1 hours. - // and if true, run the check. - // - // Also at the same time fetch latest version from github to do outdated nuclei - // and templates check. + // Tests if last checked time for nuclei-ignore file was more than 1 hour ago, if yes, updates the local content. + // Retrieves the latest version number of nuclei and nuclei-templates from GitHub, to check if the current build is using outdated versions or not. checkedIgnore := false if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour { checkedIgnore = r.checkNucleiIgnoreFileUpdates(configDir) @@ -92,7 +89,7 @@ func (r *Runner) updateTemplates() error { if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) { gologger.Info().Msgf("nuclei-templates are not installed, installing...\n") - // Use custom location if user has given a template directory + // Use the custom location if the user has given a template directory r.templatesConfig = &config.Config{ TemplatesDirectory: filepath.Join(home, "nuclei-templates"), } @@ -107,7 +104,7 @@ func (r *Runner) updateTemplates() error { } gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - r.fetchLatestVersionsFromGithub() // also fetch latest versions + r.fetchLatestVersionsFromGithub() // also fetch the latest versions if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { return err } @@ -120,13 +117,14 @@ func (r *Runner) updateTemplates() error { return nil } - // Check if last checked is more than 24 hours and we don't have updateTemplates flag. - // If not, return since we don't want to do anything now. + // If the template update was not requested explicitly by the user, + // and the last version check was less than 24 hours ago, + // then no further action is required. if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates { return nil } - // Get the configuration currently on disk. + // Get the current configuration from disk. verText := r.templatesConfig.CurrentVersion indices := reVersion.FindStringIndex(verText) if indices == nil { @@ -190,7 +188,7 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error { return nil } -// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from github +// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from GitHub func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { ignoreURL := defaultIgnoreURL if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" { @@ -290,7 +288,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU return nil, fmt.Errorf("failed to uncompress zip file: %s", err) } - // Create the template folder if it doesn't exists + // Create the template folder if it doesn't exist if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm); err != nil { return nil, fmt.Errorf("failed to create template base folder: %s", err) } diff --git a/v2/internal/severity/misc.go b/v2/internal/severity/misc.go index 8601fb368..ff9eeb06b 100644 --- a/v2/internal/severity/misc.go +++ b/v2/internal/severity/misc.go @@ -29,7 +29,7 @@ func setSeverity(severities *Severities, value string) error { return fmt.Errorf("'%s' is not a valid severity", value) } - // TODO change the Severities type to map[Severity]interface{}, where the values are struct{}{}, to "simulates" a "set" data structure + // TODO change the Severities type to map[Severity]interface{}, where the values are struct{}{}, to "simulate" a "set" data structure *severities = append(*severities, computedSeverity) return nil } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index 90f9a8de2..7e0b8f9da 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -20,7 +20,7 @@ type TagFilter struct { var ErrExcluded = errors.New("the template was excluded") // Match filters templates based on user provided tags, authors, extraTags and severity. -// If the template contains tags specified in the deny list, it will not be matched +// If the template contains tags specified in the deny-list, it will not be matched // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. From 8b8c16dd8ea11ebafef63b891bde0d4d29ac9d72 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:36:07 +0300 Subject: [PATCH 5/7] Improve readability of some methods --- v2/internal/runner/templates.go | 20 +++--- v2/internal/runner/templates_test.go | 25 ++++++++ v2/internal/runner/update.go | 93 ++++++++++++++-------------- 3 files changed, 78 insertions(+), 60 deletions(-) create mode 100644 v2/internal/runner/templates_test.go diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index a63694c76..d467c7b33 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -13,30 +13,24 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -func (r *Runner) templateLogMsg(id, name, author string, templateSeverity severity.Severity) string { +func (r *Runner) templateLogMsg(id, name string, authors []string, templateSeverity severity.Severity) string { // Display the message for the template return fmt.Sprintf("[%s] %s (%s) [%s]", r.colorizer.BrightBlue(id).String(), r.colorizer.Bold(name).String(), - r.colorizer.BrightYellow(appendAtSignToAuthors(author)).String(), + r.colorizer.BrightYellow(appendAtSignToAuthors(authors)).String(), r.addColor(templateSeverity)) } -// appendAtSignToAuthors appends @ before each author and returns final string -func appendAtSignToAuthors(author string) string { - authors := strings.Split(author, ",") +// appendAtSignToAuthors appends @ before each author and returns the final string +func appendAtSignToAuthors(authors []string) string { if len(authors) == 0 { return "@none" } - if len(authors) == 1 { - if !strings.HasPrefix(authors[0], "@") { - return fmt.Sprintf("@%s", authors[0]) - } - return authors[0] - } + values := make([]string, 0, len(authors)) for _, k := range authors { - if !strings.HasPrefix(authors[0], "@") { + if !strings.HasPrefix(k, "@") { values = append(values, fmt.Sprintf("@%s", k)) } else { values = append(values, k) @@ -52,7 +46,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) { } else { gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info.Name), - types.ToString(t.Info.Authors), + t.Info.Authors.ToSlice(), t.Info.SeverityHolder.Severity)) } } diff --git a/v2/internal/runner/templates_test.go b/v2/internal/runner/templates_test.go new file mode 100644 index 000000000..5eba0b561 --- /dev/null +++ b/v2/internal/runner/templates_test.go @@ -0,0 +1,25 @@ +package runner + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_appendAtSignToAuthors(t *testing.T) { + result := appendAtSignToAuthors([]string{"user1", "user2", "user3"}) + assert.Equal(t, result, "@user1,@user2,@user3") +} + +func Test_appendAtSignToMissingAuthors(t *testing.T) { + result := appendAtSignToAuthors([]string{}) + assert.Equal(t, result, "@none") + + result = appendAtSignToAuthors(nil) + assert.Equal(t, result, "@none") +} + +func Test_appendAtSignToOneAuthor(t *testing.T) { + result := appendAtSignToAuthors([]string{"user1"}) + assert.Equal(t, result, "@user1") +} diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 270776713..76d1efa70 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -97,8 +97,8 @@ func (r *Runner) updateTemplates() error { r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory) } - // Download the repository and also write the revision to a HEAD file. - version, asset, getErr := r.getLatestReleaseFromGithub() + // Download the repository and write the revision to a HEAD file. + version, asset, getErr := r.getLatestTemplateReleaseFromGithub() if getErr != nil { return getErr } @@ -139,7 +139,7 @@ func (r *Runner) updateTemplates() error { return err } - version, asset, err := r.getLatestReleaseFromGithub() + version, asset, err := r.getLatestTemplateReleaseFromGithub() if err != nil { return err } @@ -223,8 +223,7 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { return true } -// getLatestReleaseFromGithub returns the latest release from github -func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) { +func (r *Runner) getLatestTemplateReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) { client := github.NewClient(nil) rels, _, err := client.Repositories.ListReleases(context.Background(), userName, repoName, nil) @@ -283,7 +282,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU } reader := bytes.NewReader(buf) - z, err := zip.NewReader(reader, reader.Size()) + zipReader, err := zip.NewReader(reader, reader.Size()) if err != nil { return nil, fmt.Errorf("failed to uncompress zip file: %s", err) } @@ -293,7 +292,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU return nil, fmt.Errorf("failed to create template base folder: %s", err) } - results, err := r.compareAndWriteTemplates(z) + results, err := r.compareAndWriteTemplates(zipReader) if err != nil { return nil, fmt.Errorf("failed to write templates: %s", err) } @@ -328,9 +327,8 @@ type templateUpdateResults struct { checksums map[string]string } -// compareAndWriteTemplates compares and returns the stats of a template -// update operations. -func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) { +// compareAndWriteTemplates compares and returns the stats of a template update operations. +func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdateResults, error) { results := &templateUpdateResults{ checksums: make(map[string]string), } @@ -341,9 +339,9 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults // it is removed. This allows us fine-grained control over the download process // as well as solves a long problem with nuclei-template updates. checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum") - previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile) - for _, file := range z.File { - directory, name := filepath.Split(file.Name) + templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile) + for _, zipTemplateFile := range zipReader.File { + directory, name := filepath.Split(zipTemplateFile.Name) if name == "" { continue } @@ -365,27 +363,27 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) { isAddition = true } - f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) + templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) if err != nil { - f.Close() + templateFile.Close() return nil, fmt.Errorf("could not create uncompressed file: %s", err) } - reader, err := file.Open() + zipTemplateFileReader, err := zipTemplateFile.Open() if err != nil { - f.Close() + templateFile.Close() return nil, fmt.Errorf("could not open archive to extract file: %s", err) } hasher := md5.New() // Save file and also read into hasher for md5 - if _, err := io.Copy(f, io.TeeReader(reader, hasher)); err != nil { - f.Close() + if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, hasher)); err != nil { + templateFile.Close() return nil, fmt.Errorf("could not write template file: %s", err) } - f.Close() + templateFile.Close() - oldChecksum, checksumOK := previousChecksum[templatePath] + oldChecksum, checksumOK := templateChecksumsMap[templatePath] checksum := hex.EncodeToString(hasher.Sum(nil)) if isAddition { @@ -396,57 +394,58 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults results.checksums[templatePath] = checksum } - // If we don't find a previous file in new download and it hasn't been - // changed on the disk, delete it. - for k, v := range previousChecksum { - _, ok := results.checksums[k] - if !ok && v[0] == v[1] { - os.Remove(k) - results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) + // If we don't find the previous file in the newly downloaded list, + // and it hasn't been changed on the disk, delete it. + for templatePath, templateChecksums := range templateChecksumsMap { + _, ok := results.checksums[templatePath] + if !ok && templateChecksums[0] == templateChecksums[1] { + os.Remove(templatePath) + results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) } } return results, nil } -// readPreviousTemplatesChecksum reads the previous checksum file from the disk. -// -// It reads two checksums, the first checksum is what we expect and the second is -// the actual checksum of the file on disk currently. -func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) { - f, err := os.Open(file) +// createTemplateChecksumsMap reads the previous checksum file from the disk. +// Creates a map of template paths and their previous and currently calculated checksums as values. +func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) { + checksumFile, err := os.Open(checksumsFilePath) if err != nil { return nil, err } - defer f.Close() - scanner := bufio.NewScanner(f) + defer checksumFile.Close() + scanner := bufio.NewScanner(checksumFile) - checksum := make(map[string][2]string) + templatePathChecksumsMap := make(map[string][2]string) for scanner.Scan() { text := scanner.Text() if text == "" { continue } + parts := strings.Split(text, ",") if len(parts) < 2 { continue } - values := [2]string{parts[1]} + templatePath := parts[0] + expectedTemplateChecksum := parts[1] - f, err := os.Open(parts[0]) + templateFile, err := os.Open(templatePath) if err != nil { return nil, err } hasher := md5.New() - if _, err := io.Copy(hasher, f); err != nil { + if _, err := io.Copy(hasher, templateFile); err != nil { return nil, err } - f.Close() + templateFile.Close() + values := [2]string{expectedTemplateChecksum} values[1] = hex.EncodeToString(hasher.Sum(nil)) - checksum[parts[0]] = values + templatePathChecksumsMap[templatePath] = values } - return checksum, nil + return templatePathChecksumsMap, nil } // writeTemplatesChecksum writes the nuclei-templates checksum data to disk. @@ -493,7 +492,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st table.Render() } -// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github +// fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub func (r *Runner) fetchLatestVersionsFromGithub() { nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei") if err != nil { @@ -513,8 +512,8 @@ type githubTagData struct { Name string } -// githubFetchLatestTagRepo fetches latest tag from github -// This function was half written by github copilot AI :D. +// githubFetchLatestTagRepo fetches the latest tag of the given repository from GitHub +// This function was half written by the GitHub Copilot AI :D. func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -545,7 +544,7 @@ func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { return strings.TrimPrefix(tags[0].Name, "v"), nil } -// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases. +// updateNucleiVersionToLatest implements nuclei auto-update using GitHub Releases. func updateNucleiVersionToLatest(verbose bool) error { if verbose { log.SetLevel(log.DebugLevel) From 0acec5e039d908bb84b9166a0eaad64e3bd0f965 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 3 Sep 2021 17:25:50 +0300 Subject: [PATCH 6/7] Adding correct godoc prefixes --- v2/cmd/integration-test/dns.go | 2 +- v2/cmd/integration-test/http.go | 31 ++++++++++--------- v2/cmd/integration-test/network.go | 4 +-- v2/cmd/integration-test/workflow.go | 7 +++-- v2/internal/testutils/integration.go | 2 +- v2/pkg/protocols/headless/engine/action.go | 2 +- .../http/httpclientpool/clientpool.go | 9 +++--- 7 files changed, 30 insertions(+), 27 deletions(-) diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index 8ea5e1154..7661f48f9 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -10,7 +10,7 @@ var dnsTestCases = map[string]testutils.TestCase{ type dnsBasic struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *dnsBasic) Execute(filePath string) error { var routerErr error diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index e23d740b2..8188c27af 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" ) @@ -43,7 +44,7 @@ func httpDebugRequestDump(r *http.Request) { type httpGetHeaders struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpGetHeaders) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -67,7 +68,7 @@ func (h *httpGetHeaders) Execute(filePath string) error { type httpGetQueryString struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpGetQueryString) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -91,7 +92,7 @@ func (h *httpGetQueryString) Execute(filePath string) error { type httpGetRedirects struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpGetRedirects) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -117,7 +118,7 @@ func (h *httpGetRedirects) Execute(filePath string) error { type httpGet struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpGet) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -139,7 +140,7 @@ func (h *httpGet) Execute(filePath string) error { type httpPostBody struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpPostBody) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -172,7 +173,7 @@ func (h *httpPostBody) Execute(filePath string) error { type httpPostJSONBody struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpPostJSONBody) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -211,7 +212,7 @@ func (h *httpPostJSONBody) Execute(filePath string) error { type httpPostMultipartBody struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpPostMultipartBody) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -254,7 +255,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { type httpRawDynamicExtractor struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawDynamicExtractor) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -293,7 +294,7 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { type httpRawGetQuery struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawGetQuery) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -322,7 +323,7 @@ func (h *httpRawGetQuery) Execute(filePath string) error { type httpRawGet struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawGet) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -350,7 +351,7 @@ func (h *httpRawGet) Execute(filePath string) error { type httpRawPayload struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawPayload) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -386,7 +387,7 @@ func (h *httpRawPayload) Execute(filePath string) error { type httpRawPostBody struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawPostBody) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -419,7 +420,7 @@ func (h *httpRawPostBody) Execute(filePath string) error { type httpRawCookieReuse struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawCookieReuse) Execute(filePath string) error { router := httprouter.New() var routerErr error @@ -468,7 +469,7 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { type httpRawUnsafeRequest struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRawUnsafeRequest) Execute(filePath string) error { var routerErr error @@ -493,7 +494,7 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error { type httpRequestCondition struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpRequestCondition) Execute(filePath string) error { router := httprouter.New() var routerErr error diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index de9e86249..5bb0fe519 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -14,7 +14,7 @@ var networkTestcases = map[string]testutils.TestCase{ type networkBasic struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *networkBasic) Execute(filePath string) error { var routerErr error @@ -47,7 +47,7 @@ func (h *networkBasic) Execute(filePath string) error { type networkMultiStep struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *networkMultiStep) Execute(filePath string) error { var routerErr error diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index a8548e9b6..08d6e1385 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" ) @@ -18,7 +19,7 @@ var workflowTestcases = map[string]testutils.TestCase{ type workflowBasic struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *workflowBasic) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -40,7 +41,7 @@ func (h *workflowBasic) Execute(filePath string) error { type workflowConditionMatched struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *workflowConditionMatched) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -62,7 +63,7 @@ func (h *workflowConditionMatched) Execute(filePath string) error { type workflowConditionUnmatch struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *workflowConditionUnmatch) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 1f59c3011..03e87685a 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -34,7 +34,7 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: (\d+)`) -// RunNucleiAndGetResults returns a list of results for a template +// RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) { cmd := exec.Command(nucleiBinary, args...) diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index 1da881407..0be71fff8 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -46,7 +46,7 @@ const ( ActionWaitEvent // ActionKeyboard performs a keyboard action event on a page. ActionKeyboard - // Action debug slows down headless and adds a sleep to each page. + // ActionDebug debug slows down headless and adds a sleep to each page. ActionDebug // ActionSleep executes a sleep for a specified duration ActionSleep diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 63b1479bd..03e6e857b 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -14,13 +14,14 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/net/proxy" + "golang.org/x/net/publicsuffix" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" - "golang.org/x/net/proxy" - "golang.org/x/net/publicsuffix" ) var ( @@ -50,7 +51,7 @@ func Init(options *types.Options) error { return nil } -// // Configuration contains the custom configuration options for a connection +// ConnectionConfiguration contains the custom configuration options for a connection type ConnectionConfiguration struct { // DisableKeepAlive of the connection DisableKeepAlive bool @@ -88,7 +89,7 @@ func (c *Configuration) Hash() string { return hash } -// HasCustomOptions checks whether the configuration requires custom settings +// HasStandardOptions checks whether the configuration requires custom settings func (c *Configuration) HasStandardOptions() bool { return c.Threads == 0 && c.MaxRedirects == 0 && !c.FollowRedirects && !c.CookieReuse && c.Connection == nil } From 0ce33927c79c8871f4c45d1a934aecab1272fe02 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 7 Sep 2021 17:31:46 +0300 Subject: [PATCH 7/7] Typo fixes. --- README.md | 8 ++++---- SECURITY.md | 2 +- v2/cmd/functional-test/main.go | 3 ++- v2/internal/runner/runner.go | 2 +- v2/internal/runner/update.go | 1 + v2/pkg/model/model.go | 3 ++- v2/pkg/operators/extractors/compile.go | 6 +++--- v2/pkg/operators/extractors/extractors.go | 4 ++-- v2/pkg/operators/matchers/compile.go | 5 +++-- v2/pkg/operators/matchers/match.go | 8 ++++---- v2/pkg/operators/matchers/matchers.go | 4 ++-- v2/pkg/operators/operators.go | 8 ++++---- v2/pkg/output/output.go | 4 ++-- v2/pkg/protocols/common/generators/generators.go | 4 ++-- v2/pkg/protocols/common/generators/maps.go | 2 +- v2/pkg/protocols/common/interactsh/interactsh.go | 3 ++- v2/pkg/protocols/dns/dns.go | 5 +++-- v2/pkg/protocols/dns/dnsclientpool/clientpool.go | 2 +- v2/pkg/protocols/dns/operators.go | 2 +- v2/pkg/protocols/file/file.go | 7 ++++--- v2/pkg/protocols/file/operators.go | 2 +- v2/pkg/protocols/headless/engine/engine.go | 5 +++-- v2/pkg/protocols/headless/headless.go | 3 ++- v2/pkg/protocols/headless/operators.go | 2 +- v2/pkg/protocols/http/build_request.go | 5 +++-- v2/pkg/protocols/http/http.go | 3 ++- v2/pkg/protocols/http/httpclientpool/clientpool.go | 2 +- v2/pkg/protocols/http/operators.go | 4 ++-- v2/pkg/protocols/http/request.go | 9 +++++---- v2/pkg/protocols/network/network.go | 3 ++- v2/pkg/protocols/network/networkclientpool/clientpool.go | 2 +- v2/pkg/protocols/network/operators.go | 2 +- v2/pkg/protocols/offlinehttp/operators.go | 4 ++-- v2/pkg/protocols/protocols.go | 5 +++-- v2/pkg/reporting/dedupe/dedupe.go | 7 ++++--- v2/pkg/reporting/exporters/disk/disk.go | 2 +- v2/pkg/reporting/trackers/github/github.go | 3 ++- v2/pkg/reporting/trackers/gitlab/gitlab.go | 5 +++-- v2/pkg/workflows/workflows.go | 6 +++--- 39 files changed, 87 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 637088d6e..c60eab732 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Nuclei offers great number of features that are helpful for security engineers t -**For bugbounty hunters:** +**For Bug Bounty hunters:** Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow. @@ -237,9 +237,9 @@ Please check our other open-source projects that might fit into your bug bounty -**For pentesters:** +**For Penetration Testers:** -Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. +Nuclei immensely improve how you approach security assessment by augmenting the manual, repetitive processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix. @@ -254,7 +254,7 @@ Pen-testers get the full power of our public templates and customization capabil # For Developers and Organisations -Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and eliminate vulnerabilities from occuring in future. +Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with the latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and eliminate vulnerabilities from occurring in the future. - **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates. - **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle. diff --git a/SECURITY.md b/SECURITY.md index fc8033f4f..699a1f3e9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,4 +2,4 @@ ## Reporting a Vulnerability -DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days. +DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io, and we will acknowledge it within 3 working days. diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go index 1e54cda7e..bc95f20b5 100644 --- a/v2/cmd/functional-test/main.go +++ b/v2/cmd/functional-test/main.go @@ -10,6 +10,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" ) @@ -75,5 +76,5 @@ func runIndividualTestCase(testcase string) error { if mainOutput == devOutput { return nil } - return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) + return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 136bb91df..f8dc656e6 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -357,7 +357,7 @@ func (r *Runner) RunEnumeration() error { return nil // exit } - // Display stats for any loaded templates syntax warnings or errors + // Display stats for any loaded templates' syntax warnings or errors stats.Display(parsers.SyntaxWarningStats) stats.Display(parsers.SyntaxErrorStats) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 76d1efa70..9a9b3523c 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -25,6 +25,7 @@ import ( "github.com/google/go-github/github" "github.com/olekukonko/tablewriter" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index 46f426646..3b1efb2cc 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/alecthomas/jsonschema" + "github.com/projectdiscovery/nuclei/v2/internal/severity" "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) @@ -41,7 +42,7 @@ type Info struct { // You can go in-depth here on what the template actually does. // // examples: - // - value: "\"Bower is a package manager which stores packages informations in bower.json file\"" + // - value: "\"Bower is a package manager which stores packages information in bower.json file\"" // - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\"" Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,example=Bower is a package manager which stores packages informations in bower.json file"` // description: | diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 341340a38..e688c9520 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -8,10 +8,10 @@ import ( "github.com/itchyny/gojq" ) -// CompileExtractors performs the initial setup operation on a extractor +// CompileExtractors performs the initial setup operation on an extractor func (e *Extractor) CompileExtractors() error { var ok bool - // Setup the extractor type + // Set up the extractor type e.extractorType, ok = ExtractorTypes[e.Type] if !ok { return fmt.Errorf("unknown extractor type specified: %s", e.Type) @@ -42,7 +42,7 @@ func (e *Extractor) CompileExtractors() error { e.jsonCompiled = append(e.jsonCompiled, compiled) } - // Setup the part of the request to match, if any. + // Set up the part of the request to match, if any. if e.Part == "" { e.Part = "body" } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 624c33006..5c126a271 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -50,7 +50,7 @@ type Extractor struct { // description: | // kval contains the key-value pairs present in the HTTP response header. // kval extractor can be used to extract HTTP response header and cookie key-value pairs. - // kval extractor inputs are case insensitive, and does not support dash (-) in input which can replaced with underscores (_) + // kval extractor inputs are case-insensitive, and does not support dash (-) in input which can replaced with underscores (_) // For example, Content-Type should be replaced with content_type // // A list of supported parts is available in docs for request types. @@ -121,7 +121,7 @@ const ( JSONExtractor ) -// ExtractorTypes is an table for conversion of extractor type from string. +// ExtractorTypes is a table for conversion of extractor type from string. var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index dede36652..09213e23a 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -6,6 +6,7 @@ import ( "regexp" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" ) @@ -22,7 +23,7 @@ func (m *Matcher) CompileMatchers() error { } } - // Setup the matcher type + // Set up the matcher type m.matcherType, ok = MatcherTypes[m.Type] if !ok { return fmt.Errorf("unknown matcher type specified: %s", m.Type) @@ -50,7 +51,7 @@ func (m *Matcher) CompileMatchers() error { m.dslCompiled = append(m.dslCompiled, compiled) } - // Setup the condition type, if any. + // Set up the condition type, if any. if m.Condition != "" { m.condition, ok = ConditionTypes[m.Condition] if !ok { diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 2b31386c4..9a1144438 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -48,7 +48,7 @@ func (m *Matcher) MatchWords(corpus string) bool { if m.condition == ANDCondition { return false } - // Continue with the flow since its an OR Condition. + // Continue with the flow since it's an OR Condition. continue } @@ -76,7 +76,7 @@ func (m *Matcher) MatchRegex(corpus string) bool { if m.condition == ANDCondition { return false } - // Continue with the flow since its an OR Condition. + // Continue with the flow since it's an OR Condition. continue } @@ -105,7 +105,7 @@ func (m *Matcher) MatchBinary(corpus string) bool { if m.condition == ANDCondition { return false } - // Continue with the flow since its an OR Condition. + // Continue with the flow since it's an OR Condition. continue } @@ -141,7 +141,7 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool { if m.condition == ANDCondition { return false } - // Continue with the flow since its an OR Condition. + // Continue with the flow since it's an OR Condition. continue } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 9d9f64393..da8053b68 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -131,7 +131,7 @@ const ( DSLMatcher ) -// MatcherTypes is an table for conversion of matcher type from string. +// MatcherTypes is a table for conversion of matcher type from string. var MatcherTypes = map[string]MatcherType{ "status": StatusMatcher, "size": SizeMatcher, @@ -151,7 +151,7 @@ const ( ORCondition ) -// ConditionTypes is an table for conversion of condition type from string. +// ConditionTypes is a table for conversion of condition type from string. var ConditionTypes = map[string]ConditionType{ "and": ANDCondition, "or": ORCondition, diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 3bf0f0efa..8a9de521e 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -14,7 +14,7 @@ type Operators struct { // whether the request was successful by doing pattern matching // on request/responses. // - // Multiple matchers can be combined together with `matcher-condition` flag + // Multiple matchers can be combined with `matcher-condition` flag // which accepts either `and` or `or` as argument. Matchers []*matchers.Matcher `yaml:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"` // description: | @@ -102,7 +102,7 @@ func (r *Result) Merge(result *Result) { // MatchFunc performs matching operation for a matcher on model and returns true or false. type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool -// ExtractFunc performs extracting operation for a extractor on model and returns true or false. +// ExtractFunc performs extracting operation for an extractor on model and returns true or false. type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure @@ -147,7 +147,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return nil, false } } else { - // If the matcher has matched, and its an OR + // If the matcher has matched, and it's an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && matcher.Name != "" { result.Matches[matcher.Name] = struct{}{} @@ -161,7 +161,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac if len(result.DynamicValues) > 0 { return result, true } - // Don't print if we have matchers and they have not matched, irregardless of extractor + // Don't print if we have matchers, and they have not matched, regardless of extractor if len(r.Matchers) > 0 && !matches { return nil, false } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index fbe3767ee..50aef3f21 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -77,9 +77,9 @@ type ResultEvent struct { Matched string `json:"matched,omitempty"` // ExtractedResults contains the extraction result from the inputs. ExtractedResults []string `json:"extracted_results,omitempty"` - // Request is the optional dumped request for the match. + // Request is the optional, dumped request for the match. Request string `json:"request,omitempty"` - // Response is the optional dumped response for the match. + // Response is the optional, dumped response for the match. Response string `json:"response,omitempty"` // Metadata contains any optional metadata for the event Metadata map[string]interface{} `json:"meta,omitempty"` diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index eb313ebb5..4a60a69e8 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -16,7 +16,7 @@ type Generator struct { type Type int const ( - // Sniper replaces each variables with values at a time. + // Sniper replaces each variable with values at a time. Sniper Type = iota + 1 // PitchFork replaces variables with positional value from multiple wordlists PitchFork @@ -24,7 +24,7 @@ const ( ClusterBomb ) -// StringToType is an table for conversion of attack type from string. +// StringToType is a table for conversion of attack type from string. var StringToType = map[string]Type{ "sniper": Sniper, "pitchfork": PitchFork, diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go index 9992b56ed..31df768bb 100644 --- a/v2/pkg/protocols/common/generators/maps.go +++ b/v2/pkg/protocols/common/generators/maps.go @@ -16,7 +16,7 @@ func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} { return m } -// ExpandMapValues converts values from flat string to strings slice +// ExpandMapValues converts values from flat string to string slice func ExpandMapValues(m map[string]string) map[string][]string { m1 := make(map[string][]string, len(m)) for k, v := range m { diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 773a88e0d..d54483a35 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -11,6 +11,7 @@ import ( "github.com/karlseguin/ccache" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/interactsh/pkg/client" "github.com/projectdiscovery/interactsh/pkg/server" @@ -111,7 +112,7 @@ func New(options *Options) (*Client, error) { item := interactClient.requests.Get(interaction.UniqueID) if item == nil { // If we don't have any request for this ID, add it to temporary - // lru cache so we can correlate when we get an add request. + // lru cache, so we can correlate when we get an add request. gotItem := interactClient.interactions.Get(interaction.UniqueID) if gotItem == nil { interactClient.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 858f36bbd..3881d1a5a 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -6,6 +6,7 @@ import ( "github.com/miekg/dns" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" @@ -18,7 +19,7 @@ type Request struct { // Operators for the current request go here. operators.Operators `yaml:",inline"` - // ID is the the optional id of the request + // ID is the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` // description: | @@ -158,7 +159,7 @@ func questionTypeToInt(questionType string) uint16 { return question } -// classToInt converts a dns class name to it's internal representation +// classToInt converts a dns class name to its internal representation func classToInt(class string) uint16 { class = strings.TrimSpace(strings.ToUpper(class)) result := dns.ClassINET diff --git a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go index d3fd53877..999f2a2c9 100644 --- a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -25,7 +25,7 @@ var defaultResolvers = []string{ // Init initializes the clientpool implementation func Init(options *types.Options) error { - // Don't create clients if already created in past. + // Don't create clients if already created in the past. if normalClient != nil { return nil } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 1120aff81..49fb3732a 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -43,7 +43,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) return false } -// Extract performs extracting operation for a extractor on model and returns true or false. +// Extract performs extracting operation for an extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { part := extractor.Part switch part { diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 3f0b3b949..2d1e64393 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -26,13 +27,13 @@ type Request struct { // - value: '[]string{".avi", ".mov", ".mp3"}' ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"` - // ID is the the optional id of the request + // ID is the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"` // description: | // MaxSize is the maximum size of the file to run request on. // - // By default, nuclei will process 5MB files and not go more than that. + // By default, nuclei will process 5 MB files and not go more than that. // It can be set to much lower or higher depending on use. // examples: // - value: 2048 @@ -68,7 +69,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } r.CompiledOperators = compiled } - // By default use 5mb as max size to read. + // By default, use 5 MB as max size to read. if r.MaxSize == 0 { r.MaxSize = 5 * 1024 * 1024 } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 4b8af5053..c56147763 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -41,7 +41,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) return false } -// Extract performs extracting operation for a extractor on model and returns true or false. +// Extract performs extracting operation for an extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index ba5b4cef3..073743840 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -11,9 +11,10 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" "github.com/pkg/errors" + ps "github.com/shirou/gopsutil/v3/process" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/stringsutil" - ps "github.com/shirou/gopsutil/v3/process" ) // Browser is a browser structure for nuclei headless module @@ -103,7 +104,7 @@ func (b *Browser) killChromeProcesses() { processes, _ := ps.Processes() for _, process := range processes { - // skip non chrome processes + // skip non-chrome processes if !isChromeProcess(process) { continue } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index b4bb98451..8af250b2b 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -2,6 +2,7 @@ package headless import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" @@ -9,7 +10,7 @@ import ( // Request contains a Headless protocol request to be made from a template type Request struct { - // ID is the the optional id of the request + // ID is the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=Optional ID of the headless request"` // description: | diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index d0a87aca6..e3c5dff41 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -39,7 +39,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) return false } -// Extract performs extracting operation for a extractor on model and returns true or false. +// Extract performs extracting operation for an extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 23797e887..f68ae8118 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -15,6 +15,7 @@ import ( "github.com/corpix/uarand" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" @@ -27,7 +28,7 @@ var ( urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) ) -// generatedRequest is a single wrapped generated request for a template request +// generatedRequest is a single generated request wrapped for a template request type generatedRequest struct { original *Request rawRequest *raw.Request @@ -235,7 +236,7 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte } setHeader(req, "User-Agent", uarand.GetRandom()) - // Only set these headers on non raw requests + // Only set these headers on non-raw requests if len(r.request.Raw) == 0 { setHeader(req, "Accept", "*/*") setHeader(req, "Accept-Language", "en") diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d3fa89dab..30f4be5f0 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" @@ -31,7 +32,7 @@ type Request struct { // value: | // []string{"GET /etc/passwd HTTP/1.1\nHost:\nContent-Length: 4", "POST /.%0d./.%0d./.%0d./.%0d./bin/sh HTTP/1.1\nHost: {{Hostname}}\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0\nContent-Length: 1\nConnection: close\n\necho\necho\ncat /etc/passwd 2>&1"} Raw []string `yaml:"raw,omitempty" jsonschema:"http requests in raw format,description=HTTP Requests in Raw Format"` - // ID is the the optional id of the request + // ID is the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id for the http request,description=ID for the HTTP Request"` // description: | // Name is the optional name of the request. diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 03e6e857b..a4faef07d 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -36,7 +36,7 @@ var ( // Init initializes the clientpool implementation func Init(options *types.Options) error { - // Don't create clients if already created in past. + // Don't create clients if already created in the past. if normalClient != nil { return nil } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 81320de8f..365f62d23 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -44,7 +44,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) return false } -// Extract performs extracting operation for a extractor on model and returns true or false. +// Extract performs extracting operation for an extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { @@ -85,7 +85,7 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { return itemStr, true } -// responseToDSLMap converts a HTTP response to a map for use in DSL matching +// responseToDSLMap converts an HTTP response to a map for use in DSL matching func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index dc90cba74..092a5524b 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -13,6 +13,9 @@ import ( "time" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/multierr" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -22,8 +25,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/multierr" ) const defaultMaxWorkers = 150 @@ -323,7 +324,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } if err != nil { - // rawhttp doesn't supports draining response bodies. + // rawhttp doesn't support draining response bodies. if resp != nil && resp.Body != nil && request.rawRequest == nil { _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() @@ -332,7 +333,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ r.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send - // a callback event so in case we recieve an interaction, correlation is possible. + // a callback event so in case we receive an interaction, correlation is possible. if hasInteractMarkers { outputEvent := r.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, request.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 1178d8f7b..7566f5a2d 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -15,7 +16,7 @@ import ( // Request contains a Network protocol request to be made from a template type Request struct { - // ID is the the optional id of the request + // ID is the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` // description: | diff --git a/v2/pkg/protocols/network/networkclientpool/clientpool.go b/v2/pkg/protocols/network/networkclientpool/clientpool.go index d35d4eb61..f80897b1b 100644 --- a/v2/pkg/protocols/network/networkclientpool/clientpool.go +++ b/v2/pkg/protocols/network/networkclientpool/clientpool.go @@ -12,7 +12,7 @@ var ( // Init initializes the clientpool implementation func Init(options *types.Options /*TODO review unused parameter*/) error { - // Don't create clients if already created in past. + // Don't create clients if already created in the past. if normalClient != nil { return nil } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index c658dddca..aa8e5e11a 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -39,7 +39,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) return false } -// Extract performs extracting operation for a extractor on model and returns true or false. +// Extract performs extracting operation for an extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 3acdbf4fa..351c3a066 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -40,7 +40,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) return false } -// Extract performs extracting operation for a extractor on model and returns true or false. +// Extract performs extracting operation for an extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { @@ -77,7 +77,7 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { return itemStr, true } -// responseToDSLMap converts a HTTP response to a map for use in DSL matching +// responseToDSLMap converts an HTTP response to a map for use in DSL matching func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 4b3984b62..9ca38333e 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,6 +1,8 @@ package protocols import ( + "go.uber.org/ratelimit" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -14,7 +16,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "go.uber.org/ratelimit" ) // Executer is an interface implemented any protocol based request executer. @@ -75,7 +76,7 @@ type Request interface { GetID() string // Match performs matching operation for a matcher on model and returns true or false. Match(data map[string]interface{}, matcher *matchers.Matcher) bool - // Extract performs extracting operation for a extractor on model and returns true or false. + // Extract performs extracting operation for an extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error diff --git a/v2/pkg/reporting/dedupe/dedupe.go b/v2/pkg/reporting/dedupe/dedupe.go index 6586415b4..7a688581d 100644 --- a/v2/pkg/reporting/dedupe/dedupe.go +++ b/v2/pkg/reporting/dedupe/dedupe.go @@ -11,10 +11,11 @@ import ( "reflect" "unsafe" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Storage is a duplicate detecting storage for nuclei scan events. @@ -93,7 +94,7 @@ func (s *Storage) Index(result *output.ResultEvent) (bool, error) { exists, err := s.storage.Has(hash, nil) if err != nil { // if we have an error, return with it but mark it as true - // since we don't want to loose an issue considering it a dupe. + // since we don't want to lose an issue considering it a dupe. return true, err } if !exists { diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index 6ce86345d..c1fb92a45 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -16,7 +16,7 @@ type Exporter struct { options *Options } -// Options contains the configuration options for github issue tracker client +// Options contains the configuration options for GitHub issue tracker client type Options struct { // Directory is the directory to export found results to Directory string `yaml:"directory"` diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 89f6dceb7..96c072c18 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -8,11 +8,12 @@ import ( "github.com/google/go-github/github" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" ) -// Integration is a client for a issue tracker integration +// Integration is a client for an issue tracker integration type Integration struct { client *github.Client options *Options diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 33d4fff01..d8ffe36e6 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -1,12 +1,13 @@ package gitlab import ( + "github.com/xanzy/go-gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" - "github.com/xanzy/go-gitlab" ) -// Integration is a client for a issue tracker integration +// Integration is a client for an issue tracker integration type Integration struct { client *gitlab.Client userID int diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index fb53948fe..aa5e31b78 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -14,7 +14,7 @@ type Workflow struct { Options *protocols.ExecuterOptions `yaml:"-"` } -// WorkflowTemplate is a template to be ran as part of a workflow +// WorkflowTemplate is a template to be run as part of a workflow type WorkflowTemplate struct { // description: | // Template is a single template or directory to execute as part of workflow. @@ -31,7 +31,7 @@ type WorkflowTemplate struct { // Matchers perform name based matching to run subtemplates for a workflow. Matchers []*Matcher `yaml:"matchers,omitempty" jsonschema:"title=name based template result matchers,description=Matchers perform name based matching to run subtemplates for a workflow"` // description: | - // Subtemplates are ran if the `template` field Template matches. + // Subtemplates are run if the `template` field Template matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates,omitempty" jsonschema:"title=subtemplate based result matchers,description=Subtemplates are ran if the template field Template matches"` // Executers perform the actual execution for the workflow template Executers []*ProtocolExecuterPair `yaml:"-"` @@ -49,6 +49,6 @@ type Matcher struct { // Name is the name of the item to match. Name string `yaml:"name,omitempty" jsonschema:"title=name of item to match,description=Name of item to match"` // description: | - // Subtemplates are ran if the name of matcher matches. + // Subtemplates are run if the name of matcher matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates,omitempty" jsonschema:"title=templates to run after match,description=Templates to run after match"` }