From c125df4ef6ec9027bf444bd640e525c3e8cfc785 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Fri, 5 Nov 2021 09:21:52 -0500 Subject: [PATCH 01/27] feat: Checking socks5 proxy before launching a scan #1001 (#1169) * "#issue1001" * changes for #issue1001 * minor changes * minor * flag consolidation and proxy file #issue1001 * readme changes * review changes * enviroment variable changes * review comment changes * review changes * removed commented out code --- README.md | 5 +- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 8 +- v2/internal/runner/options.go | 25 +--- v2/internal/runner/proxy.go | 123 ++++++++++++++++++ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 5 +- .../http/httpclientpool/clientpool.go | 42 +++--- v2/pkg/types/proxy.go | 15 +++ v2/pkg/types/types.go | 6 +- 11 files changed, 172 insertions(+), 67 deletions(-) create mode 100644 v2/internal/runner/proxy.go create mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index bfa68972e..3fc76ec7a 100644 --- a/README.md +++ b/README.md @@ -162,9 +162,8 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy, -proxy-url string URL of the HTTP proxy server - -proxy-socks-url string URL of the SOCKS proxy server - -tlog, -trace-log string file to write sent requests trace log + -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list + -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 04138acd2..0c4aa69c0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,8 +121,7 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| -|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| +|proxy|输入代理地址|nuclei -proxy ./proxy.txt| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..f958ec169 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,11 +134,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc - TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), - flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), - flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), + /* TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), + flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 59791fe63..1529466fb 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,7 +3,6 @@ package runner import ( "bufio" "errors" - "net/url" "os" "path/filepath" "strings" @@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) - // Show the user the banner showBanner() @@ -89,15 +87,10 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - - if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { + //loading the proxy server list from file or cli and test the connectivity + if err := loadProxyServers(options); err != nil { return err } - - if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { - return err - } - if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -106,19 +99,6 @@ func validateOptions(options *types.Options) error { return nil } -func validateProxyURL(proxyURL, message string) error { - if proxyURL != "" && !isValidURL(proxyURL) { - return errors.New(message) - } - - return nil -} - -func isValidURL(urlString string) bool { - _, err := url.Parse(urlString) - return err == nil -} - // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -164,7 +144,6 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) - for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go new file mode 100644 index 000000000..e254472a7 --- /dev/null +++ b/v2/internal/runner/proxy.go @@ -0,0 +1,123 @@ +package runner + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var proxyURLList []url.URL + +// loadProxyServers load list of proxy servers from file or comma seperated +func loadProxyServers(options *types.Options) error { + if len(options.Proxy) == 0 { + return nil + } + for _, p := range options.Proxy { + if proxyURL, err := validateProxyURL(p); err == nil { + proxyURLList = append(proxyURLList, proxyURL) + } else if fileutil.FileExists(p) { + file, err := os.Open(p) + if err != nil { + return fmt.Errorf("could not open proxy file: %s", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + proxy := scanner.Text() + if strings.TrimSpace(proxy) == "" { + continue + } + if proxyURL, err := validateProxyURL(proxy); err != nil { + return err + } else { + proxyURLList = append(proxyURLList, proxyURL) + } + } + } else { + return fmt.Errorf("invalid proxy file or URL provided for %s", p) + } + } + return processProxyList(options) +} + +func processProxyList(options *types.Options) error { + if len(proxyURLList) == 0 { + return fmt.Errorf("could not find any valid proxy") + } else { + done := make(chan bool) + exitCounter := make(chan bool) + counter := 0 + for _, url := range proxyURLList { + go runProxyConnectivity(url, options, done, exitCounter) + } + for { + select { + case <-done: + { + close(done) + return nil + } + case <-exitCounter: + { + if counter += 1; counter == len(proxyURLList) { + return errors.New("no reachable proxy found") + } + } + } + } + } +} + +func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { + if err := testProxyConnection(proxyURL, options.Timeout); err == nil { + if types.ProxyURL == "" && types.ProxySocksURL == "" { + assignProxyURL(proxyURL, options) + done <- true + } + } + exitCounter <- true +} + +func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { + timeout := time.Duration(timeoutDelay) * time.Second + _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) + if err != nil { + return err + } + return nil +} + +func assignProxyURL(proxyURL url.URL, options *types.Options) { + os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) + if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { + types.ProxyURL = proxyURL.String() + types.ProxySocksURL = "" + gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) + } else if proxyURL.Scheme == types.SOCKS5 { + types.ProxyURL = "" + types.ProxySocksURL = proxyURL.String() + gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) + } +} + +func validateProxyURL(proxy string) (url.URL, error) { + if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { + return *url, nil + } + return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") +} + +//isSupportedProtocol checks given protocols are supported +func isSupportedProtocol(value string) bool { + return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 +} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d630..029f8bb73 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - ProxyURL: "", - ProxySocksURL: "", + Proxy: []string{}, TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a506cbda4..a42595801 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if options.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) + if types.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d541..71a9421cc 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,9 +23,8 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - - if options.ProxyURL != "" { - if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + if types.ProxyURL != "" { + if proxyURL, err := url.Parse(types.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index a4faef07d..b3d953e7e 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,9 +128,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) + if types.ProxyURL != "" { + proxyURL, err = url.Parse(types.ProxyURL) } if err != nil { return nil, err @@ -172,27 +171,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) - if proxyErr == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) + // Attempts to overwrite the dial function with the socks proxied version + if proxyURL.Scheme == types.SOCKS5 { + var proxyAuth *proxy.Auth = &proxy.Auth{} + proxyAuth.User = proxyURL.User.Username() + proxyAuth.Password, _ = proxyURL.User.Password() + + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } else { + transport.Proxy = http.ProxyURL(proxyURL) + } } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go new file mode 100644 index 000000000..a45b4eca6 --- /dev/null +++ b/v2/pkg/types/proxy.go @@ -0,0 +1,15 @@ +package types + +const ( + HTTP_PROXY_ENV = "HTTP_PROXY" + SOCKS5 = "socks5" + HTTP = "http" + HTTPS = "https" +) + +var ( + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string +) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245..549482890 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,10 +49,8 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string + // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list + Proxy goflags.NormalizedStringSlice // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 89c6e7677201001b093ef1f792331bfe7562fe15 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 5 Nov 2021 20:09:00 +0530 Subject: [PATCH 02/27] Revert "feat: Checking socks5 proxy before launching a scan #1001 (#1169)" This reverts commit c125df4ef6ec9027bf444bd640e525c3e8cfc785. --- README.md | 5 +- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 8 +- v2/internal/runner/options.go | 25 +++- v2/internal/runner/proxy.go | 123 ------------------ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 5 +- .../http/httpclientpool/clientpool.go | 38 +++--- v2/pkg/types/proxy.go | 15 --- v2/pkg/types/types.go | 6 +- 11 files changed, 65 insertions(+), 170 deletions(-) delete mode 100644 v2/internal/runner/proxy.go delete mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index 3fc76ec7a..bfa68972e 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,9 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - -trace-log string file to write sent requests trace log + -proxy, -proxy-url string URL of the HTTP proxy server + -proxy-socks-url string URL of the SOCKS proxy server + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 0c4aa69c0..04138acd2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,7 +121,8 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy|输入代理地址|nuclei -proxy ./proxy.txt| +|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| +|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f958ec169..d2f8ebb9f 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,9 +134,11 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), + /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc + TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), + flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 1529466fb..59791fe63 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,6 +3,7 @@ package runner import ( "bufio" "errors" + "net/url" "os" "path/filepath" "strings" @@ -23,6 +24,7 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) + // Show the user the banner showBanner() @@ -87,10 +89,15 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - //loading the proxy server list from file or cli and test the connectivity - if err := loadProxyServers(options); err != nil { + + if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { return err } + + if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { + return err + } + if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -99,6 +106,19 @@ func validateOptions(options *types.Options) error { return nil } +func validateProxyURL(proxyURL, message string) error { + if proxyURL != "" && !isValidURL(proxyURL) { + return errors.New(message) + } + + return nil +} + +func isValidURL(urlString string) bool { + _, err := url.Parse(urlString) + return err == nil +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -144,6 +164,7 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) + for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go deleted file mode 100644 index e254472a7..000000000 --- a/v2/internal/runner/proxy.go +++ /dev/null @@ -1,123 +0,0 @@ -package runner - -import ( - "bufio" - "errors" - "fmt" - "net" - "net/url" - "os" - "strings" - "time" - - "github.com/projectdiscovery/fileutil" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -var proxyURLList []url.URL - -// loadProxyServers load list of proxy servers from file or comma seperated -func loadProxyServers(options *types.Options) error { - if len(options.Proxy) == 0 { - return nil - } - for _, p := range options.Proxy { - if proxyURL, err := validateProxyURL(p); err == nil { - proxyURLList = append(proxyURLList, proxyURL) - } else if fileutil.FileExists(p) { - file, err := os.Open(p) - if err != nil { - return fmt.Errorf("could not open proxy file: %s", err) - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - proxy := scanner.Text() - if strings.TrimSpace(proxy) == "" { - continue - } - if proxyURL, err := validateProxyURL(proxy); err != nil { - return err - } else { - proxyURLList = append(proxyURLList, proxyURL) - } - } - } else { - return fmt.Errorf("invalid proxy file or URL provided for %s", p) - } - } - return processProxyList(options) -} - -func processProxyList(options *types.Options) error { - if len(proxyURLList) == 0 { - return fmt.Errorf("could not find any valid proxy") - } else { - done := make(chan bool) - exitCounter := make(chan bool) - counter := 0 - for _, url := range proxyURLList { - go runProxyConnectivity(url, options, done, exitCounter) - } - for { - select { - case <-done: - { - close(done) - return nil - } - case <-exitCounter: - { - if counter += 1; counter == len(proxyURLList) { - return errors.New("no reachable proxy found") - } - } - } - } - } -} - -func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { - if err := testProxyConnection(proxyURL, options.Timeout); err == nil { - if types.ProxyURL == "" && types.ProxySocksURL == "" { - assignProxyURL(proxyURL, options) - done <- true - } - } - exitCounter <- true -} - -func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { - timeout := time.Duration(timeoutDelay) * time.Second - _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) - if err != nil { - return err - } - return nil -} - -func assignProxyURL(proxyURL url.URL, options *types.Options) { - os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) - if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { - types.ProxyURL = proxyURL.String() - types.ProxySocksURL = "" - gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) - } else if proxyURL.Scheme == types.SOCKS5 { - types.ProxyURL = "" - types.ProxySocksURL = proxyURL.String() - gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) - } -} - -func validateProxyURL(proxy string) (url.URL, error) { - if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { - return *url, nil - } - return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") -} - -//isSupportedProtocol checks given protocols are supported -func isSupportedProtocol(value string) bool { - return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 -} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 029f8bb73..d62b1d630 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,7 +51,8 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - Proxy: []string{}, + ProxyURL: "", + ProxySocksURL: "", TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a42595801..a506cbda4 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if types.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) + if options.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 71a9421cc..b4747d541 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,8 +23,9 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - if types.ProxyURL != "" { - if proxyURL, err := url.Parse(types.ProxyURL); err == nil { + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index b3d953e7e..a4faef07d 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,8 +128,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - if types.ProxyURL != "" { - proxyURL, err = url.Parse(types.ProxyURL) + + if options.ProxyURL != "" { + proxyURL, err = url.Parse(options.ProxyURL) } if err != nil { return nil, err @@ -171,24 +172,27 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - if proxyURL != nil { - // Attempts to overwrite the dial function with the socks proxied version - if proxyURL.Scheme == types.SOCKS5 { - var proxyAuth *proxy.Auth = &proxy.Auth{} - proxyAuth.User = proxyURL.User.Username() - proxyAuth.Password, _ = proxyURL.User.Password() - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + // Attempts to overwrite the dial function with the socks proxied version + if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } else { - transport.Proxy = http.ProxyURL(proxyURL) + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } + if proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go deleted file mode 100644 index a45b4eca6..000000000 --- a/v2/pkg/types/proxy.go +++ /dev/null @@ -1,15 +0,0 @@ -package types - -const ( - HTTP_PROXY_ENV = "HTTP_PROXY" - SOCKS5 = "socks5" - HTTP = "http" - HTTPS = "https" -) - -var ( - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string -) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 549482890..d258f4245 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,8 +49,10 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - Proxy goflags.NormalizedStringSlice + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 348d86d330614ba6720f52bdc9e4cbc27fc88daf Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 5 Nov 2021 20:10:21 +0530 Subject: [PATCH 03/27] Revert "feat: Checking socks5 proxy before launching a scan #1001 (#1169)" This reverts commit c125df4ef6ec9027bf444bd640e525c3e8cfc785. --- README.md | 5 +- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 8 +- v2/internal/runner/options.go | 25 +++- v2/internal/runner/proxy.go | 123 ------------------ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 5 +- .../http/httpclientpool/clientpool.go | 38 +++--- v2/pkg/types/proxy.go | 15 --- v2/pkg/types/types.go | 6 +- 11 files changed, 65 insertions(+), 170 deletions(-) delete mode 100644 v2/internal/runner/proxy.go delete mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index 3fc76ec7a..bfa68972e 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,9 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - -trace-log string file to write sent requests trace log + -proxy, -proxy-url string URL of the HTTP proxy server + -proxy-socks-url string URL of the SOCKS proxy server + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 0c4aa69c0..04138acd2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,7 +121,8 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy|输入代理地址|nuclei -proxy ./proxy.txt| +|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| +|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f958ec169..d2f8ebb9f 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,9 +134,11 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), + /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc + TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), + flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 1529466fb..59791fe63 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,6 +3,7 @@ package runner import ( "bufio" "errors" + "net/url" "os" "path/filepath" "strings" @@ -23,6 +24,7 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) + // Show the user the banner showBanner() @@ -87,10 +89,15 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - //loading the proxy server list from file or cli and test the connectivity - if err := loadProxyServers(options); err != nil { + + if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { return err } + + if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { + return err + } + if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -99,6 +106,19 @@ func validateOptions(options *types.Options) error { return nil } +func validateProxyURL(proxyURL, message string) error { + if proxyURL != "" && !isValidURL(proxyURL) { + return errors.New(message) + } + + return nil +} + +func isValidURL(urlString string) bool { + _, err := url.Parse(urlString) + return err == nil +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -144,6 +164,7 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) + for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go deleted file mode 100644 index e254472a7..000000000 --- a/v2/internal/runner/proxy.go +++ /dev/null @@ -1,123 +0,0 @@ -package runner - -import ( - "bufio" - "errors" - "fmt" - "net" - "net/url" - "os" - "strings" - "time" - - "github.com/projectdiscovery/fileutil" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -var proxyURLList []url.URL - -// loadProxyServers load list of proxy servers from file or comma seperated -func loadProxyServers(options *types.Options) error { - if len(options.Proxy) == 0 { - return nil - } - for _, p := range options.Proxy { - if proxyURL, err := validateProxyURL(p); err == nil { - proxyURLList = append(proxyURLList, proxyURL) - } else if fileutil.FileExists(p) { - file, err := os.Open(p) - if err != nil { - return fmt.Errorf("could not open proxy file: %s", err) - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - proxy := scanner.Text() - if strings.TrimSpace(proxy) == "" { - continue - } - if proxyURL, err := validateProxyURL(proxy); err != nil { - return err - } else { - proxyURLList = append(proxyURLList, proxyURL) - } - } - } else { - return fmt.Errorf("invalid proxy file or URL provided for %s", p) - } - } - return processProxyList(options) -} - -func processProxyList(options *types.Options) error { - if len(proxyURLList) == 0 { - return fmt.Errorf("could not find any valid proxy") - } else { - done := make(chan bool) - exitCounter := make(chan bool) - counter := 0 - for _, url := range proxyURLList { - go runProxyConnectivity(url, options, done, exitCounter) - } - for { - select { - case <-done: - { - close(done) - return nil - } - case <-exitCounter: - { - if counter += 1; counter == len(proxyURLList) { - return errors.New("no reachable proxy found") - } - } - } - } - } -} - -func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { - if err := testProxyConnection(proxyURL, options.Timeout); err == nil { - if types.ProxyURL == "" && types.ProxySocksURL == "" { - assignProxyURL(proxyURL, options) - done <- true - } - } - exitCounter <- true -} - -func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { - timeout := time.Duration(timeoutDelay) * time.Second - _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) - if err != nil { - return err - } - return nil -} - -func assignProxyURL(proxyURL url.URL, options *types.Options) { - os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) - if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { - types.ProxyURL = proxyURL.String() - types.ProxySocksURL = "" - gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) - } else if proxyURL.Scheme == types.SOCKS5 { - types.ProxyURL = "" - types.ProxySocksURL = proxyURL.String() - gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) - } -} - -func validateProxyURL(proxy string) (url.URL, error) { - if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { - return *url, nil - } - return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") -} - -//isSupportedProtocol checks given protocols are supported -func isSupportedProtocol(value string) bool { - return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 -} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 029f8bb73..d62b1d630 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,7 +51,8 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - Proxy: []string{}, + ProxyURL: "", + ProxySocksURL: "", TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a42595801..a506cbda4 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if types.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) + if options.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 71a9421cc..b4747d541 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,8 +23,9 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - if types.ProxyURL != "" { - if proxyURL, err := url.Parse(types.ProxyURL); err == nil { + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index b3d953e7e..a4faef07d 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,8 +128,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - if types.ProxyURL != "" { - proxyURL, err = url.Parse(types.ProxyURL) + + if options.ProxyURL != "" { + proxyURL, err = url.Parse(options.ProxyURL) } if err != nil { return nil, err @@ -171,24 +172,27 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - if proxyURL != nil { - // Attempts to overwrite the dial function with the socks proxied version - if proxyURL.Scheme == types.SOCKS5 { - var proxyAuth *proxy.Auth = &proxy.Auth{} - proxyAuth.User = proxyURL.User.Username() - proxyAuth.Password, _ = proxyURL.User.Password() - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + // Attempts to overwrite the dial function with the socks proxied version + if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } else { - transport.Proxy = http.ProxyURL(proxyURL) + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } + if proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go deleted file mode 100644 index a45b4eca6..000000000 --- a/v2/pkg/types/proxy.go +++ /dev/null @@ -1,15 +0,0 @@ -package types - -const ( - HTTP_PROXY_ENV = "HTTP_PROXY" - SOCKS5 = "socks5" - HTTP = "http" - HTTPS = "https" -) - -var ( - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string -) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 549482890..d258f4245 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,8 +49,10 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - Proxy goflags.NormalizedStringSlice + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 6c819d7917f2779f32cce266ed6e56ea783df1e2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 10:44:31 +0530 Subject: [PATCH 04/27] Added additional variables for network + simplified logic --- v2/pkg/protocols/network/network.go | 16 ++------- v2/pkg/protocols/network/network_test.go | 15 ++------- v2/pkg/protocols/network/request.go | 41 ++++++++++++++---------- v2/pkg/protocols/network/request_test.go | 9 ++---- 4 files changed, 32 insertions(+), 49 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index a5274eff6..4495d0b03 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -2,7 +2,6 @@ package network import ( "fmt" - "net" "strings" "github.com/pkg/errors" @@ -78,9 +77,8 @@ type Request struct { } type addressKV struct { - ip string - port string - tls bool + address string + tls bool } // Input is the input to send on the network @@ -136,15 +134,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { shouldUseTLS = true address = strings.TrimPrefix(address, "tls://") } - if strings.Contains(address, ":") { - addressHost, addressPort, portErr := net.SplitHostPort(address) - if portErr != nil { - return errors.Wrap(portErr, "could not parse address") - } - request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) - } else { - request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS}) - } + request.addresses = append(request.addresses, addressKV{address: address, tls: shouldUseTLS}) } // Pre-compile any input dsl functions before executing the request. for _, input := range request.Inputs { diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 180b34192..733211a9e 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -17,7 +17,7 @@ func TestNetworkCompileMake(t *testing.T) { templateID := "testing-network" request := &Request{ ID: templateID, - Address: []string{"{{Hostname}}", "{{Hostname}}:8082", "tls://{{Hostname}}:443"}, + Address: []string{"tls://{{Hostname}}:443"}, ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, } @@ -28,17 +28,8 @@ func TestNetworkCompileMake(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") - require.Equal(t, 3, len(request.addresses), "could not get correct number of input address") - t.Run("check-host", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[0].ip, "could not get correct host") - }) - t.Run("check-host-with-port", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[1].ip, "could not get correct host with port") - require.Equal(t, "8082", request.addresses[1].port, "could not get correct port for host") - }) + require.Equal(t, 1, len(request.addresses), "could not get correct number of input address") t.Run("check-tls-with-port", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[2].ip, "could not get correct host with port") - require.Equal(t, "443", request.addresses[2].port, "could not get correct port for host") - require.True(t, request.addresses[2].tls, "could not get correct port for host") + require.True(t, request.addresses[0].tls, "could not get correct port for host") }) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index acbbac9af..2446bc4db 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -41,18 +41,10 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } for _, kv := range request.addresses { - actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address}) - if kv.port != "" { - if strings.Contains(address, ":") { - actualAddress, _, _ = net.SplitHostPort(actualAddress) - } - actualAddress = net.JoinHostPort(actualAddress, kv.port) - } - if input != "" { - input = actualAddress - } + variables := generateNetworkVariables(address) + actualAddress := replacer.Replace(kv.address, variables) - if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -61,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // executeAddress executes the request for an address -func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") request.options.Output.Request(request.options.TemplateID, address, "network", err) @@ -80,27 +72,27 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho break } value = generators.MergeMaps(value, payloads) - if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { - value := generators.MergeMaps(map[string]interface{}{}, payloads) - if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + value := generators.CopyMap(payloads) + if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } return nil } -func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn err error ) - request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + request.dynamicValues = generators.MergeMaps(payloads, variables) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host @@ -257,3 +249,18 @@ func getAddress(toTest string) (string, error) { } return toTest, nil } + +func generateNetworkVariables(input string) map[string]interface{} { + if !strings.Contains(input, ":") { + return map[string]interface{}{"Hostname": input, "Host": input} + } + host, port, err := net.SplitHostPort(input) + if err != nil { + return map[string]interface{}{"Hostname": input} + } + return map[string]interface{}{ + "Host": host, + "Port": port, + "Hostname": input, + } +} diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index d1f266b48..4cc578919 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -50,7 +50,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse url") - request.Address[0] = "{{Hostname}}:" + parsed.Port() + request.Address[0] = "{{Hostname}}" request.Inputs = append(request.Inputs, &Input{Data: fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host)}) executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -84,12 +84,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }) require.Nil(t, err, "could not execute network request") }) - require.NotNil(t, finalEvent, "could not get event output from request") - require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results") - require.Equal(t, "

Example Domain

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") - finalEvent = nil + require.Nil(t, finalEvent, "could not get event output from request") request.Inputs[0].Type = "hex" request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) From 4bccb6cf8a830724b3bda624cb3bfe43eff31683 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 24 Nov 2021 17:09:38 +0200 Subject: [PATCH 05/27] fix: File and directory creation permission changes Directories: 0755 (5 - group and other can read cd into the directory and read it's content) Files: 0644 (4 - group and other can only read the created files) Tests files: 0777 --- v2/cmd/cve-annotate/main.go | 3 ++- v2/cmd/docgen/docgen.go | 5 +++-- v2/internal/runner/update.go | 10 +++++----- v2/internal/runner/update_test.go | 11 ++++++----- v2/pkg/catalog/config/config.go | 6 +++--- v2/pkg/protocols/file/find_test.go | 2 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/offlinehttp/find_test.go | 2 +- v2/pkg/reporting/exporters/markdown/markdown.go | 2 +- 9 files changed, 23 insertions(+), 20 deletions(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index 4786ad245..45ede0420 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Ice3man543/nvd" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) @@ -135,7 +136,7 @@ func getCVEData(client *nvd.Client, filePath, data string) { } newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock) if changed { - _ = ioutil.WriteFile(filePath, []byte(newTemplate), 0777) + _ = ioutil.WriteFile(filePath, []byte(newTemplate), 0644) fmt.Printf("Wrote updated template to %s\n", filePath) } } diff --git a/v2/cmd/docgen/docgen.go b/v2/cmd/docgen/docgen.go index 5cd359c94..aa5405ee9 100644 --- a/v2/cmd/docgen/docgen.go +++ b/v2/cmd/docgen/docgen.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/alecthomas/jsonschema" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) @@ -21,7 +22,7 @@ func main() { if err != nil { log.Fatalf("Could not encode docs: %s\n", err) } - err = ioutil.WriteFile(os.Args[1], data, 0777) + err = ioutil.WriteFile(os.Args[1], data, 0644) if err != nil { log.Fatalf("Could not write docs: %s\n", err) } @@ -43,7 +44,7 @@ func main() { for _, match := range pathRegex.FindAllStringSubmatch(schema, -1) { schema = strings.ReplaceAll(schema, match[0], match[1]) } - err = ioutil.WriteFile(os.Args[2], []byte(schema), 0777) + err = ioutil.WriteFile(os.Args[2], []byte(schema), 0644) if err != nil { log.Fatalf("Could not write jsonschema: %s\n", err) } diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 217f34008..eaef33b39 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -54,7 +54,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju return err } configDir := filepath.Join(home, ".config", "nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) + _ = os.MkdirAll(configDir, 0755) if err := r.readInternalConfigurationFile(home, configDir); err != nil { return errors.Wrap(err, "could not read configuration file") @@ -266,7 +266,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU } // Create the template folder if it doesn't exist - if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm); err != nil { + if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, 0755); err != nil { return nil, fmt.Errorf("failed to create template base folder: %s", err) } @@ -291,7 +291,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU buffer.WriteString("\n") } - if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm); err != nil { + if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), 0644); err != nil { return nil, errors.Wrap(err, "could not write new additions file") } return results, err @@ -331,7 +331,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat } results.totalCount++ templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath) - if err := os.MkdirAll(templateDirectory, os.ModePerm); err != nil { + if err := os.MkdirAll(templateDirectory, 0755); err != nil { return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) } @@ -341,7 +341,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) { isAddition = true } - templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) + templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { templateFile.Close() return nil, fmt.Errorf("could not create uncompressed file: %s", err) diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index bc0888341..62564acd8 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -12,10 +12,11 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/stretchr/testify/require" ) func TestDownloadReleaseAndUnzipAddition(t *testing.T) { @@ -25,7 +26,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(baseTemplates) - err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create write base file") err = zipFromDirectory("base.zip", baseTemplates) @@ -50,9 +51,9 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(newTempDir) - err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create base file") - err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create new file") err = zipFromDirectory("new.zip", newTempDir) @@ -77,7 +78,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(baseTemplates) - err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create write base file") err = zipFromDirectory("base.zip", baseTemplates) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 91b153cf2..87e6c672d 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -34,7 +34,7 @@ func getConfigDetails() (string, error) { return "", errors.Wrap(err, "could not get home directory") } configDir := filepath.Join(homeDir, ".config", "nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) + _ = os.MkdirAll(configDir, 0755) templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) return templatesConfigFile, nil } @@ -67,7 +67,7 @@ func WriteConfiguration(config *Config) error { if err != nil { return err } - file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777) + file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } @@ -112,7 +112,7 @@ func getIgnoreFilePath() string { home, err := os.UserHomeDir() if err == nil { configDir := filepath.Join(home, ".config", "nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) + _ = os.MkdirAll(configDir, 0755) defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile) return defIgnoreFilePath diff --git a/v2/pkg/protocols/file/find_test.go b/v2/pkg/protocols/file/find_test.go index 7262bb26a..58eb128b6 100644 --- a/v2/pkg/protocols/file/find_test.go +++ b/v2/pkg/protocols/file/find_test.go @@ -44,7 +44,7 @@ func TestFindInputPaths(t *testing.T) { "test.js": "TEST", } for k, v := range files { - err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777) + err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), os.ModePerm) require.Nil(t, err, "could not write temporary file") } expected := []string{"config.yaml", "final.yaml", "test.js"} diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index ba057dafe..a25d5538f 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -57,7 +57,7 @@ func TestFileExecuteWithResults(t *testing.T) { "config.yaml": "TEST\r\n1.1.1.1\r\n", } for k, v := range files { - err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777) + err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), os.ModePerm) require.Nil(t, err, "could not write temporary file") } diff --git a/v2/pkg/protocols/offlinehttp/find_test.go b/v2/pkg/protocols/offlinehttp/find_test.go index d6b933ce2..367d00096 100644 --- a/v2/pkg/protocols/offlinehttp/find_test.go +++ b/v2/pkg/protocols/offlinehttp/find_test.go @@ -40,7 +40,7 @@ func TestFindResponses(t *testing.T) { "test.txt": "TEST", } for k, v := range files { - err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777) + err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), os.ModePerm) require.Nil(t, err, "could not write temporary file") } expected := []string{"config.txt", "final.txt", "test.txt"} diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 3e779ea1e..31c356af9 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -32,7 +32,7 @@ func New(options *Options) (*Exporter, error) { } directory = dir } - _ = os.MkdirAll(directory, os.ModePerm) + _ = os.MkdirAll(directory, 0755) return &Exporter{options: options, directory: directory}, nil } From f9c214a66f37a00fc29fe64232b690e6285ec59c Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 14:46:45 +0200 Subject: [PATCH 06/27] refactor: update logic refactor to make it more testable Introduced logic to test for zip slip (path traversal) --- v2/internal/runner/update.go | 111 +++++++++++++++++++----------- v2/internal/runner/update_test.go | 37 ++++++++++ 2 files changed, 109 insertions(+), 39 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index eaef33b39..55aec7049 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -316,60 +316,42 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat // If the path isn't found in new update after being read from the previous checksum, // it is removed. This allows us fine-grained control over the download process // as well as solves a long problem with nuclei-template updates. - checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum") + configuredTemplateDirectory := r.templatesConfig.TemplatesDirectory + checksumFile := filepath.Join(configuredTemplateDirectory, ".checksum") templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile) for _, zipTemplateFile := range zipReader.File { - directory, name := filepath.Split(zipTemplateFile.Name) - if name == "" { + templateAbsolutePath, skipFile, err := calculateTemplateAbsolutePath(zipTemplateFile.Name, configuredTemplateDirectory) + if err != nil { + return nil, err + } + if skipFile { continue } - paths := strings.Split(directory, string(os.PathSeparator)) - finalPath := filepath.Join(paths[1:]...) - - if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { - continue - } - results.totalCount++ - templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath) - if err := os.MkdirAll(templateDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) - } - - templatePath := filepath.Join(templateDirectory, name) isAddition := false - if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) { + if _, statErr := os.Stat(templateAbsolutePath); os.IsNotExist(statErr) { isAddition = true } - templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + + newTemplateChecksum, err := writeUnZippedTemplateFile(err, templateAbsolutePath, zipTemplateFile) if err != nil { - templateFile.Close() - return nil, fmt.Errorf("could not create uncompressed file: %s", err) + return nil, err } - zipTemplateFileReader, err := zipTemplateFile.Open() + oldTemplateChecksum, checksumOk := templateChecksumsMap[templateAbsolutePath] + + relativeTemplatePath, err := filepath.Rel(configuredTemplateDirectory, templateAbsolutePath) if err != nil { - templateFile.Close() - return nil, fmt.Errorf("could not open archive to extract file: %s", err) + return nil, fmt.Errorf("could not calculate relative path for template: %s. %s", templateAbsolutePath, err) } - hasher := md5.New() - // Save file and also read into hasher for md5 - if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, hasher)); err != nil { - templateFile.Close() - return nil, fmt.Errorf("could not write template file: %s", err) - } - templateFile.Close() - - oldChecksum, checksumOK := templateChecksumsMap[templatePath] - - checksum := hex.EncodeToString(hasher.Sum(nil)) if isAddition { - results.additions = append(results.additions, filepath.Join(finalPath, name)) - } else if checksumOK && oldChecksum[0] != checksum { - results.modifications = append(results.modifications, filepath.Join(finalPath, name)) + results.additions = append(results.additions, relativeTemplatePath) + } else if checksumOk && oldTemplateChecksum[0] != newTemplateChecksum { + results.modifications = append(results.modifications, relativeTemplatePath) } - results.checksums[templatePath] = checksum + results.checksums[templateAbsolutePath] = newTemplateChecksum + results.totalCount++ } // If we don't find the previous file in the newly downloaded list, @@ -378,12 +360,63 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat _, ok := results.checksums[templatePath] if !ok && templateChecksums[0] == templateChecksums[1] { _ = os.Remove(templatePath) - results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) + results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, configuredTemplateDirectory), string(os.PathSeparator))) } } return results, nil } +func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { + templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return "", fmt.Errorf("could not create template file: %s", err) + } + + zipTemplateFileReader, err := zipTemplateFile.Open() + if err != nil { + _ = templateFile.Close() + return "", fmt.Errorf("could not open archive to extract file: %s", err) + } + + md5Hash := md5.New() + + // Save file and also read into hash.Hash for md5 + if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, md5Hash)); err != nil { + _ = templateFile.Close() + return "", fmt.Errorf("could not write template file: %s", err) + } + + if err := templateFile.Close(); err != nil { + return "", fmt.Errorf("could not close file newly created template file: %s", err) + } + + checksum := hex.EncodeToString(md5Hash.Sum(nil)) + return checksum, nil +} + +func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory string) (string, bool, error) { + directory, fileName := filepath.Split(zipFilePath) + + if strings.TrimSpace(fileName) == "" || strings.HasPrefix(fileName, ".") || strings.EqualFold(fileName, "README.md") { + return "", true, nil + } + + directoryPathChunks := strings.Split(directory, string(os.PathSeparator)) + relativeDirectoryPathWithoutZipRoot := filepath.Join(directoryPathChunks[1:]...) + + if strings.HasPrefix(relativeDirectoryPathWithoutZipRoot, ".") { + return "", true, nil + } + + templateDirectory := filepath.Join(configuredTemplateDirectory, relativeDirectoryPathWithoutZipRoot) + + if err := os.MkdirAll(templateDirectory, 0755); err != nil { + return "", false, fmt.Errorf("failed to create template folder: %s. %s", templateDirectory, err) + } + + return filepath.Join(templateDirectory, fileName), false, nil +} + // createTemplateChecksumsMap reads the previous checksum file from the disk. // Creates a map of template paths and their previous and currently calculated checksums as values. func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) { diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index 62564acd8..e8153b42e 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -119,6 +119,43 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions") } +func TestCalculateTemplateAbsolutePath(t *testing.T) { + configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") + defer os.RemoveAll(configuredTemplateDirectory) + + t.Run("positive scenarios", func(t *testing.T) { + zipFilePathsExpectedPathsMap := map[string]string{ + "nuclei-templates/cve/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test.yaml"), + "nuclei-templates/cve/test/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test/test.yaml"), + } + + for filePathFromZip, expectedTemplateAbsPath := range zipFilePathsExpectedPathsMap { + calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) + require.Nil(t, err) + require.Equal(t, expectedTemplateAbsPath, calculatedTemplateAbsPath) + require.False(t, skipFile) + } + }) + + t.Run("negative scenarios", func(t *testing.T) { + filePathsFromZip := []string{ + "./../nuclei-templates/../cve/test.yaml", + "nuclei-templates/../cve/test.yaml", + "nuclei-templates/cve/../test.yaml", + "nuclei-templates/././../cve/test.yaml", + "nuclei-templates/.././../cve/test.yaml", + "nuclei-templates/.././../cve/../test.yaml", + } + + for _, filePathFromZip := range filePathsFromZip { + calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) + require.Nil(t, err) + require.True(t, skipFile) + require.Equal(t, "", calculatedTemplateAbsPath) + } + }) +} + func zipFromDirectory(zipPath, directory string) error { file, err := os.Create(zipPath) if err != nil { From fdd22ab668e18f001cbb56ce721e80b225b84667 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 15:18:46 +0200 Subject: [PATCH 07/27] refactor: Wrap errors using %w in fmt.Errorf see: * https://github.com/xxpxxxxp/intellij-plugin-golangci-lint/blob/master/explanation/goerr113.md * https://go.dev/blog/go1.13-errors#wrapping-errors-with-w --- v2/internal/runner/proxy.go | 2 +- v2/internal/runner/update.go | 24 ++++++++++++------------ v2/pkg/protocols/http/build_request.go | 4 ++-- v2/pkg/protocols/http/raw/raw.go | 8 ++++---- v2/pkg/reporting/trackers/jira/jira.go | 5 +++-- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index e254472a7..3d2e47fd8 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -28,7 +28,7 @@ func loadProxyServers(options *types.Options) error { } else if fileutil.FileExists(p) { file, err := os.Open(p) if err != nil { - return fmt.Errorf("could not open proxy file: %s", err) + return fmt.Errorf("could not open proxy file: %w", err) } defer file.Close() scanner := bufio.NewScanner(file) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 55aec7049..8a0fa6c5a 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -242,12 +242,12 @@ func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.Repositor func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { - return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed to create HTTP request to %s: %w", downloadURL, err) } res, err := http.DefaultClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed to download a release file from %s: %w", downloadURL, err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { @@ -256,23 +256,23 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU buf, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("failed to create buffer for zip file: %s", err) + return nil, fmt.Errorf("failed to create buffer for zip file: %w", err) } reader := bytes.NewReader(buf) zipReader, err := zip.NewReader(reader, reader.Size()) if err != nil { - return nil, fmt.Errorf("failed to uncompress zip file: %s", err) + return nil, fmt.Errorf("failed to uncompress zip file: %w", err) } // Create the template folder if it doesn't exist if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create template base folder: %s", err) + return nil, fmt.Errorf("failed to create template base folder: %w", err) } results, err := r.compareAndWriteTemplates(zipReader) if err != nil { - return nil, fmt.Errorf("failed to write templates: %s", err) + return nil, fmt.Errorf("failed to write templates: %w", err) } if r.options.Verbose { @@ -342,7 +342,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat relativeTemplatePath, err := filepath.Rel(configuredTemplateDirectory, templateAbsolutePath) if err != nil { - return nil, fmt.Errorf("could not calculate relative path for template: %s. %s", templateAbsolutePath, err) + return nil, fmt.Errorf("could not calculate relative path for template: %s. %w", templateAbsolutePath, err) } if isAddition { @@ -369,13 +369,13 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - return "", fmt.Errorf("could not create template file: %s", err) + return "", fmt.Errorf("could not create template file: %w", err) } zipTemplateFileReader, err := zipTemplateFile.Open() if err != nil { _ = templateFile.Close() - return "", fmt.Errorf("could not open archive to extract file: %s", err) + return "", fmt.Errorf("could not open archive to extract file: %w", err) } md5Hash := md5.New() @@ -383,11 +383,11 @@ func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTempla // Save file and also read into hash.Hash for md5 if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, md5Hash)); err != nil { _ = templateFile.Close() - return "", fmt.Errorf("could not write template file: %s", err) + return "", fmt.Errorf("could not write template file: %w", err) } if err := templateFile.Close(); err != nil { - return "", fmt.Errorf("could not close file newly created template file: %s", err) + return "", fmt.Errorf("could not close file newly created template file: %w", err) } checksum := hex.EncodeToString(md5Hash.Sum(nil)) @@ -411,7 +411,7 @@ func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory stri templateDirectory := filepath.Join(configuredTemplateDirectory, relativeDirectoryPathWithoutZipRoot) if err := os.MkdirAll(templateDirectory, 0755); err != nil { - return "", false, fmt.Errorf("failed to create template folder: %s. %s", templateDirectory, err) + return "", false, fmt.Errorf("failed to create template folder: %s. %w", templateDirectory, err) } return filepath.Join(templateDirectory, fileName), false, nil diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8c8a2dbc8..0fb0a1f0f 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -124,7 +124,7 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int reader := bufio.NewReader(strings.NewReader(data)) s, err := reader.ReadString('\n') if err != nil { - return nil, fmt.Errorf("could not read request: %s", err) + return nil, fmt.Errorf("could not read request: %w", err) } parts := strings.Split(s, " ") @@ -133,7 +133,7 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int } parsed, err := url.Parse(parts[1]) if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) + return nil, fmt.Errorf("could not parse request URL: %w", err) } values := generators.MergeMaps( generators.MergeMaps(dynamicValues, generateVariables(parsed, false)), diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index e2d7f8ebe..df4ff5b65 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -33,7 +33,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { parsedURL, err := url.Parse(baseURL) if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) + return nil, fmt.Errorf("could not parse request URL: %w", err) } if unsafe { @@ -42,7 +42,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { reader := bufio.NewReader(strings.NewReader(request)) s, err := reader.ReadString('\n') if err != nil { - return nil, fmt.Errorf("could not read request: %s", err) + return nil, fmt.Errorf("could not read request: %w", err) } parts := strings.Split(s, " ") @@ -101,7 +101,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if !unsafe && strings.HasPrefix(parts[1], "http") { parsed, parseErr := url.Parse(parts[1]) if parseErr != nil { - return nil, fmt.Errorf("could not parse request URL: %s", parseErr) + return nil, fmt.Errorf("could not parse request URL: %w", parseErr) } rawRequest.Path = parsed.Path @@ -133,7 +133,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { // Set the request body b, err := ioutil.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("could not read request body: %s", err) + return nil, fmt.Errorf("could not read request body: %w", err) } rawRequest.Data = string(b) if !mutlipartRequest { diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 16ef1b2a7..dede2b2ed 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/andygrunwald/go-jira" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -103,7 +104,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { d, _ := ioutil.ReadAll(resp.Body) data = string(d) } - return fmt.Errorf("%s => %s", err, data) + return fmt.Errorf("%w => %s", err, data) } return nil } @@ -140,7 +141,7 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (string, erro d, _ := ioutil.ReadAll(resp.Body) data = string(d) } - return "", fmt.Errorf("%s => %s", err, data) + return "", fmt.Errorf("%w => %s", err, data) } switch resp.Total { From bebe5cbcfd06d9429628c6bee170bb455f3ed574 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 15:39:10 +0200 Subject: [PATCH 08/27] refactor: Remove redundant character escapes from regexes --- v2/cmd/docgen/docgen.go | 2 +- v2/pkg/protocols/common/expressions/expressions.go | 3 ++- v2/pkg/protocols/common/expressions/variables.go | 2 +- v2/pkg/protocols/offlinehttp/read_response.go | 2 +- v2/pkg/templates/preprocessors.go | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/v2/cmd/docgen/docgen.go b/v2/cmd/docgen/docgen.go index aa5405ee9..1c8e9779a 100644 --- a/v2/cmd/docgen/docgen.go +++ b/v2/cmd/docgen/docgen.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) -var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z\.]+)`) +var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`) func main() { // Generate yaml syntax documentation diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 5a4ba2f3a..4f3a4b613 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -4,12 +4,13 @@ import ( "regexp" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) -var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`) +var templateExpressionRegex = regexp.MustCompile(`(?m){{[^}]+}}["')}]*`) // Evaluate checks if the match contains a dynamic variable, for each // found one we will check if it's an expression and can diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 26b566317..2aba5c324 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -6,7 +6,7 @@ import ( "strings" ) -var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`) +var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|{){2}([^}]+)(?:%7[D|d]|}){2}["')}]*`) // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. diff --git a/v2/pkg/protocols/offlinehttp/read_response.go b/v2/pkg/protocols/offlinehttp/read_response.go index f864b5ab8..c6e1d22d1 100644 --- a/v2/pkg/protocols/offlinehttp/read_response.go +++ b/v2/pkg/protocols/offlinehttp/read_response.go @@ -8,7 +8,7 @@ import ( "strings" ) -var noMinor = regexp.MustCompile(`HTTP\/([0-9]) `) +var noMinor = regexp.MustCompile(`HTTP/([0-9]) `) // readResponseFromString reads a raw http response from a string. func readResponseFromString(data string) (*http.Response, error) { diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index ef6edeb1c..6a1355e26 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -12,7 +12,7 @@ type Preprocessor interface { Process(data []byte) []byte } -var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`) +var preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) // expandPreprocessors expands the pre-processors if any for a template data. func (t *Template) expandPreprocessors(data []byte) []byte { From db0d2b00394ddb48347783f3a3132f373e73327d Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 16:33:16 +0200 Subject: [PATCH 09/27] fix: markdown anchor corrections --- DESIGN.md | 4 ++-- README_CN.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 807852739..8ff4dcb92 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -554,7 +554,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/reporting](./v2/pkg/reporting) - Reporting modules for nuclei. - [v2/pkg/reporting/exporters/sarif](./v2/pkg/reporting/exporters/sarif) - Sarif Result Exporter - [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter -- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/e) - Elasticsearch Result Exporter +- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/es) - Elasticsearch Result Exporter - [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results - [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter - [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter @@ -577,7 +577,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/model](./v2/pkg/model) - Template Info + misc - [v2/pkg/templates](./v2/pkg/templates) - Templates core starting point - [v2/pkg/templates/cache](./v2/pkg/templates/cache) - Templates cache -- [v2/pkg/protocols](./v2/pkg/protocol) - Protocol Specification +- [v2/pkg/protocols](./v2/pkg/protocols) - Protocol Specification - [v2/pkg/protocols/file](./v2/pkg/protocols/file) - File protocol - [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol - [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support diff --git a/README_CN.md b/README_CN.md index 0c4aa69c0..e79f28743 100644 --- a/README_CN.md +++ b/README_CN.md @@ -29,7 +29,7 @@ Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具， - [安装](#安装) - [Nuclei模板](#nuclei模板) - [用法](#用法) -- [运行Nuclei](#运行nuclei) +- [运行Nuclei](#运行Nuclei) - [排除模板](#排除模板) - [致谢](#致谢) From c8d009654ce0b5f4b1d22c3aa472e42f0e5b6147 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 16:57:43 +0200 Subject: [PATCH 10/27] fix: Receiver mismatch warnings --- v2/pkg/operators/matchers/compile.go | 49 ++++++++++---------- v2/pkg/operators/matchers/match.go | 50 ++++++++++----------- v2/pkg/operators/matchers/matchers.go | 10 ++--- v2/pkg/operators/matchers/matchers_types.go | 4 +- v2/pkg/templates/preprocessors.go | 2 +- v2/pkg/templates/templates.go | 18 ++++---- 6 files changed, 65 insertions(+), 68 deletions(-) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 79d062a7c..b571085c4 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -12,74 +12,73 @@ import ( ) // CompileMatchers performs the initial setup operation on a matcher -func (m *Matcher) CompileMatchers() error { +func (matcher *Matcher) CompileMatchers() error { var ok bool // Support hexadecimal encoding for matchers too. - if m.Encoding == "hex" { - for i, word := range m.Words { + if matcher.Encoding == "hex" { + for i, word := range matcher.Words { if decoded, err := hex.DecodeString(word); err == nil && len(decoded) > 0 { - m.Words[i] = string(decoded) + matcher.Words[i] = string(decoded) } } } // Set up the matcher type - computedType, err := toMatcherTypes(m.GetType().String()) + computedType, err := toMatcherTypes(matcher.GetType().String()) if err != nil { - return fmt.Errorf("unknown matcher type specified: %s", m.Type) + return fmt.Errorf("unknown matcher type specified: %s", matcher.Type) } - m.matcherType = computedType + matcher.matcherType = computedType // By default, match on body if user hasn't provided any specific items - if m.Part == "" { - m.Part = "body" + if matcher.Part == "" { + matcher.Part = "body" } - // Compile the regexes - for _, regex := range m.Regex { + for _, regex := range matcher.Regex { compiled, err := regexp.Compile(regex) if err != nil { return fmt.Errorf("could not compile regex: %s", regex) } - m.regexCompiled = append(m.regexCompiled, compiled) + matcher.regexCompiled = append(matcher.regexCompiled, compiled) } // Compile and validate binary Values in matcher - for _, value := range m.Binary { + for _, value := range matcher.Binary { if decoded, err := hex.DecodeString(value); err != nil { return fmt.Errorf("could not hex decode binary: %s", value) } else { - m.binaryDecoded = append(m.binaryDecoded, string(decoded)) + matcher.binaryDecoded = append(matcher.binaryDecoded, string(decoded)) } } // Compile the dsl expressions - for _, expr := range m.DSL { + for _, expr := range matcher.DSL { compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) if err != nil { return fmt.Errorf("could not compile dsl: %s", expr) } - m.dslCompiled = append(m.dslCompiled, compiled) + matcher.dslCompiled = append(matcher.dslCompiled, compiled) } // Set up the condition type, if any. - if m.Condition != "" { - m.condition, ok = ConditionTypes[m.Condition] + if matcher.Condition != "" { + matcher.condition, ok = ConditionTypes[matcher.Condition] if !ok { - return fmt.Errorf("unknown condition specified: %s", m.Condition) + return fmt.Errorf("unknown condition specified: %s", matcher.Condition) } } else { - m.condition = ORCondition + matcher.condition = ORCondition } - if m.CaseInsensitive { - if m.GetType() != WordsMatcher { - return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", m.Type) + if matcher.CaseInsensitive { + if matcher.GetType() != WordsMatcher { + return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", matcher.Type) } - for i := range m.Words { - m.Words[i] = strings.ToLower(m.Words[i]) + for i := range matcher.Words { + matcher.Words[i] = strings.ToLower(matcher.Words[i]) } } return nil diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 84601be6e..cda09a7a8 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -7,11 +7,11 @@ import ( ) // MatchStatusCode matches a status code check against a corpus -func (m *Matcher) MatchStatusCode(statusCode int) bool { +func (matcher *Matcher) MatchStatusCode(statusCode int) bool { // Iterate over all the status codes accepted as valid // // Status codes don't support AND conditions. - for _, status := range m.Status { + for _, status := range matcher.Status { // Continue if the status codes don't match if statusCode != status { continue @@ -23,11 +23,11 @@ func (m *Matcher) MatchStatusCode(statusCode int) bool { } // MatchSize matches a size check against a corpus -func (m *Matcher) MatchSize(length int) bool { +func (matcher *Matcher) MatchSize(length int) bool { // Iterate over all the sizes accepted as valid // // Sizes codes don't support AND conditions. - for _, size := range m.Size { + for _, size := range matcher.Size { // Continue if the size doesn't match if length != size { continue @@ -39,14 +39,14 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { - if m.CaseInsensitive { +func (matcher *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + if matcher.CaseInsensitive { corpus = strings.ToLower(corpus) } var matchedWords []string // Iterate over all the words accepted as valid - for i, word := range m.Words { + for i, word := range matcher.Words { if dynamicValues == nil { dynamicValues = make(map[string]interface{}) } @@ -60,7 +60,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} if !strings.Contains(corpus, word) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false, []string{} } // Continue with the flow since it's an OR Condition. @@ -68,14 +68,14 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} } // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true, []string{word} } matchedWords = append(matchedWords, word) // If we are at the end of the words, return with true - if len(m.Words)-1 == i { + if len(matcher.Words)-1 == i { return true, matchedWords } } @@ -83,15 +83,15 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} } // MatchRegex matches a regex check against a corpus -func (m *Matcher) MatchRegex(corpus string) (bool, []string) { +func (matcher *Matcher) MatchRegex(corpus string) (bool, []string) { var matchedRegexes []string // Iterate over all the regexes accepted as valid - for i, regex := range m.regexCompiled { + for i, regex := range matcher.regexCompiled { // Continue if the regex doesn't match if !regex.MatchString(corpus) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false, []string{} } // Continue with the flow since it's an OR Condition. @@ -100,14 +100,14 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { currentMatches := regex.FindAllString(corpus, -1) // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true, currentMatches } matchedRegexes = append(matchedRegexes, currentMatches...) // If we are at the end of the regex, return with true - if len(m.regexCompiled)-1 == i { + if len(matcher.regexCompiled)-1 == i { return true, matchedRegexes } } @@ -115,14 +115,14 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { } // MatchBinary matches a binary check against a corpus -func (m *Matcher) MatchBinary(corpus string) (bool, []string) { +func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { var matchedBinary []string // Iterate over all the words accepted as valid - for i, binary := range m.binaryDecoded { + for i, binary := range matcher.binaryDecoded { if !strings.Contains(corpus, binary) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false, []string{} } // Continue with the flow since it's an OR Condition. @@ -130,14 +130,14 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { } // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true, []string{binary} } matchedBinary = append(matchedBinary, binary) // If we are at the end of the words, return with true - if len(m.Binary)-1 == i { + if len(matcher.Binary)-1 == i { return true, matchedBinary } } @@ -145,9 +145,9 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { } // MatchDSL matches on a generic map result -func (m *Matcher) MatchDSL(data map[string]interface{}) bool { +func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid - for i, expression := range m.dslCompiled { + for i, expression := range matcher.dslCompiled { result, err := expression.Evaluate(data) if err != nil { continue @@ -160,7 +160,7 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool { if !ok || !bResult { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false } // Continue with the flow since it's an OR Condition. @@ -168,12 +168,12 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool { } // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true } // If we are at the end of the dsl, return with true - if len(m.dslCompiled)-1 == i { + if len(matcher.dslCompiled)-1 == i { return true } } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 3f37deb02..c0dbd186b 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -120,7 +120,6 @@ type Matcher struct { dslCompiled []*govaluate.EvaluableExpression } - // ConditionType is the type of condition for matcher type ConditionType int @@ -138,18 +137,17 @@ var ConditionTypes = map[string]ConditionType{ } // Result reverts the results of the match if the matcher is of type negative. -func (m *Matcher) Result(data bool) bool { - if m.Negative { +func (matcher *Matcher) Result(data bool) bool { + if matcher.Negative { return !data } return data } // ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string -func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { - if m.Negative { +func (matcher *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { + if matcher.Negative { return !data, []string{} } return data, matchedSnippet } - diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 336355374..143cfee01 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -39,8 +39,8 @@ var MatcherTypes = map[MatcherType]string{ } //GetType returns the type of the matcher -func (e *Matcher) GetType() MatcherType { - return e.Type.MatcherType +func (matcher *Matcher) GetType() MatcherType { + return matcher.Type.MatcherType } // GetSupportedMatcherTypes returns list of supported types diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index 6a1355e26..cc76ddc7d 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -15,7 +15,7 @@ type Preprocessor interface { var preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) // expandPreprocessors expands the pre-processors if any for a template data. -func (t *Template) expandPreprocessors(data []byte) []byte { +func (template *Template) expandPreprocessors(data []byte) []byte { foundMap := make(map[string]struct{}) for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index d34cd1ae1..a7f272732 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -96,23 +96,23 @@ var TemplateProtocols = []string{ } // Type returns the type of the template -func (t *Template) Type() types.ProtocolType { +func (template *Template) Type() types.ProtocolType { switch { - case len(t.RequestsDNS) > 0: + case len(template.RequestsDNS) > 0: return types.DNSProtocol - case len(t.RequestsFile) > 0: + case len(template.RequestsFile) > 0: return types.FileProtocol - case len(t.RequestsHTTP) > 0: + case len(template.RequestsHTTP) > 0: return types.HTTPProtocol - case len(t.RequestsHeadless) > 0: + case len(template.RequestsHeadless) > 0: return types.HeadlessProtocol - case len(t.RequestsNetwork) > 0: + case len(template.RequestsNetwork) > 0: return types.NetworkProtocol - case len(t.Workflow.Workflows) > 0: + case len(template.Workflow.Workflows) > 0: return types.WorkflowProtocol - case len(t.RequestsSSL) > 0: + case len(template.RequestsSSL) > 0: return types.SSLProtocol - case len(t.RequestsWebsocket) > 0: + case len(template.RequestsWebsocket) > 0: return types.WebsocketProtocol default: return types.InvalidProtocol From 3fd1f57b96336abc103e8532b5a8854d3fe3660e Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:03:56 +0200 Subject: [PATCH 11/27] refactor: godoc and comment uniformization Adding space after // and before the godoc/comment --- v2/cmd/integration-test/http.go | 2 +- v2/internal/runner/options.go | 3 ++- v2/internal/runner/proxy.go | 2 +- v2/pkg/operators/extractors/extractor_types.go | 1 - v2/pkg/operators/matchers/matchers_types.go | 1 - v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go | 5 +++-- v2/pkg/protocols/dns/dns_types.go | 1 - v2/pkg/protocols/http/http_method_types.go | 1 - v2/pkg/protocols/network/network_input_types.go | 1 - v2/pkg/testutils/testutils.go | 3 ++- 10 files changed, 9 insertions(+), 11 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 0b2bb247a..1d653afa2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -40,7 +40,7 @@ var httpTestcases = map[string]testutils.TestCase{ type httpInteractshRequest struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpInteractshRequest) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..317bd59bb 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/go-playground/validator/v10" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -93,7 +94,7 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - //loading the proxy server list from file or cli and test the connectivity + // loading the proxy server list from file or cli and test the connectivity if err := loadProxyServers(options); err != nil { return err } diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index 3d2e47fd8..6aca4be6a 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -117,7 +117,7 @@ func validateProxyURL(proxy string) (url.URL, error) { return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") } -//isSupportedProtocol checks given protocols are supported +// isSupportedProtocol checks given protocols are supported func isSupportedProtocol(value string) bool { return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 } diff --git a/v2/pkg/operators/extractors/extractor_types.go b/v2/pkg/operators/extractors/extractor_types.go index 227001a1f..330cbd094 100644 --- a/v2/pkg/operators/extractors/extractor_types.go +++ b/v2/pkg/operators/extractors/extractor_types.go @@ -20,7 +20,6 @@ const ( XPathExtractor // JSONExtractor extracts responses with json JSONExtractor - //limit limit ) diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 143cfee01..adeb0b130 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -24,7 +24,6 @@ const ( SizeMatcher // DSLMatcher matches based upon dsl syntax DSLMatcher - //limit limit ) diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 14aefff8f..0abf1b524 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/bluele/gcache" + "github.com/projectdiscovery/gologger" ) @@ -16,7 +17,7 @@ import ( // It uses an LRU cache internally for skipping unresponsive hosts // that remain so for a duration. type Cache struct { - MaxHostError int + MaxHostError int verbose bool failedTargets gcache.Cache } @@ -64,7 +65,7 @@ func (c *Cache) normalizeCacheValue(value string) string { } // ErrUnresponsiveHost is returned when a host is unresponsive -//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive") +// var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive") // Check returns true if a host should be skipped as it has been // unresponsive for a certain number of times. diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go index e1354fedf..a839862ad 100644 --- a/v2/pkg/protocols/dns/dns_types.go +++ b/v2/pkg/protocols/dns/dns_types.go @@ -21,7 +21,6 @@ const ( MX TXT AAAA - //limit limit ) diff --git a/v2/pkg/protocols/http/http_method_types.go b/v2/pkg/protocols/http/http_method_types.go index 987dde979..f68d9ac89 100644 --- a/v2/pkg/protocols/http/http_method_types.go +++ b/v2/pkg/protocols/http/http_method_types.go @@ -22,7 +22,6 @@ const ( HTTPTrace HTTPPatch HTTPPurge - //limit limit ) diff --git a/v2/pkg/protocols/network/network_input_types.go b/v2/pkg/protocols/network/network_input_types.go index f96c3e492..88b07e42e 100644 --- a/v2/pkg/protocols/network/network_input_types.go +++ b/v2/pkg/protocols/network/network_input_types.go @@ -14,7 +14,6 @@ type NetworkInputType int const ( hexType NetworkInputType = iota + 1 textType - //limit limit ) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 982fbabc3..51fec82fd 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -4,6 +4,7 @@ import ( "go.uber.org/ratelimit" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -131,7 +132,7 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro } } -// Write writes the event to file and/or screen. +// WriteFailure writes the event to file and/or screen. func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error { return nil } From 47340f06b09df6db14f8ab8d71a420df6e00655a Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:09:20 +0200 Subject: [PATCH 12/27] refactor: uniformly sorted imports --- DESIGN.md | 3 ++- v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/loader.go | 1 + v2/cmd/integration-test/websocket.go | 1 + v2/pkg/catalog/find.go | 1 + v2/pkg/core/execute.go | 5 +++-- v2/pkg/core/inputs/hybrid/hmap.go | 1 + v2/pkg/core/workflow_execute.go | 5 +++-- v2/pkg/core/workpool.go | 3 ++- v2/pkg/operators/common/dsl/dsl.go | 3 ++- v2/pkg/operators/common/dsl/dsl_test.go | 3 ++- v2/pkg/operators/extractors/extract.go | 3 +-- v2/pkg/parsers/parser_test.go | 3 ++- v2/pkg/protocols/common/generators/generators.go | 1 + v2/pkg/protocols/common/generators/generators_test.go | 3 ++- v2/pkg/protocols/common/protocolinit/init.go | 1 + v2/pkg/protocols/common/protocolstate/state.go | 1 + v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/headless/engine/http_client.go | 3 ++- v2/pkg/protocols/http/build_request_test.go | 3 ++- v2/pkg/protocols/http/request_generator_test.go | 3 ++- v2/pkg/protocols/http/utils.go | 5 +++-- v2/pkg/protocols/protocols.go | 1 + v2/pkg/protocols/ssl/ssl.go | 1 + v2/pkg/protocols/ssl/ssl_test.go | 3 ++- v2/pkg/protocols/websocket/websocket.go | 1 + v2/pkg/reporting/dedupe/dedupe_test.go | 3 ++- v2/pkg/reporting/exporters/es/elasticsearch.go | 6 +++--- v2/pkg/reporting/reporting.go | 2 +- v2/pkg/reporting/trackers/gitlab/gitlab.go | 3 ++- v2/pkg/templates/cluster.go | 3 ++- v2/pkg/templates/types/types.go | 1 + v2/pkg/templates/workflows.go | 1 + 33 files changed, 54 insertions(+), 26 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8ff4dcb92..590b226be 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -275,6 +275,8 @@ import ( "path" "github.com/logrusorgru/aurora" + "go.uber.org/ratelimit" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" @@ -291,7 +293,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "go.uber.org/ratelimit" ) func main() { diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index a44aa4a60..d55f74e0f 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index e07c45c21..0a4446042 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index b5b0e5a34..d92058916 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/gobwas/ws/wsutil" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 92e6d47f9..cc38519b9 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -7,6 +7,7 @@ import ( "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index f78400101..873a94e85 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -1,11 +1,12 @@ package core import ( + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" ) // Execute takes a list of templates/workflows that have been compiled diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 406331fe0..901bc6b7f 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/filekv" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 8c255d7ad..9ee87df4a 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,11 +1,12 @@ package core import ( + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" ) // executeWorkflow runs a workflow on an input and returns true or false diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index 77909a89f..cd2ea09ba 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -1,8 +1,9 @@ package core import ( - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/remeh/sizedwaitgroup" + + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // WorkPool implements an execution pool for executing different diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index e4cee9c5e..9e1a9f2ad 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -19,10 +19,11 @@ import ( "time" "github.com/Knetic/govaluate" + "github.com/spaolacci/murmur3" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/spaolacci/murmur3" ) const ( diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index f75baf087..e5bfb0d64 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -8,8 +8,9 @@ import ( "time" "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestDSLURLEncodeDecode(t *testing.T) { diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index e43ea145e..e06428ac2 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -1,9 +1,8 @@ package extractors import ( - "strings" - "encoding/json" + "strings" "github.com/antchfx/htmlquery" diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index ef74a317e..32a69ef5b 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -5,11 +5,12 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/stretchr/testify/require" ) func TestLoadTemplate(t *testing.T) { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index f3aaa8a34..07b3a3d0a 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -4,6 +4,7 @@ package generators import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 75e39d0ed..71a033e6c 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -3,8 +3,9 @@ package generators import ( "testing" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) func TestBatteringRamGenerator(t *testing.T) { diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index 1877a1ab3..307c33a2f 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -2,6 +2,7 @@ package protocolinit import ( "github.com/corpix/uarand" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index b43e40cf9..28c9df525 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -2,6 +2,7 @@ package protocolstate import ( "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 614159e2a..a1a7b5c37 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,8 +12,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" - "github.com/projectdiscovery/retryabledns" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/retryabledns" ) var _ protocols.Request = &Request{} diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 1f40f98f7..e61803d4b 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -12,9 +12,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "golang.org/x/net/proxy" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "golang.org/x/net/proxy" ) // newhttpClient creates a new http client for headless communication with a timeout diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index a77e3c947..4719a5d24 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/stretchr/testify/require" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 3fee8024c..ee8c524c0 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -3,9 +3,10 @@ package http import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/stretchr/testify/require" ) func TestRequestGeneratorPaths(t *testing.T) { diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 5eae57efe..834cc091d 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -11,11 +11,12 @@ import ( "strings" "github.com/pkg/errors" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" ) type redirectedResponse struct { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 7ad9d24aa..f288b717f 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -4,6 +4,7 @@ import ( "go.uber.org/ratelimit" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 388886be2..b1db17040 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -10,6 +10,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" + "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index 8d7e02179..8764f0ce3 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -3,11 +3,12 @@ package ssl import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/stretchr/testify/require" ) func TestSSLProtocol(t *testing.T) { diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index ab864f65b..53a35df3c 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -15,6 +15,7 @@ import ( "github.com/gobwas/ws" "github.com/gobwas/ws/wsutil" "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" diff --git a/v2/pkg/reporting/dedupe/dedupe_test.go b/v2/pkg/reporting/dedupe/dedupe_test.go index 5d77f35cd..eb5014425 100644 --- a/v2/pkg/reporting/dedupe/dedupe_test.go +++ b/v2/pkg/reporting/dedupe/dedupe_test.go @@ -5,8 +5,9 @@ import ( "os" "testing" - "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" ) func TestDedupeDuplicates(t *testing.T) { diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 0959cc929..6e009f42b 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -3,15 +3,15 @@ package es import ( "bytes" "crypto/tls" + "encoding/base64" + "encoding/json" "fmt" "io/ioutil" "net/http" "time" - "encoding/base64" - "encoding/json" - "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 92c162f8f..d8f000584 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -10,8 +10,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index c1438e2dc..dddfa2e65 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -3,9 +3,10 @@ package gitlab import ( "fmt" + "github.com/xanzy/go-gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" - "github.com/xanzy/go-gitlab" ) // Integration is a client for an issue tracker integration diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 1ec1e95cb..597711e17 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -3,6 +3,8 @@ package templates import ( "fmt" + "github.com/rs/xid" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -10,7 +12,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/rs/xid" ) // Cluster clusters a list of templates into a lesser number if possible based diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index d26ad7f08..0b50eb83d 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -7,6 +7,7 @@ import ( "github.com/alecthomas/jsonschema" "github.com/pkg/errors" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" ) diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 89f8d1bda..9de1f2686 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -2,6 +2,7 @@ package templates import ( "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" From fdaa8e4539b38c8980432d8c8ec3bbcebdce71ae Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:18:54 +0200 Subject: [PATCH 13/27] refactor: removed redundant type conversion --- v2/cmd/integration-test/http.go | 4 +- v2/cmd/integration-test/loader.go | 16 +-- v2/cmd/integration-test/websocket.go | 2 +- .../headless/engine/page_actions_test.go | 118 +++++++++--------- .../offlinehttp/read_response_test.go | 4 +- 5 files changed, 72 insertions(+), 72 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 1d653afa2..1890048d2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -43,14 +43,14 @@ type httpInteractshRequest struct{} // Execute executes a test case and returns an error if occurred func (h *httpInteractshRequest) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { value := r.Header.Get("url") if value != "" { if resp, _ := http.DefaultClient.Get(value); resp != nil { resp.Body.Close() } } - })) + }) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 0a4446042..3507b8a31 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -25,14 +25,14 @@ type remoteTemplateList struct{} func (h *remoteTemplateList) Execute(templateList string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is test matcher text") if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } - })) + }) - router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { file, err := os.ReadFile(templateList) if err != nil { w.WriteHeader(500) @@ -41,7 +41,7 @@ func (h *remoteTemplateList) Execute(templateList string) error { if err != nil { w.WriteHeader(500) } - })) + }) ts := httptest.NewServer(router) defer ts.Close() @@ -61,14 +61,14 @@ type remoteWorkflowList struct{} func (h *remoteWorkflowList) Execute(workflowList string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is test matcher text") if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } - })) + }) - router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/workflow_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { file, err := os.ReadFile(workflowList) if err != nil { w.WriteHeader(500) @@ -77,7 +77,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error { if err != nil { w.WriteHeader(500) } - })) + }) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index d92058916..af6d07451 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -23,7 +23,7 @@ func (h *websocketBasic) Execute(filePath string) error { connHandler := func(conn net.Conn) { for { msg, op, _ := wsutil.ReadClientData(conn) - if string(msg) != string("hello") { + if string(msg) != "hello" { return } _ = wsutil.WriteServerMessage(conn, op, []byte("world")) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index f9819756d..4c3414e47 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -28,7 +28,7 @@ func TestActionNavigate(t *testing.T) { ` - actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}} + actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}} testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -50,9 +50,9 @@ func TestActionScript(t *testing.T) { t.Run("run-and-results", func(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { @@ -64,10 +64,10 @@ func TestActionScript(t *testing.T) { t.Run("hook", func(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -88,9 +88,9 @@ func TestActionClick(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -121,9 +121,9 @@ func TestActionRightClick(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionRightClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionRightClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -146,9 +146,9 @@ func TestActionTextInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTextInput)}, Data: map[string]string{"selector": "input", "value": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionTextInput}, Data: map[string]string{"selector": "input", "value": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -163,9 +163,9 @@ func TestActionTextInput(t *testing.T) { func TestActionHeadersChange(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionSetHeader}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -190,9 +190,9 @@ func TestActionScreenshot(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScreenshot)}, Data: map[string]string{"to": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -215,9 +215,9 @@ func TestActionTimeInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTimeInput)}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionTimeInput}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -243,9 +243,9 @@ func TestActionSelectInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSelectInput)}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionSelectInput}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -266,9 +266,9 @@ func TestActionFilesInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionFilesInput)}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -292,8 +292,8 @@ func TestActionWaitLoad(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -317,8 +317,8 @@ func TestActionGetResource(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionGetResource)}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionGetResource}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -337,8 +337,8 @@ func TestActionExtract(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionExtract)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionExtract}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -356,8 +356,8 @@ func TestActionSetMethod(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetMethod)}, Data: map[string]string{"part": "x", "method": "SET"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionSetMethod}, Data: map[string]string{"part": "x", "method": "SET"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -368,9 +368,9 @@ func TestActionSetMethod(t *testing.T) { func TestActionAddHeader(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionAddHeader}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -387,11 +387,11 @@ func TestActionAddHeader(t *testing.T) { func TestActionDeleteHeader(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionDeleteHeader)}, Data: map[string]string{"part": "request", "key": "Test2"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionAddHeader}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionAddHeader}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: ActionTypeHolder{ActionType: ActionDeleteHeader}, Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -408,9 +408,9 @@ func TestActionDeleteHeader(t *testing.T) { func TestActionSetBody(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetBody)}, Data: map[string]string{"part": "request", "body": "hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionSetBody}, Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -436,10 +436,10 @@ func TestActionKeyboard(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "input"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionKeyboard)}, Data: map[string]string{"keys": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "input"}}, + {ActionType: ActionTypeHolder{ActionType: ActionKeyboard}, Data: map[string]string{"keys": "Test2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -462,8 +462,8 @@ func TestActionSleep(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSleep)}, Data: map[string]string{"duration": "2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionSleep}, Data: map[string]string{"duration": "2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -485,8 +485,8 @@ func TestActionWaitVisible(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitVisible)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitVisible}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, } t.Run("wait for an element being visible", func(t *testing.T) { diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index f47b345dc..a26f2371a 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -159,7 +159,7 @@ Server: Google Frontend t.Run("test-live-response-with-content-length", func(t *testing.T) { var ts *httptest.Server router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Server", "Google Frontend") fmt.Fprintf(w, "%s", ` @@ -172,7 +172,7 @@ Server: Google Frontend

`) - })) + }) ts = httptest.NewServer(router) defer ts.Close() From ec6889931d50fca916c97a44fad3e251d9baa439 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:57:22 +0200 Subject: [PATCH 14/27] refactor: linter driven fixes * x = x + "" => x += "" * pre-allocating slice with known size * added t.Helper() methods in test helpers * complex if-else conditions replaced by switches * errors should be checked using error.Is() instead of == * function parameter should start with lower case letter * removed unnecessary type definition * variable/label naming convention: camelCase instead of snake_case --- v2/cmd/cve-annotate/main.go | 10 ++++----- v2/internal/runner/update.go | 4 ++-- v2/pkg/model/model_test.go | 1 + v2/pkg/model/types/severity/severities.go | 2 +- .../model/types/severity/severity_holder.go | 1 - v2/pkg/model/types/severity/severity_test.go | 2 ++ v2/pkg/model/types/stringslice/stringslice.go | 7 ++++--- v2/pkg/protocols/common/generators/load.go | 2 +- .../common/hosterrorscache/hosterrorscache.go | 5 ++--- v2/pkg/protocols/headless/engine/rules.go | 21 +++++++++++-------- .../http/httpclientpool/clientpool.go | 2 +- v2/pkg/protocols/network/request.go | 4 ++-- v2/pkg/utils/yaml/yaml_decode_wrapper.go | 2 +- 13 files changed, 34 insertions(+), 29 deletions(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index 45ede0420..c2ccaa765 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -117,21 +117,21 @@ func getCVEData(client *nvd.Client, filePath, data string) { } if !strings.Contains(infoBlockClean, "classification") && (cvssScore != 0 && cvssMetrics != "") { changed = true - newInfoBlock = newInfoBlock + fmt.Sprintf("\n classification:\n cvss-metrics: %s\n cvss-score: %.2f\n cve-id: %s", cvssMetrics, cvssScore, cveName) + newInfoBlock += fmt.Sprintf("\n classification:\n cvss-metrics: %s\n cvss-score: %.2f\n cve-id: %s", cvssMetrics, cvssScore, cveName) if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") { - newInfoBlock = newInfoBlock + fmt.Sprintf("\n cwe-id: %s", strings.Join(cweID, ",")) + newInfoBlock += fmt.Sprintf("\n cwe-id: %s", strings.Join(cweID, ",")) } } // If there is no description field, fill the description from CVE information if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 { changed = true - newInfoBlock = newInfoBlock + fmt.Sprintf("\n description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value)) + newInfoBlock += fmt.Sprintf("\n description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value)) } if !strings.Contains(infoBlockClean, "reference:") && len(cveItem.CVE.References.ReferenceData) > 0 { changed = true - newInfoBlock = newInfoBlock + "\n reference:" + newInfoBlock += "\n reference:" for _, reference := range cveItem.CVE.References.ReferenceData { - newInfoBlock = newInfoBlock + fmt.Sprintf("\n - %s", reference.URL) + newInfoBlock += fmt.Sprintf("\n - %s", reference.URL) } } newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 8a0fa6c5a..99d94cc88 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -333,7 +333,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat isAddition = true } - newTemplateChecksum, err := writeUnZippedTemplateFile(err, templateAbsolutePath, zipTemplateFile) + newTemplateChecksum, err := writeUnZippedTemplateFile(templateAbsolutePath, zipTemplateFile) if err != nil { return nil, err } @@ -366,7 +366,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat return results, nil } -func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { +func writeUnZippedTemplateFile(templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return "", fmt.Errorf("could not create template file: %w", err) diff --git a/v2/pkg/model/model_test.go b/v2/pkg/model/model_test.go index bdbe27ddb..e0803c31a 100644 --- a/v2/pkg/model/model_test.go +++ b/v2/pkg/model/model_test.go @@ -72,6 +72,7 @@ func TestUnmarshal(t *testing.T) { } assertUnmarshalledTemplateInfo := func(t *testing.T, yamlPayload string) Info { + t.Helper() info := Info{} err := yaml.Unmarshal([]byte(yamlPayload), &info) assert.Nil(t, err) diff --git a/v2/pkg/model/types/severity/severities.go b/v2/pkg/model/types/severity/severities.go index 9e3244859..91ffa1ceb 100644 --- a/v2/pkg/model/types/severity/severities.go +++ b/v2/pkg/model/types/severity/severities.go @@ -43,7 +43,7 @@ func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) e } func (severities Severities) String() string { - var stringSeverities []string + var stringSeverities = make([]string, 0, len(severities)) for _, severity := range severities { stringSeverities = append(stringSeverities, severity.String()) } diff --git a/v2/pkg/model/types/severity/severity_holder.go b/v2/pkg/model/types/severity/severity_holder.go index ad4c2496d..7d52f9c81 100644 --- a/v2/pkg/model/types/severity/severity_holder.go +++ b/v2/pkg/model/types/severity/severity_holder.go @@ -6,7 +6,6 @@ import ( "github.com/alecthomas/jsonschema" ) -//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe // Holder holds a Severity type. Required for un/marshalling purposes type Holder struct { Severity Severity diff --git a/v2/pkg/model/types/severity/severity_test.go b/v2/pkg/model/types/severity/severity_test.go index b21f57265..6ba472388 100644 --- a/v2/pkg/model/types/severity/severity_test.go +++ b/v2/pkg/model/types/severity/severity_test.go @@ -30,6 +30,7 @@ func TestGetSupportedSeverities(t *testing.T) { } func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) { + t.Helper() payloads := [...]string{ payloadCreator("Info"), payloadCreator("info"), @@ -48,6 +49,7 @@ func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) e } func testUnmarshalFail(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) { + t.Helper() assert.Panics(t, func() { unmarshal(payloadCreator("invalid"), unmarshaller) }) } diff --git a/v2/pkg/model/types/stringslice/stringslice.go b/v2/pkg/model/types/stringslice/stringslice.go index 4a3e28486..55d798550 100644 --- a/v2/pkg/model/types/stringslice/stringslice.go +++ b/v2/pkg/model/types/stringslice/stringslice.go @@ -80,11 +80,12 @@ func marshalStringToSlice(unmarshal func(interface{}) error) ([]string, error) { } var result []string - if len(marshalledValuesAsSlice) > 0 { + switch { + case len(marshalledValuesAsSlice) > 0: result = marshalledValuesAsSlice - } else if utils.IsNotBlank(marshalledValueAsString) { + case utils.IsNotBlank(marshalledValueAsString): result = strings.Split(marshalledValueAsString, ",") - } else { + default: result = []string{} } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 0c44b613c..d5b00e4d3 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -53,7 +53,7 @@ func loadPayloadsFromFile(filepath string) ([]string, error) { } lines = append(lines, text) } - if err := scanner.Err(); err != nil && err != io.EOF { + if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { return lines, scanner.Err() } return lines, nil diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 0abf1b524..c509ff847 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -25,11 +25,11 @@ type Cache struct { const DefaultMaxHostsCount = 10000 // New returns a new host max errors cache -func New(MaxHostError, maxHostsCount int) *Cache { +func New(maxHostError, maxHostsCount int) *Cache { gc := gcache.New(maxHostsCount). ARC(). Build() - return &Cache{failedTargets: gc, MaxHostError: MaxHostError} + return &Cache{failedTargets: gc, MaxHostError: maxHostError} } // SetVerbose sets the cache to log at verbose level @@ -47,7 +47,6 @@ func (c *Cache) normalizeCacheValue(value string) string { finalValue := value if strings.HasPrefix(value, "http") { if parsed, err := url.Parse(value); err == nil { - hostname := parsed.Host finalPort := parsed.Port() if finalPort == "" { diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index a802a64a9..0c5359415 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -16,15 +16,16 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { continue } - if rule.Action == ActionSetMethod { + switch { + case rule.Action == ActionSetMethod: ctx.Request.Req().Method = rule.Args["method"] - } else if rule.Action == ActionAddHeader { + case rule.Action == ActionAddHeader: ctx.Request.Req().Header.Add(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionSetHeader { + case rule.Action == ActionSetHeader: ctx.Request.Req().Header.Set(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionDeleteHeader { + case rule.Action == ActionDeleteHeader: ctx.Request.Req().Header.Del(rule.Args["key"]) - } else if rule.Action == ActionSetBody { + case rule.Action == ActionSetBody: body := rule.Args["body"] ctx.Request.Req().ContentLength = int64(len(body)) ctx.Request.SetBody(body) @@ -36,13 +37,15 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { if rule.Part != "response" { continue } - if rule.Action == ActionAddHeader { + + switch { + case rule.Action == ActionAddHeader: ctx.Response.Headers().Add(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionSetHeader { + case rule.Action == ActionSetHeader: ctx.Response.Headers().Set(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionDeleteHeader { + case rule.Action == ActionDeleteHeader: ctx.Response.Headers().Del(rule.Args["key"]) - } else if rule.Action == ActionSetBody { + case rule.Action == ActionSetBody: body := rule.Args["body"] ctx.Response.Headers().Set("Content-Length", fmt.Sprintf("%d", len(body))) ctx.Response.SetBody(rule.Args["body"]) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 191ca3cf0..9305870a6 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -185,7 +185,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if proxyURL != nil { // Attempts to overwrite the dial function with the socks proxied version if proxyURL.Scheme == types.SOCKS5 { - var proxyAuth *proxy.Auth = &proxy.Auth{} + var proxyAuth = &proxy.Auth{} proxyAuth.User = proxyURL.User.Username() proxyAuth.Password, _ = proxyURL.User.Password() diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 35da81d15..e4c0ac410 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -223,12 +223,12 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input <-t.C } } - read_socket: + readSocket: for { select { case <-readInterval.C: closeTimer(readInterval) - break read_socket + break readSocket default: buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) diff --git a/v2/pkg/utils/yaml/yaml_decode_wrapper.go b/v2/pkg/utils/yaml/yaml_decode_wrapper.go index a9cb422bb..3bc4fa605 100644 --- a/v2/pkg/utils/yaml/yaml_decode_wrapper.go +++ b/v2/pkg/utils/yaml/yaml_decode_wrapper.go @@ -19,8 +19,8 @@ func DecodeAndValidate(r io.Reader, v interface{}) error { if validate == nil { validate = validator.New() } - if err := validate.Struct(v); err != nil { + if err := validate.Struct(v); err != nil { if _, ok := err.(*validator.InvalidValidationError); ok { return err } From 7e22d70dedcab5e032426c14d1a89256eac2c1d6 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 18:54:16 +0200 Subject: [PATCH 15/27] refactor/documentation: typos and grammatical errors --- .github/ISSUE_TEMPLATE/issue-report.md | 2 +- DESIGN.md | 36 +++++++++--------- README.md | 4 +- .../test-issue-tracker-config1.yaml | 30 +++++++-------- .../test-issue-tracker-config2.yaml | 38 +++++++++---------- v2/cmd/nuclei/issue-tracker-config.yaml | 36 +++++++++--------- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 2 +- v2/pkg/catalog/find.go | 2 +- v2/pkg/catalog/loader/filter/tag_filter.go | 6 +-- v2/pkg/catalog/loader/loader.go | 6 +-- v2/pkg/core/execute.go | 8 ++-- v2/pkg/core/workpool.go | 14 +++---- v2/pkg/operators/matchers/matchers.go | 2 +- v2/pkg/projectfile/httputil.go | 28 -------------- v2/pkg/projectfile/project.go | 16 ++++---- .../common/generators/attack_types.go | 8 ++-- .../protocols/common/generators/generators.go | 6 +-- .../common/generators/generators_test.go | 2 +- .../protocols/common/generators/validate.go | 16 ++++---- .../hosterrorscache/hosterrorscache_test.go | 6 +-- .../protocols/dns/dnsclientpool/clientpool.go | 2 +- v2/pkg/protocols/dns/operators.go | 10 ++--- v2/pkg/protocols/dns/request.go | 16 ++++---- v2/pkg/protocols/headless/engine/engine.go | 10 ++--- .../protocols/headless/engine/http_client.go | 4 +- .../protocols/headless/engine/page_actions.go | 12 +++--- v2/pkg/protocols/headless/engine/rules.go | 2 +- v2/pkg/protocols/http/build_request_test.go | 6 +-- v2/pkg/protocols/http/http.go | 6 +-- v2/pkg/protocols/http/http_test.go | 2 +- .../http/httpclientpool/clientpool.go | 22 +++++------ .../protocols/http/race/syncedreadcloser.go | 12 +++--- v2/pkg/protocols/http/raw/raw.go | 6 +-- v2/pkg/protocols/http/request.go | 6 +-- .../protocols/http/request_generator_test.go | 4 +- v2/pkg/protocols/http/utils.go | 8 ++-- v2/pkg/protocols/network/network.go | 6 +-- v2/pkg/protocols/websocket/websocket.go | 2 +- .../reporting/exporters/es/elasticsearch.go | 2 +- v2/pkg/reporting/reporting.go | 16 ++++---- v2/pkg/reporting/trackers/github/github.go | 8 ++-- v2/pkg/templates/templates_doc.go | 2 +- v2/pkg/testutils/testutils.go | 2 +- v2/pkg/types/types.go | 6 +-- v2/pkg/utils/stats/stats.go | 4 +- 46 files changed, 209 insertions(+), 237 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md index aeec33c17..264b09ccc 100644 --- a/.github/ISSUE_TEMPLATE/issue-report.md +++ b/.github/ISSUE_TEMPLATE/issue-report.md @@ -33,4 +33,4 @@ Example: steps to reproduce the behavior: ### Anything else: - + diff --git a/DESIGN.md b/DESIGN.md index 590b226be..31f3a2b99 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -10,9 +10,9 @@ Template is the basic unit of input to the engine which describes the requests t The template structure is described here. Template level attributes are defined here as well as convenience methods to validate, parse and compile templates creating executers. -Any attributes etc required for the template, engine or requests to function are also set here. +Any attributes etc. required for the template, engine or requests to function are also set here. -Workflows are also compiled, their templates are loaded and compiled as well. Any validations etc on the paths provided are also done here. +Workflows are also compiled, their templates are loaded and compiled as well. Any validations etc. on the paths provided are also done here. `Parse` function is the main entry point which returns a template for a `filePath` and `executorOptions`. It compiles all the requests for the templates, all the workflows, as well as any self-contained request etc. It also caches the templates in an in-memory cache. @@ -106,7 +106,7 @@ The default executer is provided in `pkg/protocols/common/executer` . It takes a A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/templates` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. -For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently from the template execution. +For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently of the template execution. With this basic premise set, we can now start exploring the current runner implementation which will also walk us through the architecture of nuclei. @@ -118,7 +118,7 @@ The first process after all CLI specific initialisation is the loading of templa #### pkg/catalog -This package is used to get paths using mixed syntax. It takes a template directory and performs resolving for template paths both from provided template directory as well as the current user directory. +This package is used to get paths using mixed syntax. It takes a template directory and performs resolving for template paths both from provided template and current user directory. The syntax is very versatile and can include filenames, glob patterns, directories, absolute paths, and relative-paths. @@ -177,7 +177,7 @@ ResultEvent structure is passed to the Nuclei Output Writer which contains the e #### pkg/protocols/common/interactsh -Interactsh module is used to provide automatic Out of Band vulnerability identification in Nuclei. +Interactsh module is used to provide automatic Out-of-Band vulnerability identification in Nuclei. It uses two LRU caches, one for storing interactions for request URLs and one for storing requests for interaction URL. These both caches are used to correlated requests received to the Interactsh OOB server and Nuclei Instance. [Interactsh Client](https://github.com/projectdiscovery/interactsh/pkg/client) package does most of the heavy lifting of this module. @@ -193,13 +193,13 @@ Next we arrive in the `RunEnumeration` function of the runner. Next the `WorkflowLoader` is initialised which used to load workflows. It exists in `v2/pkg/parsers/workflow_loader.go` -The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc to return compiled `Templates` and `Workflows`. +The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc. to return compiled `Templates` and `Workflows`. #### pkg/catalog/loader -First the input passed by the user as paths is normalised to absolute paths which is done by the `pkg/catalog` module. Next the path filter module is used to removed the excluded template/workflows paths. +First the input passed by the user as paths is normalised to absolute paths which is done by the `pkg/catalog` module. Next the path filter module is used to remove the excluded template/workflows paths. -`pkg/parsers` module's `LoadTemplate`,`LoadWorkflow` functions are used to check if the templates pass the validation + are not excluded via tags/severity/etc filters. If all checks are passed, then the template/workflow is parsed and returned in a compiled form by the `pkg/templates`'s `Parse` function. +`pkg/parsers` module's `LoadTemplate`,`LoadWorkflow` functions are used to check if the templates pass the validation + are not excluded via tags/severity/etc. filters. If all checks are passed, then the template/workflow is parsed and returned in a compiled form by the `pkg/templates`'s `Parse` function. `Parse` function performs compilation of all the requests in a template + creates Executers from them returning a runnable Template/Workflow structure. @@ -207,10 +207,10 @@ Clustering module comes in next whose job is to cluster identical HTTP GET reque ### pkg/operators -Operators package implements all of the matching and extracting logic of Nuclei. +Operators package implements all the matching and extracting logic of Nuclei. ```go -// Operators contains the operators that can be applied on protocols +// Operators contain the operators that can be applied on protocols type Operators struct { Matchers []*matchers.Matcher Extractors []*extractors.Extractor @@ -218,7 +218,7 @@ type Operators struct { } ``` -A protocol only needs to embed the `operators.Operators` type shown above and it can utilise all the matching/extracting functionality of nuclei. +A protocol only needs to embed the `operators.Operators` type shown above, and it can utilise all the matching/extracting functionality of nuclei. ```go // MatchFunc performs matching operation for a matcher on model and returns true or false. @@ -246,7 +246,7 @@ type Result struct { } ``` -The internal logics for matching and extracting for things like words, regexes, jq, paths, etc is specified in `pkg/operators/matchers`, `pkg/operators/extractors`. Those packages should be investigated for further look into the topic. +The internal logics for matching and extracting for things like words, regexes, jq, paths, etc. is specified in `pkg/operators/matchers`, `pkg/operators/extractors`. Those packages should be investigated for further look into the topic. ### Template Execution @@ -357,7 +357,7 @@ func main() { ### Adding a New Protocol -Protocols form the core of Nuclei Engine. All the request types like `http`, `dns`, etc are implemented in form of protocol requests. +Protocols form the core of Nuclei Engine. All the request types like `http`, `dns`, etc. are implemented in form of protocol requests. A protocol must implement the `Protocol` and `Request` interfaces described above in `pkg/protocols`. We'll take the example of an existing protocol implementation - websocket for this short reference around Nuclei internals. @@ -475,13 +475,13 @@ func (r *Request) Type() templateTypes.ProtocolType { } ``` -Almost all of these protocols have boilerplate functions for which default implementations have been provided in the `providers` package. Examples are the implementation of `Match`, `Extract`, `MakeResultEvent`, GetCompiledOperators`, etc which are almost same throughout Nuclei protocols code. It is enough to copy-paste them unless customization is required. +Almost all of these protocols have boilerplate functions for which default implementations have been provided in the `providers` package. Examples are the implementation of `Match`, `Extract`, `MakeResultEvent`, GetCompiledOperators`, etc. which are almost same throughout Nuclei protocols code. It is enough to copy-paste them unless customization is required. `eventcreator` package offers `CreateEventWithAdditionalOptions` function which can be used to create result events after doing request execution. Step by step description of how to add a new protocol to Nuclei - -1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with less number of options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`. +1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with fewer options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`. 2. Add the protocol request structure to the `Template` structure fields. This is done in `pkg/templates/templates.go` with the corresponding import line. @@ -527,7 +527,7 @@ func (t *Template) Type() templateTypes.ProtocolType { ```go -// Requests returns the total request count for the template +// Requests return the total request count for the template func (template *Template) Requests() int { return len(template.RequestsDNS) + ... @@ -559,7 +559,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results - [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter - [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter -- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - Github Issue Tracker Exporter +- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - GitHub Issue Tracker Exporter - [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions - [v2/pkg/parsers](./v2/pkg/parsers) - Implements template as well as workflow loader for initial template discovery, validation and - loading. - [v2/pkg/types](./v2/pkg/types) - Contains CLI options as well as misc helper functions. @@ -583,7 +583,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol - [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support - [v2/pkg/protocols/common/interactsh](./v2/pkg/protocols/common/interactsh) - Interactsh integration -- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc) +- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc.) - [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer - [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers - [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator diff --git a/README.md b/README.md index efe80f381..427b11e97 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest ### Nuclei Templates -Nuclei has had built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated. +Nuclei has built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated. You may still use the `update-templates` flag to update the nuclei templates at any time; You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/). @@ -250,7 +250,7 @@ Please check our other open-source projects that might fit into your bug bounty Nuclei immensely improve how you approach security assessment by augmenting the manual, repetitive processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. -Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix. +Pen-testers get the full power of our public templates and customization capabilities to speed up their assessment process, and specifically with the regression cycle where you can easily verify the fix. - Easily create your compliance, standards suite (e.g. OWASP Top 10) checklist. - With capabilities like [fuzz](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing) and [workflows](https://nuclei.projectdiscovery.io/templating-guide/#workflows), complex manual steps and repetitive assessment can be easily automated with Nuclei. diff --git a/integration_tests/test-issue-tracker-config1.yaml b/integration_tests/test-issue-tracker-config1.yaml index dd8ceb180..2f8b587f7 100644 --- a/integration_tests/test-issue-tracker-config1.yaml +++ b/integration_tests/test-issue-tracker-config1.yaml @@ -3,35 +3,35 @@ allow-list: deny-list: severity: low -# github contains configuration options for github issue tracker +# GitHub contains configuration options for GitHub issue tracker github: - # base-url is the optional self-hosted github application url + # base-url is the optional self-hosted GitHub application url base-url: https://localhost:8443/github - # username is the username of the github user + # username is the username of the GitHub user username: test-username - # owner is the owner name of the repository for issues. + # owner is the owner name of the repository for issues owner: test-owner - # token is the token for github account. + # token is the token for GitHub account token: test-token - # project-name is the name of the repository. + # project-name is the name of the repository project-name: test-project # issue-label is the label of the created issue type issue-label: bug -# gitlab contains configuration options for gitlab issue tracker +# GitLab contains configuration options for gitlab issue tracker gitlab: - # base-url is the optional self-hosted gitlab application url + # base-url is the optional self-hosted GitLab application url base-url: https://localhost:8443/gitlab - # username is the username of the gitlab user + # username is the username of the GitLab user username: test-username - # token is the token for gitlab account. + # token is the token for GitLab account token: test-token - # project-name is the name/id of the project(repository). + # project-name is the name/id of the project(repository) project-name: "1234" # issue-label is the label of the created issue type issue-label: bug -# jira contains configuration options for jira issue tracker +# Jira contains configuration options for Jira issue tracker jira: # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used cloud: true @@ -39,11 +39,11 @@ jira: update-existing: false # URL is the jira application url url: https://localhost/jira - # account-id is the account-id of the jira user or username in case of on-prem Jira + # account-id is the account-id of the Jira user or username in case of on-prem Jira account-id: test-account-id - # email is the email of the user for jira instance + # email is the email of the user for Jira instance email: test@test.com - # token is the token for jira instance or password in case of on-prem Jira + # token is the token for Jira instance or password in case of on-prem Jira token: test-token # project-name is the name of the project. project-name: test-project-name diff --git a/integration_tests/test-issue-tracker-config2.yaml b/integration_tests/test-issue-tracker-config2.yaml index c76b773eb..f548dbfbd 100644 --- a/integration_tests/test-issue-tracker-config2.yaml +++ b/integration_tests/test-issue-tracker-config2.yaml @@ -5,47 +5,47 @@ allow-list: deny-list: severity: low -# github contains configuration options for github issue tracker -github: - # base-url is the optional self-hosted github application url - base-url: https://localhost:8443/github - # username is the username of the github user +# GitHub contains configuration options for GitHub issue tracker +GitHub: + # base-url is the optional self-hosted GitHub application url + base-url: https://localhost:8443/GitHub + # username is the username of the GitHub user username: test-username # owner is the owner name of the repository for issues. owner: test-owner - # token is the token for github account. + # token is the token for GitHub account. token: test-token # project-name is the name of the repository. project-name: test-project # issue-label is the label of the created issue type issue-label: bug -# gitlab contains configuration options for gitlab issue tracker -gitlab: - # base-url is the optional self-hosted gitlab application url - base-url: https://localhost:8443/gitlab - # username is the username of the gitlab user +# GitLab contains configuration options for GitLab issue tracker +GitLab: + # base-url is the optional self-hosted GitLab application url + base-url: https://localhost:8443/GitLab + # username is the username of the GitLab user username: test-username - # token is the token for gitlab account. + # token is the token for GitLab account. token: test-token # project-name is the name/id of the project(repository). project-name: "1234" # issue-label is the label of the created issue type issue-label: bug -# jira contains configuration options for jira issue tracker -jira: +# Jira contains configuration options for Jira issue tracker +Jira: # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used cloud: true # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created update-existing: false - # URL is the jira application url - url: https://localhost/jira - # account-id is the account-id of the jira user or username in case of on-prem Jira + # URL is the Jira application url + url: https://localhost/Jira + # account-id is the account-id of the Jira user or username in case of on-prem Jira account-id: test-account-id - # email is the email of the user for jira instance + # email is the email of the user for Jira instance email: test@test.com - # token is the token for jira instance or password in case of on-prem Jira + # token is the token for Jira instance or password in case of on-prem Jira token: test-token # project-name is the name of the project. project-name: test-project-name diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index 5364db5f5..508446243 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -5,51 +5,51 @@ #deny-list: # severity: info, low, medium -# github contains configuration options for github issue tracker -#github: -# # base-url (optional) is the self-hosted github application url +# GitHub contains configuration options for GitHub issue tracker +#GitHub: +# # base-url (optional) is the self-hosted GitHub application url # base-url: "" -# # username is the username of the github user +# # username is the username of the GitHub user # username: "" # # owner is the owner name of the repository for issues. # owner: "" -# # token is the token for github account. +# # token is the token for GitHub account. # token: "" # # project-name is the name of the repository. # project-name: "" # # issue-label (optional) is the label of the created issue type # issue-label: "" -# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# # severity-as-label (optional) sets the severity as the label of the created issue type # severity-as-label: false -# gitlab contains configuration options for gitlab issue tracker -#gitlab: -# # base-url (optional) is the self-hosted gitlab application url +# GitLab contains configuration options for GitLab issue tracker +#GitLab: +# # base-url (optional) is the self-hosted GitLab application url # base-url: "" -# # username is the username of the gitlab user +# # username is the username of the GitLab user # username: "" -# # token is the token for gitlab account. +# # token is the token for GitLab account. # token: "" # # project-id is the ID of the repository. # project-id: "" # # issue-label (optional) is the label of the created issue type # issue-label: "" -# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# # severity-as-label (optional) sets the severity as the label of the created issue type # severity-as-label: false -# jira contains configuration options for jira issue tracker -#jira: +# Jira contains configuration options for Jira issue tracker +#Jira: # # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true # # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created # update-existing: false -# # URL is the jira application url +# # URL is the Jira application URL # url: "" -# # account-id is the account-id of the jira user or username in case of on-prem Jira +# # account-id is the account-id of the Jira user or username in case of on-prem Jira # account-id: "" -# # email is the email of the user for jira instance +# # email is the email of the user for Jira instance # email: "" -# # token is the token for jira instance or password in case of on-prem Jira +# # token is the token for Jira instance or password in case of on-prem Jira # token: "" # # project-name is the name of the project. # project-name: "" diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 116479732..059a96167 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -110,7 +110,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), - flagSet.IntVar(&options.InteractionsCooldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.IntVar(&options.InteractionsCoolDownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 055eca1ab..a5a74f773 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -151,7 +151,7 @@ func New(options *types.Options) (*Runner, error) { opts.Authorization = options.InteractshToken opts.CacheSize = int64(options.InteractionsCacheSize) opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second - opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second + opts.ColldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second opts.NoInteractsh = runner.options.NoInteractsh diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index cc38519b9..7b3ffc6b4 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -80,7 +80,7 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) { } // convertPathToAbsolute resolves the paths provided to absolute paths -// before doing any operations on them regardless of them being blob, folders, files, etc. +// before doing any operations on them regardless of them being BLOB, folders, files, etc. func (c *Catalog) convertPathToAbsolute(t string) (string, error) { if strings.Contains(t, "*") { file := filepath.Base(t) diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index 1420003ec..271b8e207 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -230,9 +230,9 @@ func splitCommaTrim(value string) []string { if !strings.Contains(value, ",") { return []string{strings.ToLower(value)} } - splitted := strings.Split(value, ",") - final := make([]string, len(splitted)) - for i, value := range splitted { + split := strings.Split(value, ",") + final := make([]string, len(split)) + for i, value := range split { final[i] = strings.ToLower(strings.TrimSpace(value)) } return final diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 1a61cb25b..ff8121588 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -98,8 +98,8 @@ func New(config *Config) (*Store, error) { finalWorkflows: config.Workflows, } - urlbasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 - if urlbasedTemplatesProvided { + urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 + if urlBasedTemplatesProvided { remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) if err != nil { return store, err @@ -109,7 +109,7 @@ func New(config *Config) (*Store, error) { } // Handle a case with no templates or workflows, where we use base directory - if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlbasedTemplatesProvided { + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided { store.finalTemplates = []string{config.TemplatesDirectory} } diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 873a94e85..60b6c20bf 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -18,7 +18,7 @@ func (e *Engine) Execute(templates []*templates.Template, target InputProvider) return e.ExecuteWithOpts(templates, target, false) } -// ExecuteWithOpts is execute with the full options +// ExecuteWithOpts executes with the full options func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { var finalTemplates []*templates.Template if !noCluster { @@ -72,9 +72,9 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template return } - wg.Waitgroup.Add() + wg.WaitGroup.Add() go func(value string) { - defer wg.Waitgroup.Done() + defer wg.WaitGroup.Done() var match bool var err error @@ -90,5 +90,5 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template results.CAS(false, match) }(scannedValue) }) - wg.Waitgroup.Wait() + wg.WaitGroup.Wait() } diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index cd2ea09ba..46ca8549d 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -7,7 +7,7 @@ import ( ) // WorkPool implements an execution pool for executing different -// types of task with different concurreny requirements. +// types of task with different concurrency requirements. // // It also allows Configuration of such requirements. This is used // for per-module like separate headless concurrency etc. @@ -17,7 +17,7 @@ type WorkPool struct { config WorkPoolConfig } -// WorkPoolConfig is the configuration for workpool +// WorkPoolConfig is the configuration for work pool type WorkPoolConfig struct { // InputConcurrency is the concurrency for inputs values. InputConcurrency int @@ -41,18 +41,18 @@ func NewWorkPool(config WorkPoolConfig) *WorkPool { } } -// Wait waits for all the workpool waitgroups to finish +// Wait waits for all the work pool wait groups to finish func (w *WorkPool) Wait() { w.Default.Wait() w.Headless.Wait() } -// InputWorkPool is a workpool per-input +// InputWorkPool is a work pool per-input type InputWorkPool struct { - Waitgroup *sizedwaitgroup.SizedWaitGroup + WaitGroup *sizedwaitgroup.SizedWaitGroup } -// InputPool returns a workpool for an input type +// InputPool returns a work pool for an input type func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool { var count int if templateType == types.HeadlessProtocol { @@ -61,5 +61,5 @@ func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool { count = w.config.InputConcurrency } swg := sizedwaitgroup.New(count) - return &InputWorkPool{Waitgroup: &swg} + return &InputWorkPool{WaitGroup: &swg} } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index c0dbd186b..daa381351 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -62,7 +62,7 @@ type Matcher struct { // description: | // Words contains word patterns required to be present in the response part. // examples: - // - name: Match for outlook mail protection domain + // - name: Match for Outlook mail protection domain // value: > // []string{"mail.protection.outlook.com"} // - name: Match for application/json in response headers diff --git a/v2/pkg/projectfile/httputil.go b/v2/pkg/projectfile/httputil.go index 6479fd20b..6be0b40db 100644 --- a/v2/pkg/projectfile/httputil.go +++ b/v2/pkg/projectfile/httputil.go @@ -79,23 +79,6 @@ func newInternalResponse() *InternalResponse { } } -// Unused -// func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest { -// intReq := newInternalRquest() - -// intReq.Target = target -// intReq.HTTPMajor = req.ProtoMajor -// intReq.HTTPMinor = req.ProtoMinor -// for k, v := range req.Header { -// intReq.Headers[k] = v -// } -// intReq.Headers = req.Header -// intReq.Method = req.Method -// intReq.Body = body - -// return intReq -// } - func toInternalResponse(resp *http.Response, body []byte) *InternalResponse { intResp := newInternalResponse() @@ -125,14 +108,3 @@ func fromInternalResponse(intResp *InternalResponse) *http.Response { Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)), } } - -// Unused -// func fromInternalRequest(intReq *InternalRequest) *http.Request { -// return &http.Request{ -// ProtoMinor: intReq.HTTPMinor, -// ProtoMajor: intReq.HTTPMajor, -// Header: intReq.Headers, -// ContentLength: int64(len(intReq.Body)), -// Body: ioutil.NopCloser(bytes.NewReader(intReq.Body)), -// } -// } diff --git a/v2/pkg/projectfile/project.go b/v2/pkg/projectfile/project.go index 71ad9e365..a64d7db1b 100644 --- a/v2/pkg/projectfile/project.go +++ b/v2/pkg/projectfile/project.go @@ -42,13 +42,13 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) { return nil, fmt.Errorf("not found") } - var httprecord HTTPRecord - httprecord.Response = newInternalResponse() - if err := unmarshal(data, &httprecord); err != nil { + var httpRecord HTTPRecord + httpRecord.Response = newInternalResponse() + if err := unmarshal(data, &httpRecord); err != nil { return nil, err } - return fromInternalResponse(httprecord.Response), nil + return fromInternalResponse(httpRecord.Response), nil } func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error { @@ -57,10 +57,10 @@ func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error { return err } - var httprecord HTTPRecord - httprecord.Request = req - httprecord.Response = toInternalResponse(resp, data) - data, err = marshal(httprecord) + var httpRecord HTTPRecord + httpRecord.Request = req + httpRecord.Response = toInternalResponse(resp, data) + data, err = marshal(httpRecord) if err != nil { return err } diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index 769026907..3fc3bdea7 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -13,12 +13,12 @@ type AttackType int // Supported values for the ProtocolType const ( - // BatteringRamAttack replaces same payload into all of the defined payload positions at once. + // BatteringRamAttack replaces same payload in all the defined payload positions at once BatteringRamAttack AttackType = iota + 1 // PitchForkAttack replaces variables with positional value from multiple wordlists PitchForkAttack - // ClusterbombAttack replaces variables with all possible combinations of values - ClusterbombAttack + // ClusterBombAttack replaces variables with all possible combinations of values + ClusterBombAttack limit ) @@ -26,7 +26,7 @@ const ( var attackTypeMappings = map[AttackType]string{ BatteringRamAttack: "batteringram", PitchForkAttack: "pitchfork", - ClusterbombAttack: "clusterbomb", + ClusterBombAttack: "clusterbomb", } func GetSupportedAttackTypes() []AttackType { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 07b3a3d0a..ca8f9b655 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -111,7 +111,7 @@ func (i *Iterator) Total() int { count = len(p.values) } } - case ClusterbombAttack: + case ClusterBombAttack: count = 1 for _, p := range i.payloads { count *= len(p.values) @@ -127,7 +127,7 @@ func (i *Iterator) Value() (map[string]interface{}, bool) { return i.batteringRamValue() case PitchForkAttack: return i.pitchforkValue() - case ClusterbombAttack: + case ClusterBombAttack: return i.clusterbombValue() default: return i.batteringRamValue() @@ -184,7 +184,7 @@ func (i *Iterator) clusterbombValue() (map[string]interface{}, bool) { signalNext = false } if !p.next() { - // No more inputs in this inputprovider + // No more inputs in this input provider if index == i.msbIterator { // Reset all previous wordlists and increment the msb counter i.msbIterator++ diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 71a033e6c..24a9e2731 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -54,7 +54,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterbombAttack, "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index 294ddca3a..305cae092 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -13,15 +13,15 @@ import ( // validate validates the payloads if any. func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePath string) error { for name, payload := range payloads { - switch pt := payload.(type) { + switch payloadType := payload.(type) { case string: // check if it's a multiline string list - if len(strings.Split(pt, "\n")) != 1 { + if len(strings.Split(payloadType, "\n")) != 1 { return errors.New("invalid number of lines in payload") } // check if it's a worldlist file and try to load it - if fileExists(pt) { + if fileExists(payloadType) { continue } @@ -29,18 +29,18 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat pathTokens := strings.Split(templatePath, string(os.PathSeparator)) for i := range pathTokens { - tpath := filepath.Join(filepath.Join(pathTokens[:i]...), pt) - if fileExists(tpath) { - payloads[name] = tpath + payloadPath := filepath.Join(filepath.Join(pathTokens[:i]...), payloadType) + if fileExists(payloadPath) { + payloads[name] = payloadPath changed = true break } } if !changed { - return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) + return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", payloadType, name) } case interface{}: - loadedPayloads := types.ToStringSlice(pt) + loadedPayloads := types.ToStringSlice(payloadType) if len(loadedPayloads) == 0 { return fmt.Errorf("the payload %s does not contain enough elements", name) } diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index fa13bd82e..a366a65bd 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -11,15 +11,15 @@ func TestCacheCheckMarkFailed(t *testing.T) { cache.MarkFailed("http://example.com:80") if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil { - require.Equal(t, 1, value, "could not get correct markfailed") + require.Equal(t, 1, value, "could not get correct number of marked failed hosts") } cache.MarkFailed("example.com:80") if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil { - require.Equal(t, 2, value, "could not get correct markfailed") + require.Equal(t, 2, value, "could not get correct number of marked failed hosts") } cache.MarkFailed("example.com") if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil { - require.Equal(t, 1, value, "could not get correct markfailed") + require.Equal(t, 1, value, "could not get correct number of marked failed hosts") } for i := 0; i < 3; i++ { cache.MarkFailed("test") diff --git a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go index 46308c9fe..d603621a9 100644 --- a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -23,7 +23,7 @@ var defaultResolvers = []string{ "8.8.4.4:53", // Google } -// Init initializes the clientpool implementation +// Init initializes the client pool implementation func Init(options *types.Options) error { // Don't create clients if already created in the past. if normalClient != nil { diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 0c1ff929c..f8b445fd6 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -76,7 +76,7 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (in } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, tracedata *retryabledns.TraceData) output.InternalEvent { +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, traceData *retryabledns.TraceData) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, @@ -91,7 +91,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, "type": request.Type().String(), - "trace": traceToString(tracedata, false), + "trace": traceToString(traceData, false), } } @@ -133,10 +133,10 @@ func questionToString(resourceRecords []dns.Question) string { return buffer.String() } -func traceToString(tracedata *retryabledns.TraceData, withSteps bool) string { +func traceToString(traceData *retryabledns.TraceData, withSteps bool) string { buffer := &bytes.Buffer{} - if tracedata != nil { - for i, dnsRecord := range tracedata.DNSData { + if traceData != nil { + for i, dnsRecord := range traceData.DNSData { if withSteps { buffer.WriteString(fmt.Sprintf("request %d to resolver %s:\n", i, strings.Join(dnsRecord.Resolver, ","))) } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index a1a7b5c37..1b6516b85 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -74,15 +74,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) // perform trace if necessary - var tracedata *retryabledns.TraceData + var traceData *retryabledns.TraceData if request.Trace { - tracedata, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion) + traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion) if err != nil { request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) } } - outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, tracedata) + outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, traceData) for k, v := range previous { outputEvent[k] = v } @@ -92,7 +92,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review dumpResponse(event, request.options, response.String(), domain) if request.Trace { - dumpTraceData(event, request.options, traceToString(tracedata, true), domain) + dumpTraceData(event, request.options, traceToString(traceData, true), domain) } callback(event) @@ -112,15 +112,15 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols. } } -func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, tracedata, domain string) { +func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, traceData, domain string) { cliOptions := requestOptions.Options if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if responsehighlighter.HasBinaryContent(tracedata) { + if responsehighlighter.HasBinaryContent(traceData) { hexDump = true - tracedata = hex.Dump([]byte(tracedata)) + traceData = hex.Dump([]byte(traceData)) } - highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, tracedata, cliOptions.NoColor, hexDump) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, traceData, cliOptions.NoColor, hexDump) gologger.Debug().Msgf("[%s] Dumped DNS Trace data for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) } } diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index c435bef40..d73e79044 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -21,7 +21,7 @@ import ( type Browser struct { customAgent string tempDir string - previouspids map[int32]struct{} // track already running pids + previousPIDs map[int32]struct{} // track already running PIDs engine *rod.Browser httpclient *http.Client options *types.Options @@ -33,7 +33,7 @@ func New(options *types.Options) (*Browser, error) { if err != nil { return nil, errors.Wrap(err, "could not create temporary directory") } - previouspids := findChromeProcesses() + previousPIDs := findChromeProcesses() chromeLauncher := launcher.New(). Leakless(false). @@ -89,7 +89,7 @@ func New(options *types.Options) (*Browser, error) { customAgent = uarand.GetRandom() } - httpclient, err := newhttpClient(options) + httpclient, err := newHttpClient(options) if err != nil { return nil, err } @@ -101,7 +101,7 @@ func New(options *types.Options) (*Browser, error) { httpclient: httpclient, options: options, } - engine.previouspids = previouspids + engine.previousPIDs = previousPIDs return engine, nil } @@ -123,7 +123,7 @@ func (b *Browser) killChromeProcesses() { continue } // skip chrome processes that were already running - if _, ok := b.previouspids[process.Pid]; ok { + if _, ok := b.previousPIDs[process.Pid]; ok { continue } _ = process.Kill() diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index e61803d4b..9b1c5b0df 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -18,8 +18,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// newhttpClient creates a new http client for headless communication with a timeout -func newhttpClient(options *types.Options) (*http.Client, error) { +// newHttpClient creates a new http client for headless communication with a timeout +func newHttpClient(options *types.Options) (*http.Client, error) { dialer := protocolstate.Dialer // Set the base TLS configuration definition diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 234d726eb..b186dcc5a 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -399,12 +399,12 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi return errors.Wrap(err, "could not scroll into view") } - selectedbool := false + selectedBool := false if act.GetArg("selected") == "true" { - selectedbool = true + selectedBool = true } by := act.GetArg("selector") - if err := element.Select([]string{value}, selectedbool, selectorBy(by)); err != nil { + if err := element.Select([]string{value}, selectedBool, selectorBy(by)); err != nil { return errors.Wrap(err, "could not select input") } return nil @@ -509,7 +509,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused protoEvent := &protoEvent{event: event} // Uses another instance in order to be able to chain the timeout only to the wait operation - pagec := p.page + pageCopy := p.page timeout := act.GetArg("timeout") if timeout != "" { ts, err := strconv.Atoi(timeout) @@ -517,11 +517,11 @@ func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused return errors.Wrap(err, "could not get timeout") } if ts > 0 { - pagec = p.page.Timeout(time.Duration(ts) * time.Second) + pageCopy = p.page.Timeout(time.Duration(ts) * time.Second) } } // Just wait the event to happen - pagec.WaitEvent(protoEvent)() + pageCopy.WaitEvent(protoEvent)() return nil } diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 0c5359415..64e9f4da3 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -8,7 +8,7 @@ import ( // routingRuleHandler handles proxy rule for actions related to request/response modification func (p *Page) routingRuleHandler(ctx *rod.Hijack) { - // usually browsers don't use chunked transfer encoding so we set the content-length nevertheless + // usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) for _, rule := range p.rules { diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 4719a5d24..7cca368a6 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -136,7 +136,7 @@ func TestMakeRequestFromRawWithPayloads(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -175,7 +175,7 @@ func TestMakeRequestFromRawPayloadExpressions(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -227,7 +227,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { ServerURL: options.InteractshURL, CacheSize: int64(options.InteractionsCacheSize), Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second, + ColldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, }) require.Nil(t, err, "could not create interactsh client") diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 17a87588d..a96177862 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -41,12 +41,12 @@ type Request struct { // Name is the optional name of the request. // // If a name is specified, all the named request in a template can be matched upon - // in a combined manner allowing multirequest based matchers. + // in a combined manner allowing multi-request based matchers. Name string `yaml:"name,omitempty" jsonschema:"title=name for the http request,description=Optional name for the HTTP Request"` // description: | // Attack is the type of payload combinations to perform. // - // batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates + // batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: // - "batteringram" @@ -137,7 +137,7 @@ type Request struct { dynamicValues map[string]interface{} // description: | - // SelfContained specifies if the request is self contained. + // SelfContained specifies if the request is self-contained. SelfContained bool `yaml:"-" json:"-"` // description: | diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index 4da40ce6f..aca703e9f 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -23,7 +23,7 @@ func TestHTTPCompile(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 9305870a6..5859e10cd 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -30,7 +30,7 @@ var ( // Dialer is a copy of the fastdialer from protocolstate Dialer *fastdialer.Dialer - rawhttpClient *rawhttp.Client + rawHttpClient *rawhttp.Client poolMutex *sync.RWMutex normalClient *retryablehttp.Client clientPool map[string]*retryablehttp.Client @@ -98,12 +98,12 @@ func (c *Configuration) HasStandardOptions() bool { // GetRawHTTP returns the rawhttp request client func GetRawHTTP(options *types.Options) *rawhttp.Client { - if rawhttpClient == nil { - rawhttpOptions := rawhttp.DefaultOptions - rawhttpOptions.Timeout = time.Duration(options.Timeout) * time.Second - rawhttpClient = rawhttp.NewClient(rawhttpOptions) + if rawHttpClient == nil { + rawHttpOptions := rawhttp.DefaultOptions + rawHttpOptions.Timeout = time.Duration(options.Timeout) * time.Second + rawHttpClient = rawhttp.NewClient(rawHttpOptions) } - return rawhttpClient + return rawHttpClient } // Get creates or gets a client for the protocol based on custom configuration @@ -138,7 +138,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // Multiple Host - retryablehttpOptions := retryablehttp.DefaultOptionsSpraying + retryableHttpOptions := retryablehttp.DefaultOptionsSpraying disableKeepAlives := true maxIdleConns := 0 maxConnsPerHost := 0 @@ -146,14 +146,14 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if configuration.Threads > 0 { // Single host - retryablehttpOptions = retryablehttp.DefaultOptionsSingle + retryableHttpOptions = retryablehttp.DefaultOptionsSingle disableKeepAlives = false maxIdleConnsPerHost = 500 maxConnsPerHost = 500 } - retryablehttpOptions.RetryWaitMax = 10 * time.Second - retryablehttpOptions.RetryMax = options.Retries + retryableHttpOptions.RetryWaitMax = 10 * time.Second + retryableHttpOptions.RetryMax = options.Retries followRedirects := configuration.FollowRedirects maxRedirects := configuration.MaxRedirects @@ -213,7 +213,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl Transport: transport, Timeout: time.Duration(options.Timeout) * time.Second, CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), - }, retryablehttpOptions) + }, retryableHttpOptions) if jar != nil { client.HTTPClient.Jar = jar } diff --git a/v2/pkg/protocols/http/race/syncedreadcloser.go b/v2/pkg/protocols/http/race/syncedreadcloser.go index c8f5de7ea..160f64ced 100644 --- a/v2/pkg/protocols/http/race/syncedreadcloser.go +++ b/v2/pkg/protocols/http/race/syncedreadcloser.go @@ -13,7 +13,7 @@ type SyncedReadCloser struct { data []byte p int64 length int64 - opengate chan struct{} + openGate chan struct{} enableBlocking bool } @@ -29,7 +29,7 @@ func NewSyncedReadCloser(r io.ReadCloser) *SyncedReadCloser { } r.Close() s.length = int64(len(s.data)) - s.opengate = make(chan struct{}) + s.openGate = make(chan struct{}) s.enableBlocking = true return &s } @@ -48,13 +48,13 @@ func (s *SyncedReadCloser) SetOpenGate(status bool) { // OpenGate opens the gate allowing all requests to be completed func (s *SyncedReadCloser) OpenGate() { - s.opengate <- struct{}{} + s.openGate <- struct{}{} } // OpenGateAfter schedules gate to be opened after a duration func (s *SyncedReadCloser) OpenGateAfter(d time.Duration) { time.AfterFunc(d, func() { - s.opengate <- struct{}{} + s.openGate <- struct{}{} }) } @@ -84,7 +84,7 @@ func (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) { func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { // If the data fits in the buffer blocks awaiting the sync instruction if s.p+int64(len(p)) >= s.length && s.enableBlocking { - <-s.opengate + <-s.openGate } n = copy(p, s.data[s.p:]) s.p += int64(n) @@ -94,7 +94,7 @@ func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { return n, err } -// Close implements close method for io.ReadSeeker +// Close closes an io.ReadSeeker func (s *SyncedReadCloser) Close() error { return nil } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index df4ff5b65..60db9b387 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -57,7 +57,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { // Set the request Method rawRequest.Method = parts[0] - var mutlipartRequest bool + var multiPartRequest bool // Accepts all malformed headers var key, value string for { @@ -76,7 +76,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { value = p[1] } if strings.Contains(key, "Content-Type") && strings.Contains(value, "multipart/") { - mutlipartRequest = true + multiPartRequest = true } // in case of unsafe requests multiple headers should be accepted @@ -136,7 +136,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { return nil, fmt.Errorf("could not read request body: %w", err) } rawRequest.Data = string(b) - if !mutlipartRequest { + if !multiPartRequest { rawRequest.Data = strings.TrimSuffix(rawRequest.Data, "\r\n") } return rawRequest, nil diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f2040782..bf92315d7 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -232,7 +232,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou reqURL = generatedHttpRequest.URL() } request.dynamicValues = generatedHttpRequest.dynamicValues - // Check if hosts just keep erroring + // Check if hosts keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break } @@ -269,7 +269,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou requestCount++ request.options.Progress.IncrementRequests() - // If this was a match and we want to stop at first match, skip all further requests. + // If this was a match, and we want to stop at first match, skip all further requests. if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { break } @@ -301,7 +301,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } dumpedRequestString := string(dumpedRequest) - // Check if are there any unresolved variables. If yes, skip unless overriden by user. + // Check if are there any unresolved variables. If yes, skip unless overridden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) return errStopExecution diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index ee8c524c0..8fb0fe70b 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -30,7 +30,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") @@ -54,7 +54,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 834cc091d..17cbd7ffe 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -98,13 +98,13 @@ func normalizeResponseBody(resp *http.Response, response *redirectedResponse) er // gb18030 supersedes gb2312 responseContentType := resp.Header.Get("Content-Type") if isContentTypeGbk(responseContentType) { - response.fullResponse, err = decodegbk(response.fullResponse) + response.fullResponse, err = decodeGBK(response.fullResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") } // the uncompressed body needs to be decoded to standard utf8 - response.body, err = decodegbk(response.body) + response.body, err = decodeGBK(response.body) if err != nil { return errors.Wrap(err, "could not gbk decode") } @@ -150,8 +150,8 @@ func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte, return bodyDec, nil } -// decodegbk converts GBK to UTF-8 -func decodegbk(s []byte) ([]byte, error) { +// decodeGBK converts GBK to UTF-8 +func decodeGBK(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) d, e := ioutil.ReadAll(O) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 09dcfb451..554d3bea8 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -34,7 +34,7 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates + // Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: // - "batteringram" @@ -68,7 +68,7 @@ type Request struct { ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` // description: | - // SelfContained specifies if the request is self contained. + // SelfContained specifies if the request is self-contained. SelfContained bool `yaml:"-" json:"-"` // Operators for the current request go here. @@ -110,7 +110,7 @@ type Input struct { // Read is the number of bytes to read from socket. // // This can be used for protocols which expect an immediate response. You can - // read and write responses one after another and evetually perform matching + // read and write responses one after another and eventually perform matching // on every data captured with `name` attribute. // // The [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this. diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 53a35df3c..d7641dd23 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -300,7 +300,7 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(err, "could not write request to server") } - // Only perform matching and writes in case we recieve + // Only perform matching and writes in case we receive // text or binary opcode from the websocket server. if opCode != ws.OpText && opCode != ws.OpBinary { continue diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 6e009f42b..a79e80124 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -16,7 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) -// Options contains necessary options required for elasticsearch communicaiton +// Options contains necessary options required for elasticsearch communication type Options struct { // IP for elasticsearch instance IP string `yaml:"ip" validate:"required,ip"` diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index d8f000584..f51470e1c 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -24,10 +24,10 @@ type Options struct { AllowList *Filter `yaml:"allow-list"` // DenyList contains a list of denied events for reporting module DenyList *Filter `yaml:"deny-list"` - // Github contains configuration options for Github Issue Tracker - Github *github.Options `yaml:"github"` - // Gitlab contains configuration options for Gitlab Issue Tracker - Gitlab *gitlab.Options `yaml:"gitlab"` + // GitHub contains configuration options for GitHub Issue Tracker + GitHub *github.Options `yaml:"github"` + // GitLab contains configuration options for GitLab Issue Tracker + GitLab *gitlab.Options `yaml:"gitlab"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` // MarkdownExporter contains configuration options for Markdown Exporter Module @@ -107,15 +107,15 @@ type Client struct { // New creates a new nuclei issue tracker reporting client func New(options *Options, db string) (*Client, error) { client := &Client{options: options} - if options.Github != nil { - tracker, err := github.New(options.Github) + if options.GitHub != nil { + tracker, err := github.New(options.GitHub) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") } client.trackers = append(client.trackers, tracker) } - if options.Gitlab != nil { - tracker, err := gitlab.New(options.Gitlab) + if options.GitLab != nil { + tracker, err := gitlab.New(options.GitLab) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") } diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 47225ada1..352c5183f 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -21,15 +21,15 @@ type Integration struct { options *Options } -// Options contains the configuration options for github issue tracker client +// Options contains the configuration options for GitHub issue tracker client type Options struct { - // BaseURL (optional) is the self-hosted github application url + // BaseURL (optional) is the self-hosted GitHub application url BaseURL string `yaml:"base-url" validate:"omitempty,url"` // Username is the username of the github user Username string `yaml:"username" validate:"required"` - // Owner (manadatory) is the owner name of the repository for issues. + // Owner is the owner name of the repository for issues. Owner string `yaml:"owner" validate:"required"` - // Token is the token for github account. + // Token is the token for GitHub account. Token string `yaml:"token" validate:"required"` // ProjectName is the name of the repository. ProjectName string `yaml:"project-name" validate:"required"` diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index eb547d0a6..088eab1b8 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -331,7 +331,7 @@ func init() { HTTPRequestDoc.Fields[6].Name = "name" HTTPRequestDoc.Fields[6].Type = "string" HTTPRequestDoc.Fields[6].Note = "" - HTTPRequestDoc.Fields[6].Description = "Name is the optional name of the request.\n\nIf a name is specified, all the named request in a template can be matched upon\nin a combined manner allowing multirequest based matchers." + HTTPRequestDoc.Fields[6].Description = "Name is the optional name of the request.\n\nIf a name is specified, all the named request in a template can be matched upon\nin a combined manner allowing multi-request based matchers." HTTPRequestDoc.Fields[6].Comments[encoder.LineComment] = "Name is the optional name of the request." HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 51fec82fd..99ddf9cef 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -61,7 +61,7 @@ var DefaultOptions = &types.Options{ InteractshURL: "https://interactsh.com", InteractionsCacheSize: 5000, InteractionsEviction: 60, - InteractionsCooldownPeriod: 5, + InteractionsCoolDownPeriod: 5, InteractionsPollDuration: 5, } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 1a0935338..69a26f5e0 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -70,7 +70,7 @@ type Options struct { ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string - // MarkdownExportDirectory is the directory to export reports in markdown format + // MarkdownExportDirectory is the directory to export reports in Markdown format MarkdownExportDirectory string // SarifExport is the file to export sarif output format to SarifExport string @@ -107,9 +107,9 @@ type Options struct { // Eviction is the number of seconds after which to automatically discard // interaction requests. InteractionsEviction int - // InteractionsCooldownPeriod is additional seconds to wait for interactions after closing + // InteractionsCoolDownPeriod is additional seconds to wait for interactions after closing // of the poller. - InteractionsCooldownPeriod int + InteractionsCoolDownPeriod int // OfflineHTTP is a flag that specific offline processing of http response // using same matchers/extractors from http protocol without the need // to send a new request, reading responses from a file. diff --git a/v2/pkg/utils/stats/stats.go b/v2/pkg/utils/stats/stats.go index d25d14334..9b2b7f2f0 100644 --- a/v2/pkg/utils/stats/stats.go +++ b/v2/pkg/utils/stats/stats.go @@ -30,7 +30,7 @@ func NewEntry(name, description string) { Default.NewEntry(name, description) } -// Increment incrmements the value for a name string +// Increment increments the value for a name string func Increment(name string) { Default.Increment(name) } @@ -57,7 +57,7 @@ func (s *Storage) NewEntry(name, description string) { s.mutex.Unlock() } -// Increment incrmements the value for a name string +// Increment increments the value for a name string func (s *Storage) Increment(name string) { s.mutex.RLock() data, ok := s.data[name] From 5eca474c5e227db149865a3d4619d11914b30680 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 26 Nov 2021 11:38:09 +0200 Subject: [PATCH 16/27] refactor(CR): better way of writing switch statement --- v2/pkg/protocols/headless/engine/rules.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 64e9f4da3..8dc206876 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -16,16 +16,16 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { continue } - switch { - case rule.Action == ActionSetMethod: + switch rule.Action { + case ActionSetMethod: ctx.Request.Req().Method = rule.Args["method"] - case rule.Action == ActionAddHeader: + case ActionAddHeader: ctx.Request.Req().Header.Add(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionSetHeader: + case ActionSetHeader: ctx.Request.Req().Header.Set(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionDeleteHeader: + case ActionDeleteHeader: ctx.Request.Req().Header.Del(rule.Args["key"]) - case rule.Action == ActionSetBody: + case ActionSetBody: body := rule.Args["body"] ctx.Request.Req().ContentLength = int64(len(body)) ctx.Request.SetBody(body) @@ -38,14 +38,14 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { continue } - switch { - case rule.Action == ActionAddHeader: + switch rule.Action { + case ActionAddHeader: ctx.Response.Headers().Add(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionSetHeader: + case ActionSetHeader: ctx.Response.Headers().Set(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionDeleteHeader: + case ActionDeleteHeader: ctx.Response.Headers().Del(rule.Args["key"]) - case rule.Action == ActionSetBody: + case ActionSetBody: body := rule.Args["body"] ctx.Response.Headers().Set("Content-Length", fmt.Sprintf("%d", len(body))) ctx.Response.SetBody(rule.Args["body"]) From dfe284664ce1f334a9a5baf04e925264ab62f219 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 26 Nov 2021 18:51:02 +0530 Subject: [PATCH 17/27] Fixed a crash with http module (#1285) --- v2/pkg/protocols/http/request.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f2040782..a0bce62ee 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -405,6 +405,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } var dumpedResponse []redirectedResponse + var gotData []byte // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader @@ -422,6 +423,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return errors.Wrap(err, "could not read http body") } } + gotData = data resp.Body.Close() dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) @@ -432,14 +434,17 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} } - for _, response := range dumpedResponse { - // if nuclei-project is enabled store the response if not previously done - if request.options.ProjectFile != nil && !fromCache { - if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil { - return errors.Wrap(err, "could not store in project file") - } + // if nuclei-project is enabled store the response if not previously done + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil { + return errors.Wrap(err, "could not store in project file") } + } + for _, response := range dumpedResponse { + if response.resp == nil { + continue // Skip nil responses + } matchedURL := reqURL if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { matchedURL = generatedRequest.rawRequest.FullURL From cd651ddf6f6af7ddd4d5fd313c885527305c0bdb Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 26 Nov 2021 17:12:27 +0200 Subject: [PATCH 18/27] fix: Escaped the dot in a regex matching a domain --- v2/cmd/docgen/docgen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/docgen/docgen.go b/v2/cmd/docgen/docgen.go index 1c8e9779a..7cfdaf275 100644 --- a/v2/cmd/docgen/docgen.go +++ b/v2/cmd/docgen/docgen.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) -var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`) +var pathRegex = regexp.MustCompile(`github\.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`) func main() { // Generate yaml syntax documentation From b59582bde7188932cd9776c5433e43eff4caba4c Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 28 Nov 2021 04:31:39 +0530 Subject: [PATCH 19/27] Fixed stdin input parsing bug (#1286) --- v2/internal/runner/options.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..c382f69d2 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -71,9 +71,8 @@ func hasStdin() bool { return false } - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) != 0 isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - return isPipedFromChrDev || isPipedFromFIFO } From b3805999f357843cd138f5fab3719795fb880910 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Sat, 27 Nov 2021 17:10:27 -0600 Subject: [PATCH 20/27] Unsafe flag with base template (#1279) * unsafe flag with base template --- v2/pkg/protocols/http/build_request.go | 6 ++++-- v2/pkg/protocols/http/request.go | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8c8a2dbc8..be1d13110 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -307,10 +307,12 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte } req.Body = ioutil.NopCloser(strings.NewReader(body)) } - setHeader(req, "User-Agent", uarand.GetRandom()) + if !r.request.Unsafe { + setHeader(req, "User-Agent", uarand.GetRandom()) + } // Only set these headers on non-raw requests - if len(r.request.Raw) == 0 { + if len(r.request.Raw) == 0 && !r.request.Unsafe { setHeader(req, "Accept", "*/*") setHeader(req, "Accept-Language", "en") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index a0bce62ee..a38b56b9c 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -310,9 +310,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) - } + } } - var formedURL string var hostname string timeStart := time.Now() From 5a1c7a62b2d6041895050c48bc000ab54e3c6ec4 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 28 Nov 2021 04:55:18 +0530 Subject: [PATCH 21/27] missing goimports --- v2/pkg/protocols/network/network.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 82eb4aa50..ef03cc92b 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,8 +1,6 @@ package network import ( - "fmt" - "net" "strings" "github.com/pkg/errors" From 75333dae3b5f558eee60f261b96db938a1e5802e Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 28 Nov 2021 05:29:01 +0530 Subject: [PATCH 22/27] Revert "Fixed stdin input parsing bug (#1286)" This reverts commit b59582bde7188932cd9776c5433e43eff4caba4c. --- v2/internal/runner/options.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index c382f69d2..f11d2055c 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -71,8 +71,9 @@ func hasStdin() bool { return false } - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) != 0 + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 + return isPipedFromChrDev || isPipedFromFIFO } From b1a0da290a034f413ee614ed0e83089beecb1e2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 10:52:29 +0530 Subject: [PATCH 23/27] chore(deps): bump alpine from 3.14 to 3.15.0 (#1305) Bumps alpine from 3.14 to 3.15.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c5c46e839..322fd54d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.17.3-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest -FROM alpine:3.14 +FROM alpine:3.15.0 RUN apk add --no-cache bind-tools ca-certificates chromium COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei ENTRYPOINT ["nuclei"] From 345a5a3de5ed305472b3de11656e369f0962bde5 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 29 Nov 2021 06:49:56 +0000 Subject: [PATCH 24/27] Auto Generate Syntax Docs + JSONSchema [Mon Nov 29 06:49:56 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 10 +++++----- v2/pkg/templates/templates_doc.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 6c1742432..be39aa225 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -847,7 +847,7 @@ ID is the optional id of the request Name is the optional name of the request. If a name is specified, all the named request in a template can be matched upon -in a combined manner allowing multirequest based matchers. +in a combined manner allowing multi-request based matchers. @@ -862,7 +862,7 @@ in a combined manner allowing multirequest based matchers. Attack is the type of payload combinations to perform. -batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates +batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. @@ -1454,7 +1454,7 @@ Examples: ```yaml -# Match for outlook mail protection domain +# Match for Outlook mail protection domain words: - mail.protection.outlook.com ``` @@ -2452,7 +2452,7 @@ host: Attack is the type of payload combinations to perform. -Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates +Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. @@ -2676,7 +2676,7 @@ Valid values: Read is the number of bytes to read from socket. This can be used for protocols which expect an immediate response. You can -read and write responses one after another and evetually perform matching +read and write responses one after another and eventually perform matching on every data captured with `name` attribute. The [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this. diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 088eab1b8..b436ef472 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -336,7 +336,7 @@ func init() { HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" HTTPRequestDoc.Fields[7].Note = "" - HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." HTTPRequestDoc.Fields[7].Values = []string{ "batteringram", @@ -559,7 +559,7 @@ func init() { MATCHERSMatcherDoc.Fields[7].Description = "Words contains word patterns required to be present in the response part." MATCHERSMatcherDoc.Fields[7].Comments[encoder.LineComment] = "Words contains word patterns required to be present in the response part." - MATCHERSMatcherDoc.Fields[7].AddExample("Match for outlook mail protection domain", []string{"mail.protection.outlook.com"}) + MATCHERSMatcherDoc.Fields[7].AddExample("Match for Outlook mail protection domain", []string{"mail.protection.outlook.com"}) MATCHERSMatcherDoc.Fields[7].AddExample("Match for application/json in response headers", []string{"application/json"}) MATCHERSMatcherDoc.Fields[8].Name = "regex" @@ -938,7 +938,7 @@ func init() { NETWORKRequestDoc.Fields[2].Name = "attack" NETWORKRequestDoc.Fields[2].Type = "generators.AttackTypeHolder" NETWORKRequestDoc.Fields[2].Note = "" - NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." NETWORKRequestDoc.Fields[2].Values = []string{ "batteringram", @@ -1020,7 +1020,7 @@ func init() { NETWORKInputDoc.Fields[2].Name = "read" NETWORKInputDoc.Fields[2].Type = "int" NETWORKInputDoc.Fields[2].Note = "" - NETWORKInputDoc.Fields[2].Description = "Read is the number of bytes to read from socket.\n\nThis can be used for protocols which expect an immediate response. You can\nread and write responses one after another and evetually perform matching\non every data captured with `name` attribute.\n\nThe [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this." + NETWORKInputDoc.Fields[2].Description = "Read is the number of bytes to read from socket.\n\nThis can be used for protocols which expect an immediate response. You can\nread and write responses one after another and eventually perform matching\non every data captured with `name` attribute.\n\nThe [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this." NETWORKInputDoc.Fields[2].Comments[encoder.LineComment] = "Read is the number of bytes to read from socket." NETWORKInputDoc.Fields[2].AddExample("", 1024) From c9943c0b2a4f3a9e15d1bdc53dec5c96f68327fc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 29 Nov 2021 14:38:55 +0530 Subject: [PATCH 25/27] fix: #1274 spawned nuclei child process hangs reading stdin Next and final attempt at fixing the stdin parsing issue when spawing as child from nodejs --- v2/internal/runner/options.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..d9eaf3adc 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -66,15 +66,14 @@ func ParseOptions(options *types.Options) { // hasStdin returns true if we have stdin input func hasStdin() bool { - stat, err := os.Stdin.Stat() + fi, err := os.Stdin.Stat() if err != nil { return false } - - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 - isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - - return isPipedFromChrDev || isPipedFromFIFO + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } + return true } // validateOptions validates the configuration options passed From 65a1858f940797d735bc42a7f3a06f2a2883289d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 30 Nov 2021 00:17:52 +0530 Subject: [PATCH 26/27] fix #1303: speed regression due to missing goroutine in executer Fixed speed regression introduced in dev due to missing go() statement when executing template input --- v2/pkg/core/execute.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 60b6c20bf..9e58a3d3f 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -39,15 +39,17 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp } wg.Add() - switch { - case template.SelfContained: - // Self Contained requests are executed here separately - e.executeSelfContainedTemplateWithInput(template, results) - default: - // All other request types are executed here - e.executeModelWithInput(templateType, template, target, results) - } - wg.Done() + go func(tpl *templates.Template) { + switch { + case tpl.SelfContained: + // Self Contained requests are executed here separately + e.executeSelfContainedTemplateWithInput(tpl, results) + default: + // All other request types are executed here + e.executeModelWithInput(templateType, tpl, target, results) + } + wg.Done() + }(template) } e.workPool.Wait() return results From 684f332599f674210512003e3bdef545578eb73f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 30 Nov 2021 12:04:47 +0530 Subject: [PATCH 27/27] fix: {{Hostname}} to {{Host}} in test --- v2/pkg/protocols/network/network_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 730f9584a..27a64d592 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -17,7 +17,7 @@ func TestNetworkCompileMake(t *testing.T) { templateID := "testing-network" request := &Request{ ID: templateID, - Address: []string{"tls://{{Hostname}}:443"}, + Address: []string{"tls://{{Host}}:443"}, ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, }