diff --git a/Dockerfile b/Dockerfile index 607984fc8..00f5c1538 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.2-alpine as build-env +FROM golang:1.19.3-alpine as build-env RUN apk add build-base RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go index 6df1ce4da..c7b716ebd 100644 --- a/v2/cmd/functional-test/main.go +++ b/v2/cmd/functional-test/main.go @@ -66,6 +66,10 @@ func runTestCases(file *os.File, debug bool) (bool, []string) { if testCase == "" { continue } + // skip comments + if strings.HasPrefix(testCase, "#") { + continue + } if runTestCase(testCase, debug) { errored = true failedTestCases = append(failedTestCases, testCase) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index b31170223..c4d66abc9 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,4 +1,7 @@ +# Simple binary invocation {{binary}} + +# Template tags filter {{binary}} -tags cve -ntv 8.8.8,8.8.9 {{binary}} -tags cve {{binary}} -tags cve,exposure @@ -49,7 +52,32 @@ {{binary}} -tags cve -author geeknik,pdteam -tc severity=='high' {{binary}} -tc contains(authors,'pdteam') {{binary}} -t cves/ -t exposures/ -tc contains(tags,'cve') -exclude-templates cves/2020/CVE-2020-9757.yaml + +# Workflow Filters {{binary}} -w workflows {{binary}} -w workflows -author geeknik,pdteam {{binary}} -w workflows -severity high,critical {{binary}} -w workflows -author geeknik,pdteam -severity high,critical + +# Input Types +# http protocol +# host +{{binary}} -id tech-detect -u scanme.sh +# host:port +{{binary}} -id tech-detect -u scanme.sh:80 +# scheme://host:port +{{binary}} -id tech-detect -u http://scanme.sh:80 +# scheme://host +{{binary}} -id tech-detect -u https://scanme.sh + +# Network Protocol +# host +{{binary}} -id ftp-weak-credentials -u scanme.sh +# host:port +{{binary}} -id ftp-weak-credentials -u scanme.sh:21 + +# SSL Protocol +# host +{{binary}} -id tls-version -u scanme.sh +# host:port +{{binary}} -id tls-version -u scanme.sh:22 diff --git a/v2/go.mod b/v2/go.mod index 848ed4777..75814d547 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -57,7 +57,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.129 +require github.com/aws/aws-sdk-go v1.44.132 require ( github.com/DataDog/gostackparse v0.6.0 @@ -77,13 +77,13 @@ require ( github.com/labstack/echo/v4 v4.9.1 github.com/mholt/archiver v3.1.1+incompatible github.com/mitchellh/go-homedir v1.1.0 - github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.3 github.com/projectdiscovery/nvd v1.0.9 github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa + github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/tlsx v0.0.9 - github.com/projectdiscovery/utils v0.0.1 + github.com/projectdiscovery/utils v0.0.2 github.com/projectdiscovery/wappalyzergo v0.0.67 github.com/stretchr/testify v1.8.1 gopkg.in/src-d/go-git.v4 v4.13.1 diff --git a/v2/go.sum b/v2/go.sum index 1dfaacd7b..8a121bc3a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -609,8 +609,6 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7 github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 h1:3Xn/CN6GVY+7mVuGgt5bfp0F9JwcWqnvwfb23Jf8Vxg= -github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7/go.mod h1:inRbqVxN7ri77yTJY3ZtGtKegIFa3Qnarh7Xp9P7LgY= github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= @@ -696,6 +694,8 @@ github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/rawhttp v0.1.2 h1:fCK42+qc5qYR4Dj/BVoukQ182h4n/w0dRcGVO92T7eI= github.com/projectdiscovery/rawhttp v0.1.2/go.mod h1:Q5PDAmKzjAjweEp0CQr9301nyxCOkzA9ImK6qLjgk+8= +github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= +github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= @@ -721,8 +721,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= -github.com/projectdiscovery/utils v0.0.1 h1:8jxai1EF3z/lpHeUCQEyNvEzs2Y53bzsMzEhoFjjwo0= -github.com/projectdiscovery/utils v0.0.1/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= +github.com/projectdiscovery/utils v0.0.2 h1:lif4OYBqd8jCf0glRBfSs2lT4nMtNjHjeysRw8HIW8M= +github.com/projectdiscovery/utils v0.0.2/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ= github.com/projectdiscovery/wappalyzergo v0.0.67/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= diff --git a/v2/internal/runner/enumerate.go b/v2/internal/runner/enumerate.go index 574721325..15e1bfb74 100644 --- a/v2/internal/runner/enumerate.go +++ b/v2/internal/runner/enumerate.go @@ -44,7 +44,7 @@ func (r *Runner) getScanList() error { if !v.Finished { status = "RUNNING" t = time.Now().UTC() - duration = t.Sub(v.CreatedAt) + duration = t.Sub(v.CreatedAt).Round(60 * time.Second) } val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss) diff --git a/v2/pkg/input/input.go b/v2/pkg/input/input.go index 67c18994a..5ac96dc7d 100644 --- a/v2/pkg/input/input.go +++ b/v2/pkg/input/input.go @@ -36,17 +36,17 @@ func (h *Helper) Close() error { func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) string { switch protocol { case templateTypes.DNSProtocol, templateTypes.WHOISProtocol: - return h.convertInputToType(input, inputTypeHost, "") + return h.convertInputToType(input, typeHostOnly, "") case templateTypes.FileProtocol, templateTypes.OfflineHTTPProtocol: - return h.convertInputToType(input, inputTypeFilepath, "") + return h.convertInputToType(input, typeFilepath, "") case templateTypes.HTTPProtocol, templateTypes.HeadlessProtocol: - return h.convertInputToType(input, inputTypeURL, "") + return h.convertInputToType(input, typeURL, "") case templateTypes.NetworkProtocol: - return h.convertInputToType(input, inputTypeHostPort, "") + return h.convertInputToType(input, typeHostWithOptionalPort, "") case templateTypes.SSLProtocol: - return h.convertInputToType(input, inputTypeHostPort, "443") + return h.convertInputToType(input, typeHostWithPort, "443") case templateTypes.WebsocketProtocol: - return h.convertInputToType(input, inputTypeWebsocket, "") + return h.convertInputToType(input, typeWebsocket, "") } return input } @@ -54,18 +54,18 @@ func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) st type inputType int const ( - inputTypeHost inputType = iota + 1 - inputTypeURL - inputTypeFilepath - inputTypeHostPort - inputTypeWebsocket + typeHostOnly inputType = iota + 1 + typeHostWithPort + typeHostWithOptionalPort + typeURL + typeFilepath + typeWebsocket ) // convertInputToType converts an input based on an inputType. // Various formats are supported for inputs and their transformation func (h *Helper) convertInputToType(input string, inputType inputType, defaultPort string) string { notURL := !strings.Contains(input, "://") - parsed, _ := url.Parse(input) var host, port string if !notURL { @@ -73,9 +73,11 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo } else { host, port, _ = net.SplitHostPort(input) } + hasPort := port != "" - if inputType == inputTypeFilepath { - if port != "" { + if inputType == typeFilepath { + // if it has ports most likely it's not a file + if hasPort { return "" } if filepath.IsAbs(input) { @@ -87,7 +89,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo if _, err := filepath.Match(input, ""); err != filepath.ErrBadPattern && notURL { return input } - } else if inputType == inputTypeHost { + } else if inputType == typeHostOnly { if host != "" { return host } @@ -96,7 +98,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo } else { return input } - } else if inputType == inputTypeURL { + } else if inputType == typeURL { if parsed != nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { return input } @@ -105,7 +107,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo return string(probed) } } - } else if inputType == inputTypeHostPort { + } else if inputType == typeHostWithPort { if host != "" && port != "" { return net.JoinHostPort(host, port) } @@ -115,7 +117,18 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo if defaultPort != "" { return net.JoinHostPort(input, defaultPort) } - } else if inputType == inputTypeWebsocket { + } else if inputType == typeHostWithOptionalPort { + if host != "" && port != "" { + return net.JoinHostPort(host, port) + } + if parsed != nil && port == "" && parsed.Scheme == "https" { + return net.JoinHostPort(parsed.Host, "443") + } + if defaultPort != "" { + return net.JoinHostPort(input, defaultPort) + } + return input + } else if inputType == typeWebsocket { if parsed != nil && (parsed.Scheme == "ws" || parsed.Scheme == "wss") { return input } diff --git a/v2/pkg/input/input_test.go b/v2/pkg/input/input_test.go index c253d81a5..6a145326b 100644 --- a/v2/pkg/input/input_test.go +++ b/v2/pkg/input/input_test.go @@ -24,37 +24,37 @@ func TestConvertInputToType(t *testing.T) { defaultPort string }{ // host - {"google.com", inputTypeHost, "google.com", ""}, - {"google.com:443", inputTypeHost, "google.com", ""}, - {"https://google.com", inputTypeHost, "google.com", ""}, - {"https://google.com:443", inputTypeHost, "google.com", ""}, + {"google.com", typeHostOnly, "google.com", ""}, + {"google.com:443", typeHostOnly, "google.com", ""}, + {"https://google.com", typeHostOnly, "google.com", ""}, + {"https://google.com:443", typeHostOnly, "google.com", ""}, // url - {"test.com", inputTypeURL, "", ""}, - {"google.com", inputTypeURL, "https://google.com", ""}, - {"https://google.com", inputTypeURL, "https://google.com", ""}, + {"test.com", typeURL, "", ""}, + {"google.com", typeURL, "https://google.com", ""}, + {"https://google.com", typeURL, "https://google.com", ""}, // file - {"google.com:443", inputTypeFilepath, "", ""}, - {"https://google.com:443", inputTypeFilepath, "", ""}, - {"/example/path", inputTypeFilepath, "/example/path", ""}, - {"input_test.go", inputTypeFilepath, "input_test.go", ""}, - {"../input", inputTypeFilepath, "../input", ""}, - {"input_test.*", inputTypeFilepath, "input_test.*", ""}, + {"google.com:443", typeFilepath, "", ""}, + {"https://google.com:443", typeFilepath, "", ""}, + {"/example/path", typeFilepath, "/example/path", ""}, + {"input_test.go", typeFilepath, "input_test.go", ""}, + {"../input", typeFilepath, "../input", ""}, + {"input_test.*", typeFilepath, "input_test.*", ""}, // host-port - {"google.com", inputTypeHostPort, "", ""}, - {"google.com:443", inputTypeHostPort, "google.com:443", ""}, - {"https://google.com", inputTypeHostPort, "google.com:443", ""}, - {"https://google.com:443", inputTypeHostPort, "google.com:443", ""}, + {"google.com", typeHostWithPort, "", ""}, + {"google.com:443", typeHostWithPort, "google.com:443", ""}, + {"https://google.com", typeHostWithPort, "google.com:443", ""}, + {"https://google.com:443", typeHostWithPort, "google.com:443", ""}, // host-port with default port - {"google.com", inputTypeHostPort, "google.com:443", "443"}, + {"google.com", typeHostWithPort, "google.com:443", "443"}, // websocket - {"google.com", inputTypeWebsocket, "", ""}, - {"google.com:443", inputTypeWebsocket, "", ""}, - {"https://google.com:443", inputTypeWebsocket, "", ""}, - {"wss://google.com", inputTypeWebsocket, "wss://google.com", ""}, + {"google.com", typeWebsocket, "", ""}, + {"google.com:443", typeWebsocket, "", ""}, + {"https://google.com:443", typeWebsocket, "", ""}, + {"wss://google.com", typeWebsocket, "wss://google.com", ""}, } for _, test := range tests { diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index 80216dda8..4e2cc33e4 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois/rdapclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -27,7 +28,13 @@ func Init(options *types.Options) error { if err := signerpool.Init(options); err != nil { return err } - return networkclientpool.Init(options) + if err := networkclientpool.Init(options); err != nil { + return err + } + if err := rdapclientpool.Init(options); err != nil { + return err + } + return nil } var userAgents = []string{ diff --git a/v2/pkg/protocols/whois/rdapclientpool/clientpool.go b/v2/pkg/protocols/whois/rdapclientpool/clientpool.go new file mode 100644 index 000000000..6aa513fb3 --- /dev/null +++ b/v2/pkg/protocols/whois/rdapclientpool/clientpool.go @@ -0,0 +1,38 @@ +package rdapclientpool + +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/rdap" +) + +var normalClient *rdap.Client + +// Init initializes the client pool implementation +func Init(options *types.Options) error { + // Don't create clients if already created in the past. + if normalClient != nil { + return nil + } + + normalClient = &rdap.Client{} + if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse { + normalClient.Verbose = func(text string) { + gologger.Debug().Msgf("rdap: %s", text) + } + } + return nil +} + +// Configuration contains the custom configuration options for a client - placeholder +type Configuration struct{} + +// Hash returns the hash of the configuration to allow client pooling - placeholder +func (c *Configuration) Hash() string { + return "" +} + +// Get creates or gets a client for the protocol based on custom configuration +func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) { + return normalClient, nil +} diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index a464e2f63..7d7fcf7a1 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -6,8 +6,8 @@ import ( "time" jsoniter "github.com/json-iterator/go" - "github.com/openrdap/rdap" "github.com/pkg/errors" + "github.com/projectdiscovery/rdap" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois/rdapclientpool" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -57,12 +58,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } request.options = options - request.client = &rdap.Client{} - if request.options.Options.Verbose || request.options.Options.Debug || request.options.Options.DebugRequests { - request.client.Verbose = func(text string) { - gologger.Debug().Msgf("rdap: %s", text) - } - } + request.client, _ = rdapclientpool.Get(options.Options, nil) if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators