Merge branch 'dev' into issue-2772-s3-provider-support

2025-12-29 20:33:05 +00:00 · 2022-11-09 12:52:17 +05:30 · 2022-11-09 12:52:17 +05:30 · ab3203b243
commit ab3203b243
parent c31517fc13 9e56451d2e
11 changed files with 143 additions and 57 deletions
--- a/2
+++ b/2
@ -1,4 +1,4 @@
-FROM golang:1.19.2-alpine as build-env
+FROM golang:1.19.3-alpine as build-env
 RUN apk add build-base
 RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

--- a/v2/cmd/functional-test/main.go
+++ b/v2/cmd/functional-test/main.go
@ -66,6 +66,10 @@ func runTestCases(file *os.File, debug bool) (bool, []string) {
 		if testCase == "" {
 			continue
 		}
+		// skip comments
+		if strings.HasPrefix(testCase, "#") {
+			continue
+		}
 		if runTestCase(testCase, debug) {
 			errored = true
 			failedTestCases = append(failedTestCases, testCase)
--- a/v2/cmd/functional-test/testcases.txt
+++ b/v2/cmd/functional-test/testcases.txt
@ -1,4 +1,7 @@
+# Simple binary invocation
 {{binary}}
+
+# Template tags filter
 {{binary}} -tags cve -ntv 8.8.8,8.8.9
 {{binary}} -tags cve
 {{binary}} -tags cve,exposure
@ -49,7 +52,32 @@
 {{binary}} -tags cve -author geeknik,pdteam  -tc severity=='high'
 {{binary}} -tc contains(authors,'pdteam')
 {{binary}} -t cves/ -t exposures/ -tc contains(tags,'cve') -exclude-templates cves/2020/CVE-2020-9757.yaml
+
+# Workflow Filters
 {{binary}} -w workflows
 {{binary}} -w workflows -author geeknik,pdteam
 {{binary}} -w workflows -severity high,critical
 {{binary}} -w workflows -author geeknik,pdteam  -severity high,critical
+
+# Input Types
+# http protocol
+# host
+{{binary}} -id tech-detect -u scanme.sh
+# host:port
+{{binary}} -id tech-detect -u scanme.sh:80
+# scheme://host:port
+{{binary}} -id tech-detect -u http://scanme.sh:80
+# scheme://host
+{{binary}} -id tech-detect -u https://scanme.sh
+
+# Network Protocol
+# host
+{{binary}} -id ftp-weak-credentials -u scanme.sh
+# host:port
+{{binary}} -id ftp-weak-credentials -u scanme.sh:21
+
+# SSL Protocol
+# host
+{{binary}} -id tls-version -u scanme.sh
+# host:port
+{{binary}} -id tls-version -u scanme.sh:22
--- a/v2/go.mod
+++ b/v2/go.mod
@ -57,7 +57,7 @@ require (
 	moul.io/http2curl v1.0.0
 )

-require github.com/aws/aws-sdk-go v1.44.129
+require github.com/aws/aws-sdk-go v1.44.132

 require (
 	github.com/DataDog/gostackparse v0.6.0
@ -77,13 +77,13 @@ require (
 	github.com/labstack/echo/v4 v4.9.1
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/goflags v0.1.3
 	github.com/projectdiscovery/nvd v1.0.9
 	github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa
+	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/tlsx v0.0.9
-	github.com/projectdiscovery/utils v0.0.1
+	github.com/projectdiscovery/utils v0.0.2
 	github.com/projectdiscovery/wappalyzergo v0.0.67
 	github.com/stretchr/testify v1.8.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
--- a/v2/go.sum
+++ b/v2/go.sum
@ -609,8 +609,6 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7
 github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 h1:3Xn/CN6GVY+7mVuGgt5bfp0F9JwcWqnvwfb23Jf8Vxg=
-github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7/go.mod h1:inRbqVxN7ri77yTJY3ZtGtKegIFa3Qnarh7Xp9P7LgY=
 github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8=
 github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U=
 github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U=
@ -696,6 +694,8 @@ github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/rawhttp v0.1.2 h1:fCK42+qc5qYR4Dj/BVoukQ182h4n/w0dRcGVO92T7eI=
 github.com/projectdiscovery/rawhttp v0.1.2/go.mod h1:Q5PDAmKzjAjweEp0CQr9301nyxCOkzA9ImK6qLjgk+8=
+github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
+github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
@ -721,8 +721,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
 github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU=
 github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI=
-github.com/projectdiscovery/utils v0.0.1 h1:8jxai1EF3z/lpHeUCQEyNvEzs2Y53bzsMzEhoFjjwo0=
-github.com/projectdiscovery/utils v0.0.1/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI=
+github.com/projectdiscovery/utils v0.0.2 h1:lif4OYBqd8jCf0glRBfSs2lT4nMtNjHjeysRw8HIW8M=
+github.com/projectdiscovery/utils v0.0.2/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI=
 github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ=
 github.com/projectdiscovery/wappalyzergo v0.0.67/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
 github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
--- a/v2/internal/runner/enumerate.go
+++ b/v2/internal/runner/enumerate.go
@ -44,7 +44,7 @@ func (r *Runner) getScanList() error {
 		if !v.Finished {
 			status = "RUNNING"
 			t = time.Now().UTC()
-			duration = t.Sub(v.CreatedAt)
+			duration = t.Sub(v.CreatedAt).Round(60 * time.Second)
 		}

 		val := v.CreatedAt.In(loc).Format(DDMMYYYYhhmmss)
--- a/v2/pkg/input/input.go
+++ b/v2/pkg/input/input.go
@ -36,17 +36,17 @@ func (h *Helper) Close() error {
 func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) string {
 	switch protocol {
 	case templateTypes.DNSProtocol, templateTypes.WHOISProtocol:
-		return h.convertInputToType(input, inputTypeHost, "")
+		return h.convertInputToType(input, typeHostOnly, "")
 	case templateTypes.FileProtocol, templateTypes.OfflineHTTPProtocol:
-		return h.convertInputToType(input, inputTypeFilepath, "")
+		return h.convertInputToType(input, typeFilepath, "")
 	case templateTypes.HTTPProtocol, templateTypes.HeadlessProtocol:
-		return h.convertInputToType(input, inputTypeURL, "")
+		return h.convertInputToType(input, typeURL, "")
 	case templateTypes.NetworkProtocol:
-		return h.convertInputToType(input, inputTypeHostPort, "")
+		return h.convertInputToType(input, typeHostWithOptionalPort, "")
 	case templateTypes.SSLProtocol:
-		return h.convertInputToType(input, inputTypeHostPort, "443")
+		return h.convertInputToType(input, typeHostWithPort, "443")
 	case templateTypes.WebsocketProtocol:
-		return h.convertInputToType(input, inputTypeWebsocket, "")
+		return h.convertInputToType(input, typeWebsocket, "")
 	}
 	return input
 }
@ -54,18 +54,18 @@ func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) st
 type inputType int

 const (
-	inputTypeHost inputType = iota + 1
-	inputTypeURL
-	inputTypeFilepath
-	inputTypeHostPort
-	inputTypeWebsocket
+	typeHostOnly inputType = iota + 1
+	typeHostWithPort
+	typeHostWithOptionalPort
+	typeURL
+	typeFilepath
+	typeWebsocket
 )

 // convertInputToType converts an input based on an inputType.
 // Various formats are supported for inputs and their transformation
 func (h *Helper) convertInputToType(input string, inputType inputType, defaultPort string) string {
 	notURL := !strings.Contains(input, "://")
-
 	parsed, _ := url.Parse(input)
 	var host, port string
 	if !notURL {
@ -73,9 +73,11 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 	} else {
 		host, port, _ = net.SplitHostPort(input)
 	}
+	hasPort := port != ""

-	if inputType == inputTypeFilepath {
-		if port != "" {
+	if inputType == typeFilepath {
+		// if it has ports most likely it's not a file
+		if hasPort {
 			return ""
 		}
 		if filepath.IsAbs(input) {
@ -87,7 +89,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 		if _, err := filepath.Match(input, ""); err != filepath.ErrBadPattern && notURL {
 			return input
 		}
-	} else if inputType == inputTypeHost {
+	} else if inputType == typeHostOnly {
 		if host != "" {
 			return host
 		}
@ -96,7 +98,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 		} else {
 			return input
 		}
-	} else if inputType == inputTypeURL {
+	} else if inputType == typeURL {
 		if parsed != nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
 			return input
 		}
@ -105,7 +107,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 				return string(probed)
 			}
 		}
-	} else if inputType == inputTypeHostPort {
+	} else if inputType == typeHostWithPort {
 		if host != "" && port != "" {
 			return net.JoinHostPort(host, port)
 		}
@ -115,7 +117,18 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 		if defaultPort != "" {
 			return net.JoinHostPort(input, defaultPort)
 		}
-	} else if inputType == inputTypeWebsocket {
+	} else if inputType == typeHostWithOptionalPort {
+		if host != "" && port != "" {
+			return net.JoinHostPort(host, port)
+		}
+		if parsed != nil && port == "" && parsed.Scheme == "https" {
+			return net.JoinHostPort(parsed.Host, "443")
+		}
+		if defaultPort != "" {
+			return net.JoinHostPort(input, defaultPort)
+		}
+		return input
+	} else if inputType == typeWebsocket {
 		if parsed != nil && (parsed.Scheme == "ws" || parsed.Scheme == "wss") {
 			return input
 		}
--- a/v2/pkg/input/input_test.go
+++ b/v2/pkg/input/input_test.go
@ -24,37 +24,37 @@ func TestConvertInputToType(t *testing.T) {
 		defaultPort string
 	}{
 		// host
-		{"google.com", inputTypeHost, "google.com", ""},
-		{"google.com:443", inputTypeHost, "google.com", ""},
-		{"https://google.com", inputTypeHost, "google.com", ""},
-		{"https://google.com:443", inputTypeHost, "google.com", ""},
+		{"google.com", typeHostOnly, "google.com", ""},
+		{"google.com:443", typeHostOnly, "google.com", ""},
+		{"https://google.com", typeHostOnly, "google.com", ""},
+		{"https://google.com:443", typeHostOnly, "google.com", ""},

 		// url
-		{"test.com", inputTypeURL, "", ""},
-		{"google.com", inputTypeURL, "https://google.com", ""},
-		{"https://google.com", inputTypeURL, "https://google.com", ""},
+		{"test.com", typeURL, "", ""},
+		{"google.com", typeURL, "https://google.com", ""},
+		{"https://google.com", typeURL, "https://google.com", ""},

 		// file
-		{"google.com:443", inputTypeFilepath, "", ""},
-		{"https://google.com:443", inputTypeFilepath, "", ""},
-		{"/example/path", inputTypeFilepath, "/example/path", ""},
-		{"input_test.go", inputTypeFilepath, "input_test.go", ""},
-		{"../input", inputTypeFilepath, "../input", ""},
-		{"input_test.*", inputTypeFilepath, "input_test.*", ""},
+		{"google.com:443", typeFilepath, "", ""},
+		{"https://google.com:443", typeFilepath, "", ""},
+		{"/example/path", typeFilepath, "/example/path", ""},
+		{"input_test.go", typeFilepath, "input_test.go", ""},
+		{"../input", typeFilepath, "../input", ""},
+		{"input_test.*", typeFilepath, "input_test.*", ""},

 		// host-port
-		{"google.com", inputTypeHostPort, "", ""},
-		{"google.com:443", inputTypeHostPort, "google.com:443", ""},
-		{"https://google.com", inputTypeHostPort, "google.com:443", ""},
-		{"https://google.com:443", inputTypeHostPort, "google.com:443", ""},
+		{"google.com", typeHostWithPort, "", ""},
+		{"google.com:443", typeHostWithPort, "google.com:443", ""},
+		{"https://google.com", typeHostWithPort, "google.com:443", ""},
+		{"https://google.com:443", typeHostWithPort, "google.com:443", ""},
 		// host-port with default port
-		{"google.com", inputTypeHostPort, "google.com:443", "443"},
+		{"google.com", typeHostWithPort, "google.com:443", "443"},

 		// websocket
-		{"google.com", inputTypeWebsocket, "", ""},
-		{"google.com:443", inputTypeWebsocket, "", ""},
-		{"https://google.com:443", inputTypeWebsocket, "", ""},
-		{"wss://google.com", inputTypeWebsocket, "wss://google.com", ""},
+		{"google.com", typeWebsocket, "", ""},
+		{"google.com:443", typeWebsocket, "", ""},
+		{"https://google.com:443", typeWebsocket, "", ""},
+		{"wss://google.com", typeWebsocket, "wss://google.com", ""},
 	}

 	for _, test := range tests {
--- a/v2/pkg/protocols/common/protocolinit/init.go
+++ b/v2/pkg/protocols/common/protocolinit/init.go
@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois/rdapclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )

@ -27,7 +28,13 @@ func Init(options *types.Options) error {
 	if err := signerpool.Init(options); err != nil {
 		return err
 	}
-	return networkclientpool.Init(options)
+	if err := networkclientpool.Init(options); err != nil {
+		return err
+	}
+	if err := rdapclientpool.Init(options); err != nil {
+		return err
+	}
+	return nil
 }

 var userAgents = []string{
--- a/v2/pkg/protocols/whois/rdapclientpool/clientpool.go
+++ b/v2/pkg/protocols/whois/rdapclientpool/clientpool.go
@ -0,0 +1,38 @@
+package rdapclientpool
+
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/rdap"
+)
+
+var normalClient *rdap.Client
+
+// Init initializes the client pool implementation
+func Init(options *types.Options) error {
+	// Don't create clients if already created in the past.
+	if normalClient != nil {
+		return nil
+	}
+
+	normalClient = &rdap.Client{}
+	if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse {
+		normalClient.Verbose = func(text string) {
+			gologger.Debug().Msgf("rdap: %s", text)
+		}
+	}
+	return nil
+}
+
+// Configuration contains the custom configuration options for a client - placeholder
+type Configuration struct{}
+
+// Hash returns the hash of the configuration to allow client pooling - placeholder
+func (c *Configuration) Hash() string {
+	return ""
+}
+
+// Get creates or gets a client for the protocol based on custom configuration
+func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) {
+	return normalClient, nil
+}
--- a/v2/pkg/protocols/whois/whois.go
+++ b/v2/pkg/protocols/whois/whois.go
@ -6,8 +6,8 @@ import (
 	"time"

 	jsoniter "github.com/json-iterator/go"
-	"github.com/openrdap/rdap"
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/rdap"

 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois/rdapclientpool"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
@ -57,12 +58,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	}

 	request.options = options
-	request.client = &rdap.Client{}
-	if request.options.Options.Verbose || request.options.Options.Debug || request.options.Options.DebugRequests {
-		request.client.Verbose = func(text string) {
-			gologger.Debug().Msgf("rdap: %s", text)
-		}
-	}
+	request.client, _ = rdapclientpool.Get(options.Options, nil)

 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators