From 7e933972a4fc1b8737d7e5c6122c430f9382cc9a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 22 Nov 2020 17:07:40 +0700 Subject: [PATCH 01/23] :hammer: Add randomly User-Agent header request --- v2/go.mod | 1 + v2/go.sum | 22 ++++++++++++++++++---- v2/internal/runner/options.go | 2 ++ v2/internal/runner/processor.go | 3 +++ v2/pkg/executer/executer_http.go | 7 +++++++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 1e8218a80..dfcfed3b4 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible + github.com/corpix/uarand v0.1.1 github.com/d5/tengo/v2 v2.6.2 github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 diff --git a/v2/go.sum b/v2/go.sum index e336c0eb0..57159baff 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,27 +1,32 @@ -github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -33,12 +38,16 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -92,6 +101,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -101,14 +111,18 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 197f880e0..27cb69ec0 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -45,6 +45,7 @@ type Options struct { TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests Templates multiStringFlag // Signature specifies the template/templates to use ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude + RandomAgent bool // Generate random User-Agent CustomHeaders requests.CustomHeaders // Custom global headers Threads int // Thread controls the number of concurrent requests to make. BurpCollaboratorBiid string // Burp Collaborator BIID for polling @@ -79,6 +80,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output") flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") + flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value") flag.Var(&options.CustomHeaders, "H", "Custom Header.") flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 7b2c1126f..8b3bd66cf 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -62,6 +62,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, + RandomAgent: r.options.RandomAgent, CustomHeaders: r.options.CustomHeaders, JSON: r.options.JSON, JSONRequests: r.options.JSONRequests, @@ -225,6 +226,7 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, + RandomAgent: r.options.RandomAgent, CustomHeaders: r.options.CustomHeaders, JSON: r.options.JSON, JSONRequests: r.options.JSONRequests, @@ -298,6 +300,7 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, + RandomAgent: r.options.RandomAgent, CustomHeaders: r.options.CustomHeaders, CookieJar: jar, TraceLog: r.traceLog, diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index cd7c05847..8c1764fdb 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" @@ -59,6 +60,7 @@ type HTTPExecuter struct { CookieJar *cookiejar.Jar traceLog tracelog.Log decolorizer *regexp.Regexp + randomAgent bool coloredOutput bool debug bool Results bool @@ -72,6 +74,7 @@ type HTTPExecuter struct { // HTTPOptions contains configuration options for the HTTP executer. type HTTPOptions struct { CustomHeaders requests.CustomHeaders + RandomAgent bool ProxyURL string ProxySocksURL string Template *templates.Template @@ -677,6 +680,10 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect } func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { + if e.randomAgent { + e.customHeaders = append([]string{"User-Agent: " + uarand.GetRandom()}, e.customHeaders...) + } + for _, customHeader := range e.customHeaders { // This should be pre-computed somewhere and done only once tokens := strings.SplitN(customHeader, ":", two) From af50d40c1bb87392bcc41083b5e4fb166c964b5a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 22 Nov 2020 21:59:34 +0700 Subject: [PATCH 02/23] :hammer: Missing HTTPOptions for randomAgent & simplify the logic --- v2/pkg/executer/executer_http.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 8c1764fdb..ee3eebd3a 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -143,6 +143,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { template: options.Template, bulkHTTPRequest: options.BulkHTTPRequest, writer: options.Writer, + randomAgent: options.RandomAgent, customHeaders: options.CustomHeaders, CookieJar: options.CookieJar, coloredOutput: options.ColoredOutput, @@ -388,6 +389,11 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result } func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error { + // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given + if e.randomAgent { + e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) + } + e.setCustomHeaders(request) var ( @@ -680,10 +686,6 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect } func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { - if e.randomAgent { - e.customHeaders = append([]string{"User-Agent: " + uarand.GetRandom()}, e.customHeaders...) - } - for _, customHeader := range e.customHeaders { // This should be pre-computed somewhere and done only once tokens := strings.SplitN(customHeader, ":", two) From d66c52fd36634c927f4c5117269647168fd256a2 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 22 Nov 2020 22:00:39 +0700 Subject: [PATCH 03/23] :blue_book: Adding -random-agent in the flag table --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ec4558b6a..688db240f 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ This will display help for the tool. Here are all the switches it supports. |version |Show version of nuclei |nuclei -version| |proxy-url |Proxy URL |nuclei -proxy-url hxxp://127.0.0.1:8080| |proxy-socks-url |Socks proxyURL |nuclei -proxy-socks-url socks5://127.0.0.1:8080 | +|random-agent |Use randomly selected HTTP User-Agent header value |nuclei -random-agent | |H |Custom Header |nuclei -H "x-bug-bounty: hacker" | ## Installation Instructions From 628865f5322360d5161e62a9a10eef7e12e9945f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 22 Nov 2020 22:05:11 +0700 Subject: [PATCH 04/23] :rotating_light: Suppress golang-ci-lint --- v2/pkg/executer/executer_http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index ee3eebd3a..7bfbccfd8 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -391,6 +391,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if e.randomAgent { + // nolint:errcheck // ignoring error e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) } From a7fe8e317f370d4a5a96c0bcf488e904a171b4ad Mon Sep 17 00:00:00 2001 From: bauthard <8293321+bauthard@users.noreply.github.com> Date: Mon, 23 Nov 2020 11:55:17 +0530 Subject: [PATCH 05/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 688db240f..fc6530497 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ This will display help for the tool. Here are all the switches it supports. |version |Show version of nuclei |nuclei -version| |proxy-url |Proxy URL |nuclei -proxy-url hxxp://127.0.0.1:8080| |proxy-socks-url |Socks proxyURL |nuclei -proxy-socks-url socks5://127.0.0.1:8080 | -|random-agent |Use randomly selected HTTP User-Agent header value |nuclei -random-agent | +|random-agent |Use random User-Agents |nuclei -random-agent | |H |Custom Header |nuclei -H "x-bug-bounty: hacker" | ## Installation Instructions From e5d4c7a6d0214b53733fe7725aeb267316f7b9d7 Mon Sep 17 00:00:00 2001 From: bauthard <8293321+bauthard@users.noreply.github.com> Date: Mon, 23 Nov 2020 11:58:12 +0530 Subject: [PATCH 06/23] dev banner update --- v2/internal/runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index f0e360a5f..88a1de78d 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,11 +7,11 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.2.0 + /_/ /_/\__,_/\___/_/\___/_/ v2.2.1-dev ` // Version is the current version of nuclei -const Version = `2.2.0` +const Version = `2.2.1-dev` // showBanner is used to show the banner to the user func showBanner() { From bb277c70b0f0904c48525c0ab02bfa2abf2d4ef7 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 23 Nov 2020 21:37:52 +0100 Subject: [PATCH 07/23] adding metadata for exact collab match --- v2/go.mod | 2 +- v2/go.sum | 2 ++ v2/pkg/collaborator/collaborator.go | 24 +++++++++++++++--------- v2/pkg/collaborator/util.go | 9 +++++++++ v2/pkg/executer/executer_http.go | 5 +++++ 5 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 v2/pkg/collaborator/util.go diff --git a/v2/go.mod b/v2/go.mod index dfcfed3b4..0884574b1 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,7 +14,7 @@ require ( github.com/miekg/dns v1.1.35 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.5 - github.com/projectdiscovery/collaborator v0.0.1 + github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed github.com/projectdiscovery/fastdialer v0.0.1 github.com/projectdiscovery/gologger v1.0.1 github.com/projectdiscovery/hmap v0.0.1 diff --git a/v2/go.sum b/v2/go.sum index 57159baff..db688e69c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -57,6 +57,8 @@ github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI= github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= +github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed h1:0B4ccP68IL6MNHz99dAFeh0E3AY7fU5wVYwZJVX8ZG4= +github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg= github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= diff --git a/v2/pkg/collaborator/collaborator.go b/v2/pkg/collaborator/collaborator.go index f4d34c9e3..cfa434402 100644 --- a/v2/pkg/collaborator/collaborator.go +++ b/v2/pkg/collaborator/collaborator.go @@ -2,6 +2,7 @@ package collaborator import ( "strings" + "sync" "time" "github.com/projectdiscovery/collaborator" @@ -17,6 +18,7 @@ var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds) var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()} type BurpCollaborator struct { + sync.RWMutex options *Options // unused Collab *collaborator.BurpCollaborator } @@ -41,19 +43,23 @@ func (b *BurpCollaborator) Poll() { } } -func (b *BurpCollaborator) Has(s string) bool { +func (b *BurpCollaborator) Has(s string) (found bool) { + foundAt := 0 for _, r := range b.Collab.RespBuffer { for i := 0; i < len(r.Responses); i++ { - // search in dns - if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) { - return true - } - // search in http - if strings.Contains(r.Responses[i].Data.RequestDecoded, s) { - return true + // search in dns - http - smtp + b.RLock() + found = strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) || strings.Contains(r.Responses[i].Data.RequestDecoded, s) || strings.Contains(r.Responses[i].Data.MessageDecoded, s) + b.RUnlock() + if found { + b.Lock() + r.Responses = removeMatch(r.Responses, foundAt) + b.Unlock() + break } + } } - return false + return } diff --git a/v2/pkg/collaborator/util.go b/v2/pkg/collaborator/util.go new file mode 100644 index 000000000..a6e35675b --- /dev/null +++ b/v2/pkg/collaborator/util.go @@ -0,0 +1,9 @@ +package collaborator + +import ( + "github.com/projectdiscovery/collaborator" +) + +func removeMatch(responses []collaborator.BurpResponse, index int) []collaborator.BurpResponse { + return append(responses[:index], responses[index+1:]...) +} diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 7bfbccfd8..d2ed5d149 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -530,6 +530,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // hardcode stopping storing data after defaultMaxHistorydata items if len(result.historyData) < defaultMaxHistorydata { result.Lock() + // update history data with current reqURL and hostname + result.historyData["reqURL"] = reqURL + if parsed, err := url.Parse(reqURL); err == nil { + result.historyData["Hostname"] = parsed.Host + } result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) result.Unlock() } From c35b79d3f61cef1f2e09777554d21512f5e98b82 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 24 Nov 2020 19:12:14 +0100 Subject: [PATCH 08/23] improving parser regex --- v2/pkg/requests/bulk-http-request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index b90a0836f..476ac57d9 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -202,7 +202,7 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL dynamicValues := make(map[string]interface{}) // find all potentials tokens between {{}} - var re = regexp.MustCompile(`(?m)\{\{.+}}`) + var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) for _, match := range re.FindAllString(raw, -1) { // check if the match contains a dynamic variable expr := generators.TrimDelimiters(match) From 1f6b83f312fe8183df866e2e644430119c3d862e Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 24 Nov 2020 21:06:13 +0100 Subject: [PATCH 09/23] improved raw requests same header handling --- v2/pkg/requests/bulk-http-request.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index b90a0836f..4152bb373 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -400,7 +400,14 @@ func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest, value = p[1] } - rawRequest.Headers[key] = value + // in case of unsafe requests multiple headers should be accepted + // therefore use the full line as key + _, found := rawRequest.Headers[key] + if r.Unsafe && found { + rawRequest.Headers[line] = "" + } else { + rawRequest.Headers[key] = value + } } // Handle case with the full http url in path. In that case, From 3831bfffa5fdc0a19d31ec974005ae73b2b0405a Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 25 Nov 2020 18:27:14 +0100 Subject: [PATCH 10/23] making payloads usable in DSL --- v2/pkg/executer/executer_http.go | 7 +++++++ v2/pkg/matchers/match.go | 2 +- v2/pkg/requests/bulk-http-request.go | 5 +++++ v2/pkg/workflows/var.go | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 7bfbccfd8..8c06f4eab 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -531,6 +531,13 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, if len(result.historyData) < defaultMaxHistorydata { result.Lock() result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) + // retrieve current payloads + currentPayloads := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + if currentPayloads != nil { + // merge them to history data + result.historyData = generators.MergeMaps(result.historyData, currentPayloads) + } + result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) result.Unlock() } diff --git a/v2/pkg/matchers/match.go b/v2/pkg/matchers/match.go index e33262e4b..a2a258409 100644 --- a/v2/pkg/matchers/match.go +++ b/v2/pkg/matchers/match.go @@ -199,7 +199,7 @@ func (m *Matcher) matchBinary(corpus string) bool { // matchDSL matches on a generic map result func (m *Matcher) matchDSL(mp map[string]interface{}) bool { - // Iterate over all the regexes accepted as valid + // Iterate over all the expressions accepted as valid for i, expression := range m.dslCompiled { result, err := expression.Evaluate(mp) if err != nil { diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index b90a0836f..356b6c2ef 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -482,3 +482,8 @@ func (r *BulkHTTPRequest) Total() int { func (r *BulkHTTPRequest) Increment(reqURL string) { r.gsfm.Increment(reqURL) } + +// GetPayloadsValues for the specified URL +func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) map[string]interface{} { + return r.gsfm.Value(reqURL) +} diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 9af658896..02765564b 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -66,7 +66,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount()) for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP { - // apply externally supplied payloads if any + // apply externally supplied headers if any request.Headers = generators.MergeMapsWithStrings(request.Headers, headers) // apply externally supplied payloads if any request.Payloads = generators.MergeMaps(request.Payloads, externalVars) From cc31d6a660a7d745d327094301a05f2f2c16b8a3 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 25 Nov 2020 22:17:57 +0100 Subject: [PATCH 11/23] adding helper functions to payloads --- v2/pkg/executer/executer_http.go | 29 +++++++++++++--------- v2/pkg/requests/bulk-http-request.go | 37 +++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 8c06f4eab..9a1b7b6f1 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -218,11 +218,13 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) // Workers that keeps enqueuing new requests maxWorkers := e.bulkHTTPRequest.Threads swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) && !result.Done { + for e.bulkHTTPRequest.Next(reqURL) { request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) if err != nil { - result.Error = err - p.Drop(remaining) + if err != requests.ErrNoPayload { + result.Error = err + p.Drop(remaining) + } } else { swg.Add() go func(httpRequest *requests.HTTPRequest) { @@ -288,10 +290,13 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { maxWorkers = pipeOptions.MaxPendingRequests } swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) && !result.Done { + for e.bulkHTTPRequest.Next(reqURL) { request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) if err != nil { - result.Error = err + // ignore the error due to the base request having null paylods + if err != requests.ErrNoPayload { + result.Error = err + } } else { swg.Add() go func(httpRequest *requests.HTTPRequest) { @@ -353,12 +358,15 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result remaining := e.bulkHTTPRequest.GetRequestCount() e.bulkHTTPRequest.CreateGenerator(reqURL) - for e.bulkHTTPRequest.Next(reqURL) && !result.Done { + for e.bulkHTTPRequest.Next(reqURL) { requestNumber++ httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) if err != nil { - result.Error = err - p.Drop(remaining) + // ignore the error due to the base request having null paylods + if err != requests.ErrNoPayload { + result.Error = err + p.Drop(remaining) + } } else { e.ratelimiter.Take() // If the request was built correctly then execute it @@ -532,8 +540,8 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, result.Lock() result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) // retrieve current payloads - currentPayloads := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - if currentPayloads != nil { + currentPayloads, err := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + if err == nil { // merge them to history data result.historyData = generators.MergeMaps(result.historyData, currentPayloads) } @@ -718,7 +726,6 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { type Result struct { sync.Mutex GotResults bool - Done bool Meta map[string]interface{} Matches map[string]interface{} Extractions map[string]interface{} diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 356b6c2ef..085df5d81 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -184,7 +184,12 @@ func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, d r.gsfm.InitOrSkip(baseURL) r.ReadOne(baseURL) - return r.handleRawWithPaylods(ctx, data, baseURL, values, r.gsfm.Value(baseURL)) + payloads, err := r.GetPayloadsValues(baseURL) + if err != nil { + return nil, err + } + + return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) } // otherwise continue with normal flow @@ -484,6 +489,32 @@ func (r *BulkHTTPRequest) Increment(reqURL string) { } // GetPayloadsValues for the specified URL -func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) map[string]interface{} { - return r.gsfm.Value(reqURL) +func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface{}, error) { + payloadProcessedValues := make(map[string]interface{}) + payloadsFromTemplate := r.gsfm.Value(reqURL) + for k, v := range payloadsFromTemplate { + // attempts to expand expressions + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(v.(string), generators.HelperFunctions()) + if err != nil { + // it is a simple literal payload => proceed with literal value + payloadProcessedValues[k] = v + continue + } + // it is an expression - try to solve it + expValue, err := compiled.Evaluate(payloadsFromTemplate) + if err != nil { + // an error occurred => proceed with literal value + payloadProcessedValues[k] = v + continue + } + payloadProcessedValues[k] = expValue + } + var err error + if len(payloadProcessedValues) == 0 { + err = ErrNoPayload + } + return payloadProcessedValues, err } + +// ErrNoPayload error to avoid the additional base null request +var ErrNoPayload = fmt.Errorf("No payload found") From 8796b8b628aaf344c921322e949077d2d513b801 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 26 Nov 2020 02:09:26 +0100 Subject: [PATCH 12/23] wip - internal logic --- v2/pkg/executer/executer_http.go | 81 ++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 9a1b7b6f1..ac62254ec 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -178,16 +178,21 @@ func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result { for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ { swg.Add() // base request + result.Lock() request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { result.Error = err - return result } go func(httpRequest *requests.HTTPRequest) { defer swg.Done() // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "") + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") if err != nil { result.Error = errors.Wrap(err, "could not handle http request") } @@ -219,12 +224,16 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) maxWorkers := e.bulkHTTPRequest.Threads swg := sizedwaitgroup.New(maxWorkers) for e.bulkHTTPRequest.Next(reqURL) { + result.Lock() request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { - if err != requests.ErrNoPayload { - result.Error = err - p.Drop(remaining) - } + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err + p.Drop(remaining) } else { swg.Add() go func(httpRequest *requests.HTTPRequest) { @@ -233,7 +242,7 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) e.ratelimiter.Take() // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "") + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") if err != nil { e.traceLog.Request(e.template.ID, reqURL, "http", err) result.Error = errors.Wrap(err, "could not handle http request") @@ -291,12 +300,15 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { } swg := sizedwaitgroup.New(maxWorkers) for e.bulkHTTPRequest.Next(reqURL) { + result.Lock() request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { - // ignore the error due to the base request having null paylods - if err != requests.ErrNoPayload { - result.Error = err - } + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err } else { swg.Add() go func(httpRequest *requests.HTTPRequest) { @@ -306,7 +318,7 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { // If the request was built correctly then execute it request.Pipeline = true request.PipelineClient = pipeclient - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "") + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") if err != nil { e.traceLog.Request(e.template.ID, reqURL, "http", err) result.Error = errors.Wrap(err, "could not handle http request") @@ -360,18 +372,21 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result for e.bulkHTTPRequest.Next(reqURL) { requestNumber++ + result.Lock() httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { - // ignore the error due to the base request having null paylods - if err != requests.ErrNoPayload { - result.Error = err - p.Drop(remaining) - } + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err + p.Drop(remaining) } else { e.ratelimiter.Take() // If the request was built correctly then execute it format := "%s_" + strconv.Itoa(requestNumber) - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format) + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format) if err != nil { result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) @@ -396,7 +411,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result return result } -func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error { +func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if e.randomAgent { // nolint:errcheck // ignoring error @@ -534,25 +549,31 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, headers := headersToString(resp.Header) + var matchData map[string]interface{} + if payloads != nil { + matchData = generators.MergeMaps(result.historyData, payloads) + } + // store for internal purposes the DSL matcher data // hardcode stopping storing data after defaultMaxHistorydata items if len(result.historyData) < defaultMaxHistorydata { result.Lock() result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) - // retrieve current payloads - currentPayloads, err := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - if err == nil { + if payloads == nil { // merge them to history data - result.historyData = generators.MergeMaps(result.historyData, currentPayloads) + result.historyData = generators.MergeMaps(result.historyData, payloads) } result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) + + // complement match data with new one if necessary + matchData = generators.MergeMaps(matchData, result.historyData) result.Unlock() } matcherCondition := e.bulkHTTPRequest.GetMatchersCondition() for _, matcher := range e.bulkHTTPRequest.Matchers { // Check if the matcher matched - if !matcher.Match(resp, body, headers, duration, result.historyData) { + if !matcher.Match(resp, body, headers, duration, matchData) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { return nil @@ -567,7 +588,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, result.Meta = request.Meta result.GotResults = true result.Unlock() - e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL) + e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL) } } } @@ -598,7 +619,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL) + e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL) result.Lock() result.GotResults = true result.Unlock() From 4390a59c486cdf470f49467d8fbe9487814babf4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 26 Nov 2020 09:57:26 +0100 Subject: [PATCH 13/23] generic interface => string conversion --- v2/pkg/generators/dsl.go | 74 +++++++++++++++++++-------------------- v2/pkg/generators/util.go | 4 +++ 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/v2/pkg/generators/dsl.go b/v2/pkg/generators/dsl.go index c8b6792bc..5899ff652 100644 --- a/v2/pkg/generators/dsl.go +++ b/v2/pkg/generators/dsl.go @@ -35,112 +35,112 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { // strings functions["len"] = func(args ...interface{}) (interface{}, error) { - length := len(args[0].(string)) + length := len(toString(args[0])) return float64(length), nil } functions["toupper"] = func(args ...interface{}) (interface{}, error) { - return strings.ToUpper(args[0].(string)), nil + return strings.ToUpper(toString(args[0])), nil } functions["tolower"] = func(args ...interface{}) (interface{}, error) { - return strings.ToLower(args[0].(string)), nil + return strings.ToLower(toString(args[0])), nil } functions["replace"] = func(args ...interface{}) (interface{}, error) { - return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil + return strings.ReplaceAll(toString(args[0]), toString(args[1]), toString(args[2])), nil } functions["replace_regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(args[1].(string)) + compiled, err := regexp.Compile(toString(args[1])) if err != nil { return nil, err } - return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil + return compiled.ReplaceAllString(toString(args[0]), toString(args[2])), nil } functions["trim"] = func(args ...interface{}) (interface{}, error) { - return strings.Trim(args[0].(string), args[2].(string)), nil + return strings.Trim(toString(args[0]), toString(args[2])), nil } functions["trimleft"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimLeft(args[0].(string), args[1].(string)), nil + return strings.TrimLeft(toString(args[0]), toString(args[1])), nil } functions["trimright"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimRight(args[0].(string), args[1].(string)), nil + return strings.TrimRight(toString(args[0]), toString(args[1])), nil } functions["trimspace"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSpace(args[0].(string)), nil + return strings.TrimSpace(toString(args[0])), nil } functions["trimprefix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimPrefix(args[0].(string), args[1].(string)), nil + return strings.TrimPrefix(toString(args[0]), toString(args[1])), nil } functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSuffix(args[0].(string), args[1].(string)), nil + return strings.TrimSuffix(toString(args[0]), toString(args[1])), nil } functions["reverse"] = func(args ...interface{}) (interface{}, error) { - return reverseString(args[0].(string)), nil + return reverseString(toString(args[0])), nil } // encoding functions["base64"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string))) + sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0]))) return sEnc, nil } // python encodes to base64 with lines of 76 bytes terminated by new line "\n" functions["base64_py"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string))) + sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0]))) return insertInto(sEnc, 76, '\n'), nil } functions["base64_decode"] = func(args ...interface{}) (interface{}, error) { - return base64.StdEncoding.DecodeString(args[0].(string)) + return base64.StdEncoding.DecodeString(toString(args[0])) } functions["url_encode"] = func(args ...interface{}) (interface{}, error) { - return url.PathEscape(args[0].(string)), nil + return url.PathEscape(toString(args[0])), nil } functions["url_decode"] = func(args ...interface{}) (interface{}, error) { - return url.PathUnescape(args[0].(string)) + return url.PathUnescape(toString(args[0])) } functions["hex_encode"] = func(args ...interface{}) (interface{}, error) { - return hex.EncodeToString([]byte(args[0].(string))), nil + return hex.EncodeToString([]byte(toString(args[0]))), nil } functions["hex_decode"] = func(args ...interface{}) (interface{}, error) { - hx, _ := hex.DecodeString(args[0].(string)) + hx, _ := hex.DecodeString(toString(args[0])) return string(hx), nil } functions["html_escape"] = func(args ...interface{}) (interface{}, error) { - return html.EscapeString(args[0].(string)), nil + return html.EscapeString(toString(args[0])), nil } functions["html_unescape"] = func(args ...interface{}) (interface{}, error) { - return html.UnescapeString(args[0].(string)), nil + return html.UnescapeString(toString(args[0])), nil } // hashing functions["md5"] = func(args ...interface{}) (interface{}, error) { - hash := md5.Sum([]byte(args[0].(string))) + hash := md5.Sum([]byte(toString(args[0]))) return hex.EncodeToString(hash[:]), nil } functions["sha256"] = func(args ...interface{}) (interface{}, error) { h := sha256.New() - _, err := h.Write([]byte(args[0].(string))) + _, err := h.Write([]byte(toString(args[0]))) if err != nil { return nil, err @@ -151,7 +151,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { functions["sha1"] = func(args ...interface{}) (interface{}, error) { h := sha1.New() - _, err := h.Write([]byte(args[0].(string))) + _, err := h.Write([]byte(toString(args[0]))) if err != nil { return nil, err @@ -161,21 +161,21 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { } functions["mmh3"] = func(args ...interface{}) (interface{}, error) { - return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil + return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(toString(args[0])), 0))), nil } // search functions["contains"] = func(args ...interface{}) (interface{}, error) { - return strings.Contains(args[0].(string), args[1].(string)), nil + return strings.Contains(toString(args[0]), toString(args[1])), nil } functions["regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(args[0].(string)) + compiled, err := regexp.Compile(toString(args[0])) if err != nil { return nil, err } - return compiled.MatchString(args[1].(string)), nil + return compiled.MatchString(toString(args[1])), nil } // random generators @@ -183,10 +183,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { chars := letters + numbers bad := "" if len(args) >= 1 { - chars = args[0].(string) + chars = toString(args[0]) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -203,10 +203,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } if len(args) >= withBaseRandArgsSize { - base = args[2].(string) + base = toString(args[2]) } base = TrimAll(base, bad) @@ -223,7 +223,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -240,7 +240,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -257,7 +257,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -289,7 +289,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { // Collaborator functions["collab"] = func(args ...interface{}) (interface{}, error) { // check if collaborator contains a specific pattern - return collaborator.DefaultCollaborator.Has(args[0].(string)), nil + return collaborator.DefaultCollaborator.Has(toString(args[0])), nil } return functions diff --git a/v2/pkg/generators/util.go b/v2/pkg/generators/util.go index 3446c1a2a..b7761a546 100644 --- a/v2/pkg/generators/util.go +++ b/v2/pkg/generators/util.go @@ -199,3 +199,7 @@ func insertInto(s string, interval int, sep rune) string { buffer.WriteRune(sep) return buffer.String() } + +func toString(v interface{}) string { + return fmt.Sprint(v) +} From c154a738eae1828f3d2fb5bb9fc8af39fc36bedf Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 26 Nov 2020 10:19:27 +0100 Subject: [PATCH 14/23] output interface => string --- v2/pkg/executer/output_http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go index c40ead5a5..7ce421136 100644 --- a/v2/pkg/executer/output_http.go +++ b/v2/pkg/executer/output_http.go @@ -1,6 +1,7 @@ package executer import ( + "fmt" "net/http" "net/http/httputil" "strings" @@ -119,9 +120,8 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res builder.WriteString(" [") var metas []string - for name, value := range req.Meta { - metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String()) + metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String()) } builder.WriteString(strings.Join(metas, ",")) From 99f71a4f3518c048d74d55f1ad11ec1d501019c7 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 26 Nov 2020 11:00:25 +0100 Subject: [PATCH 15/23] fixing casting errors --- v2/pkg/requests/bulk-http-request.go | 10 ++++++++-- v2/pkg/requests/util.go | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 085df5d81..c0dcbfc47 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -493,8 +493,14 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface payloadProcessedValues := make(map[string]interface{}) payloadsFromTemplate := r.gsfm.Value(reqURL) for k, v := range payloadsFromTemplate { + kexp := v.(string) + // if it doesn't containg markups, we just continue + if !hasMarker(kexp) { + payloadProcessedValues[k] = v + continue + } // attempts to expand expressions - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(v.(string), generators.HelperFunctions()) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions()) if err != nil { // it is a simple literal payload => proceed with literal value payloadProcessedValues[k] = v @@ -507,7 +513,7 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface payloadProcessedValues[k] = v continue } - payloadProcessedValues[k] = expValue + payloadProcessedValues[k] = fmt.Sprint(expValue) } var err error if len(payloadProcessedValues) == 0 { diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go index bdd633a83..ce8ba795b 100644 --- a/v2/pkg/requests/util.go +++ b/v2/pkg/requests/util.go @@ -71,3 +71,7 @@ func ExpandMapValues(m map[string]string) (m1 map[string][]string) { } return } + +func hasMarker(s string) bool { + return strings.Contains(s, markerParenthesisOpen) || strings.Contains(s, markerParenthesisClose) || strings.Contains(s, markerGeneral) +} From eaf504bc2281c897095cf27dc9ceabbbe06729d5 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 26 Nov 2020 11:39:02 +0100 Subject: [PATCH 16/23] updating fast dialer (support for hosts file) --- v2/go.mod | 5 ++--- v2/go.sum | 27 ++++++++++----------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index dfcfed3b4..8bcdcc731 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -15,7 +15,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.5 github.com/projectdiscovery/collaborator v0.0.1 - github.com/projectdiscovery/fastdialer v0.0.1 + github.com/projectdiscovery/fastdialer v0.0.2 github.com/projectdiscovery/gologger v1.0.1 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/rawhttp v0.0.4 @@ -24,8 +24,7 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/segmentio/ksuid v1.0.3 github.com/spaolacci/murmur3 v1.1.0 - github.com/stretchr/testify v1.5.1 go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/v2/go.sum b/v2/go.sum index 57159baff..f9d275828 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,32 +1,34 @@ +github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= +github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -45,20 +47,17 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0= github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI= github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= -github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg= -github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc= +github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc= +github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= @@ -77,7 +76,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -101,7 +99,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -111,20 +108,16 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 96049a32fa70aef0483d4180d386a57b3c2d973a Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 13 Dec 2020 13:31:12 +0530 Subject: [PATCH 17/23] Sandboxed workflows to only run safe commands --- v2/internal/runner/processor.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 8b3bd66cf..2c425e360 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -143,10 +143,19 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo defer wg.Done() script := tengo.NewScript(logicBytes) - script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) + var moduleNames = []string{ + "math", + "text", + "rand", + "fmt", + "json", + "base64", + "hex", + "enum", + } + script.SetImports(stdlib.GetModuleMap(moduleNames...)) variables := make(map[string]*workflows.NucleiVar) - for _, workflowTemplate := range *workflowTemplatesList { name := workflowTemplate.Name variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL} From cf2a98b03d76327c20fb0183b7f1c849eb692c91 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 13 Dec 2020 14:04:58 +0530 Subject: [PATCH 18/23] Added flag to control workflow sandboxing --- v2/internal/runner/options.go | 2 ++ v2/internal/runner/processor.go | 18 +++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 27cb69ec0..85b6d35c0 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -14,6 +14,7 @@ import ( // the template requesting process. // nolint // false positive, options are allocated once and are necessary as is type Options struct { + Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled Debug bool // Debug mode allows debugging request/responses for the engine Silent bool // Silent suppresses any extra text and only writes found URLs on screen. Version bool // Version specifies if we should just show version and exit @@ -66,6 +67,7 @@ func (m *multiStringFlag) Set(value string) error { func ParseOptions() *Options { options := &Options{} + flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.") diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 2c425e360..9bfd30ae3 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -28,6 +28,8 @@ type workflowTemplates struct { Templates []*workflows.Template } +var sandboxedModules = []string{"math", "text", "rand", "fmt", "json", "base64", "hex", "enum"} + // processTemplateWithList processes a template and runs the enumeration on all the targets func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { var httpExecuter *executer.HTTPExecuter @@ -128,7 +130,7 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow) if err != nil { gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err) - return result + return false } logicBytes := []byte(workflow.Logic) @@ -143,17 +145,11 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo defer wg.Done() script := tengo.NewScript(logicBytes) - var moduleNames = []string{ - "math", - "text", - "rand", - "fmt", - "json", - "base64", - "hex", - "enum", + if !r.options.Sandbox { + script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) + } else { + script.SetImports(stdlib.GetModuleMap(sandboxedModules...)) } - script.SetImports(stdlib.GetModuleMap(moduleNames...)) variables := make(map[string]*workflows.NucleiVar) for _, workflowTemplate := range *workflowTemplatesList { From 2ae13d845ab29cc72e0e76533534e5bb6fb62a54 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 13 Dec 2020 14:17:58 +0530 Subject: [PATCH 19/23] Max limit on execution of a workflow --- v2/internal/runner/options.go | 4 +++- v2/internal/runner/processor.go | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 85b6d35c0..8dfa7f190 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -14,7 +14,8 @@ import ( // the template requesting process. // nolint // false positive, options are allocated once and are necessary as is type Options struct { - Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled + MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL + Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled Debug bool // Debug mode allows debugging request/responses for the engine Silent bool // Silent suppresses any extra text and only writes found URLs on screen. Version bool // Version specifies if we should just show version and exit @@ -68,6 +69,7 @@ func ParseOptions() *Options { options := &Options{} flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") + flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes") flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.") diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 9bfd30ae3..2cd288613 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -8,6 +8,7 @@ import ( "path" "path/filepath" "strings" + "time" tengo "github.com/d5/tengo/v2" "github.com/d5/tengo/v2/stdlib" @@ -132,11 +133,9 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err) return false } - logicBytes := []byte(workflow.Logic) wg := sizedwaitgroup.New(r.options.BulkSize) - r.hm.Scan(func(k, _ []byte) error { targetURL := string(k) wg.Add() @@ -163,7 +162,10 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo variables[name] = variable } - _, err := script.RunContext(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.options.MaxWorkflowDuration)*time.Minute) + defer cancel() + + _, err := script.RunContext(ctx) if err != nil { gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err) } From 4fa66bd46c000fbbbab4408b8a6ded0e7c15b98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Zamanillo?= Date: Sun, 13 Dec 2020 22:07:00 +0100 Subject: [PATCH 20/23] bump golangci-lint to 1.33 --- .github/workflows/build.yaml | 4 ++-- .golangci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ead969e43..5e6435e01 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v1 + uses: golangci/golangci-lint-action@v2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.31 + version: v1.33 args: --timeout 5m working-directory: v2/ diff --git a/.golangci.yml b/.golangci.yml index 917ad0fec..694cea254 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -110,6 +110,6 @@ linters: # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly prepare: - echo "here I can run custom commands, but no preparation needed for this repo" From 9082769d6789c265e2d9bfe584a3835db8a298d2 Mon Sep 17 00:00:00 2001 From: Shouichi Kamiya Date: Mon, 14 Dec 2020 12:56:58 +0900 Subject: [PATCH 21/23] Remove space from -severity cli flag in README CLI's -severity flag is space sensitive. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 609e58052..f887c1792 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ Remember to change `/path-to-nuclei-templates` to the real path on your host fil You can run the templates based on the specific severity of the template, single and multiple severity can be used for scan. ```sh -nuclei -l urls.txt -t cves/ -severity critical, medium +nuclei -l urls.txt -t cves/ -severity critical,medium ``` The above example will run all the templates under `cves` directory with `critical` and `medium` severity. From 15708cb9411f3519b6b927d2243a56d077849cea Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 17 Dec 2020 20:33:42 +0530 Subject: [PATCH 22/23] Added simple json based http metrics support --- v2/internal/progress/progress.go | 102 ++++++++++++++++++++++--------- v2/internal/runner/options.go | 4 ++ v2/internal/runner/runner.go | 6 +- 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 6c12be52f..112f5c4e1 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -1,8 +1,13 @@ package progress import ( + "context" + "encoding/json" "fmt" + "net" + "net/http" "os" + "strconv" "strings" "time" @@ -13,12 +18,13 @@ import ( // Progress is a progress instance for showing program stats type Progress struct { active bool - stats clistats.StatisticsClient tickDuration time.Duration + stats clistats.StatisticsClient + server *http.Server } // NewProgress creates and returns a new progress tracking object. -func NewProgress(active bool) *Progress { +func NewProgress(active, metrics bool, port int) (*Progress, error) { var tickDuration time.Duration if active { tickDuration = 5 * time.Second @@ -26,29 +32,44 @@ func NewProgress(active bool) *Progress { tickDuration = -1 } - var progress Progress - if active { - stats, err := clistats.New() - if err != nil { - gologger.Warningf("Couldn't create progress engine: %s\n", err) - } - progress.active = active - progress.stats = stats - progress.tickDuration = tickDuration - } + progress := &Progress{} - return &progress + stats, err := clistats.New() + if err != nil { + return nil, err + } + progress.active = active + progress.stats = stats + progress.tickDuration = tickDuration + + if metrics { + http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { + metrics := progress.getMetrics() + _ = json.NewEncoder(w).Encode(metrics) + }) + progress.server = &http.Server{ + Addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), + Handler: http.DefaultServeMux, + } + go func() { + if err := progress.server.ListenAndServe(); err != nil { + gologger.Warningf("Could not serve metrics: %s\n", err) + } + }() + } + return progress, nil } // Init initializes the progress display mechanism by setting counters, etc. func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { + p.stats.AddStatic("templates", rulesCount) + p.stats.AddStatic("hosts", hostCount) + p.stats.AddStatic("startedAt", time.Now()) + p.stats.AddCounter("requests", uint64(0)) + p.stats.AddCounter("errors", uint64(0)) + p.stats.AddCounter("total", uint64(requestCount)) + if p.active { - p.stats.AddStatic("templates", rulesCount) - p.stats.AddStatic("hosts", hostCount) - p.stats.AddStatic("startedAt", time.Now()) - p.stats.AddCounter("requests", uint64(0)) - p.stats.AddCounter("errors", uint64(0)) - p.stats.AddCounter("total", uint64(requestCount)) if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil { gologger.Warningf("Couldn't start statistics: %s\n", err) } @@ -57,25 +78,19 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { // AddToTotal adds a value to the total request count func (p *Progress) AddToTotal(delta int64) { - if p.active { - p.stats.IncrementCounter("total", int(delta)) - } + p.stats.IncrementCounter("total", int(delta)) } // Update progress tracking information and increments the request counter by one unit. func (p *Progress) Update() { - if p.active { - p.stats.IncrementCounter("requests", 1) - } + p.stats.IncrementCounter("requests", 1) } // Drop drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. func (p *Progress) Drop(count int64) { - if p.active { - // mimic dropping by incrementing the completed requests - p.stats.IncrementCounter("errors", int(count)) - } + // mimic dropping by incrementing the completed requests + p.stats.IncrementCounter("errors", int(count)) } const bufferSize = 128 @@ -125,6 +140,34 @@ func makePrintCallback() func(stats clistats.StatisticsClient) { } } +// getMetrics returns a map of important metrics for client +func (p *Progress) getMetrics() map[string]interface{} { + results := make(map[string]interface{}) + + startedAt, _ := p.stats.GetStatic("startedAt") + duration := time.Since(startedAt.(time.Time)) + + results["startedAt"] = startedAt.(time.Time) + results["duration"] = fmtDuration(duration) + templates, _ := p.stats.GetStatic("templates") + results["templates"] = clistats.String(templates) + hosts, _ := p.stats.GetStatic("hosts") + results["hosts"] = clistats.String(hosts) + requests, _ := p.stats.GetCounter("requests") + results["requests"] = clistats.String(requests) + total, _ := p.stats.GetCounter("total") + results["total"] = clistats.String(total) + results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds())) + errors, _ := p.stats.GetCounter("errors") + results["errors"] = clistats.String(errors) + + //nolint:gomnd // this is not a magic number + percentData := (float64(requests) * float64(100)) / float64(total) + percent := clistats.String(uint64(percentData)) + results["percent"] = percent + return results +} + // fmtDuration formats the duration for the time elapsed func fmtDuration(d time.Duration) string { d = d.Round(time.Second) @@ -143,4 +186,5 @@ func (p *Progress) Stop() { gologger.Warningf("Couldn't stop statistics: %s\n", err) } } + p.server.Shutdown(context.Background()) } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 8dfa7f190..55ca127dc 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -15,6 +15,8 @@ import ( // nolint // false positive, options are allocated once and are necessary as is type Options struct { MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL + Metrics bool // Metrics enables display of metrics via an http endpoint + MetricsPort int // MetricsPort is the port to show metrics on Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled Debug bool // Debug mode allows debugging request/responses for the engine Silent bool // Silent suppresses any extra text and only writes found URLs on screen. @@ -69,6 +71,8 @@ func ParseOptions() *Options { options := &Options{} flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") + flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") + flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes") flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 46c7d4e88..79ded4bf0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -173,7 +173,11 @@ func New(options *Options) (*Runner, error) { } // Creates the progress tracking object - runner.progress = progress.NewProgress(options.EnableProgressBar) + var progressErr error + runner.progress, progressErr = progress.NewProgress(options.EnableProgressBar, options.Metrics, options.MetricsPort) + if progressErr != nil { + return nil, progressErr + } // create project file if requested or load existing one if options.Project { From bc14c2db0002dedc048320c64b8f284b271b6819 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 17 Dec 2020 23:39:38 +0530 Subject: [PATCH 23/23] Linting issues fix --- v2/go.mod | 1 + v2/go.sum | 3 +++ v2/internal/progress/progress.go | 2 +- v2/internal/runner/options.go | 15 +++++++-------- v2/pkg/collaborator/collaborator.go | 1 - v2/pkg/executer/executer_http.go | 24 ++++++++++++------------ v2/pkg/requests/bulk-http-request.go | 4 ++-- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index cb2bc84cd..1f63ac369 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -24,6 +24,7 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/segmentio/ksuid v1.0.3 github.com/spaolacci/murmur3 v1.1.0 + github.com/stretchr/testify v1.5.1 go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b gopkg.in/yaml.v2 v2.4.0 diff --git a/v2/go.sum b/v2/go.sum index e33e59472..ac2df7df3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -14,6 +14,7 @@ github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= @@ -51,6 +52,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0= github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= @@ -78,6 +80,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 112f5c4e1..a9b5c2ead 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -186,5 +186,5 @@ func (p *Progress) Stop() { gologger.Warningf("Couldn't stop statistics: %s\n", err) } } - p.server.Shutdown(context.Background()) + _ = p.server.Shutdown(context.Background()) } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 55ca127dc..b00bb485f 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -12,11 +12,9 @@ import ( // Options contains the configuration options for tuning // the template requesting process. -// nolint // false positive, options are allocated once and are necessary as is type Options struct { - MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL + RandomAgent bool // Generate random User-Agent Metrics bool // Metrics enables display of metrics via an http endpoint - MetricsPort int // MetricsPort is the port to show metrics on Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled Debug bool // Debug mode allows debugging request/responses for the engine Silent bool // Silent suppresses any extra text and only writes found URLs on screen. @@ -32,13 +30,17 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests) NoMeta bool // Don't display metadata for the matches + Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times + MetricsPort int // MetricsPort is the port to show metrics on + MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL BulkSize int // Number of targets analyzed in parallel for each template TemplateThreads int // Number of templates executed in parallel - Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times - ProjectPath string // Nuclei uses a user defined project folder Timeout int // Timeout is the seconds to wait for a response from the server. Retries int // Retries is the number of times to retry the request RateLimit int // Rate-Limit of requests per specified target + Threads int // Thread controls the number of concurrent requests to make. + BurpCollaboratorBiid string // Burp Collaborator BIID for polling + ProjectPath string // Nuclei uses a user defined project folder Severity string // Filter templates based on their severity and only run the matching ones. Target string // Target is a single URL/Domain to scan usng a template Targets string // Targets specifies the targets to scan using templates. @@ -49,10 +51,7 @@ type Options struct { TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests Templates multiStringFlag // Signature specifies the template/templates to use ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude - RandomAgent bool // Generate random User-Agent CustomHeaders requests.CustomHeaders // Custom global headers - Threads int // Thread controls the number of concurrent requests to make. - BurpCollaboratorBiid string // Burp Collaborator BIID for polling } type multiStringFlag []string diff --git a/v2/pkg/collaborator/collaborator.go b/v2/pkg/collaborator/collaborator.go index cfa434402..4b6383b23 100644 --- a/v2/pkg/collaborator/collaborator.go +++ b/v2/pkg/collaborator/collaborator.go @@ -57,7 +57,6 @@ func (b *BurpCollaborator) Has(s string) (found bool) { b.Unlock() break } - } } diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 2e1475a6f..c8b58b403 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -73,19 +73,7 @@ type HTTPExecuter struct { // HTTPOptions contains configuration options for the HTTP executer. type HTTPOptions struct { - CustomHeaders requests.CustomHeaders RandomAgent bool - ProxyURL string - ProxySocksURL string - Template *templates.Template - BulkHTTPRequest *requests.BulkHTTPRequest - Writer *bufwriter.Writer - Timeout int - Retries int - CookieJar *cookiejar.Jar - Colorizer *colorizer.NucleiColorizer - Decolorizer *regexp.Regexp - TraceLog tracelog.Log Debug bool JSON bool JSONRequests bool @@ -93,6 +81,18 @@ type HTTPOptions struct { CookieReuse bool ColoredOutput bool StopAtFirstMatch bool + Timeout int + Retries int + ProxyURL string + ProxySocksURL string + Template *templates.Template + BulkHTTPRequest *requests.BulkHTTPRequest + Writer *bufwriter.Writer + CustomHeaders requests.CustomHeaders + CookieJar *cookiejar.Jar + Colorizer *colorizer.NucleiColorizer + Decolorizer *regexp.Regexp + TraceLog tracelog.Log PF *projetctfile.ProjectFile RateLimiter ratelimit.Limiter Dialer *fastdialer.Dialer diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index fac066036..907e62418 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -501,7 +501,7 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface payloadsFromTemplate := r.gsfm.Value(reqURL) for k, v := range payloadsFromTemplate { kexp := v.(string) - // if it doesn't containg markups, we just continue + // if it doesn't containing markups, we just continue if !hasMarker(kexp) { payloadProcessedValues[k] = v continue @@ -530,4 +530,4 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface } // ErrNoPayload error to avoid the additional base null request -var ErrNoPayload = fmt.Errorf("No payload found") +var ErrNoPayload = fmt.Errorf("no payload found")