merging dev branch

v2/go.mod

@@ -14,8 +14,8 @@ require (
 	github.com/miekg/dns v1.1.35
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.5
-	github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed
+	github.com/projectdiscovery/collaborator v0.0.2
-	github.com/projectdiscovery/fastdialer v0.0.1
+	github.com/projectdiscovery/fastdialer v0.0.2
 	github.com/projectdiscovery/gologger v1.0.1
 	github.com/projectdiscovery/hmap v0.0.1
 	github.com/projectdiscovery/rawhttp v0.0.4
@@ -24,8 +24,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/segmentio/ksuid v1.0.3
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/stretchr/testify v1.5.1
 	go.uber.org/ratelimit v0.1.0
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -1,32 +1,34 @@
+github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
+github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
 github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
 github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
 github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
 github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
 github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -45,22 +47,19 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
 github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
 github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
 github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed h1:0B4ccP68IL6MNHz99dAFeh0E3AY7fU5wVYwZJVX8ZG4=
+github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
-github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
+github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
+github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc=
-github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
+github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
 github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
@@ -79,7 +78,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@@ -103,7 +101,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -113,20 +110,16 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

v2/pkg/executer/executer_http.go

@@ -178,16 +178,21 @@ func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
 	for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
 		swg.Add()
 		// base request
+		result.Lock()
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
-			return result
 		}
 		go func(httpRequest *requests.HTTPRequest) {
 			defer swg.Done()
 
 			// If the request was built correctly then execute it
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
 			if err != nil {
 				result.Error = errors.Wrap(err, "could not handle http request")
 			}
@@ -218,9 +223,15 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
 	// Workers that keeps enqueuing new requests
 	maxWorkers := e.bulkHTTPRequest.Threads
 	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+	for e.bulkHTTPRequest.Next(reqURL) {
+		result.Lock()
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
 			p.Drop(remaining)
 		} else {
@@ -231,7 +242,7 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
 				e.ratelimiter.Take()
 
 				// If the request was built correctly then execute it
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
 				if err != nil {
 					e.traceLog.Request(e.template.ID, reqURL, "http", err)
 					result.Error = errors.Wrap(err, "could not handle http request")
@@ -288,9 +299,15 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
 		maxWorkers = pipeOptions.MaxPendingRequests
 	}
 	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+	for e.bulkHTTPRequest.Next(reqURL) {
+		result.Lock()
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
 		} else {
 			swg.Add()
@@ -301,7 +318,7 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
 				// If the request was built correctly then execute it
 				request.Pipeline = true
 				request.PipelineClient = pipeclient
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
 				if err != nil {
 					e.traceLog.Request(e.template.ID, reqURL, "http", err)
 					result.Error = errors.Wrap(err, "could not handle http request")
@@ -353,17 +370,23 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
 	remaining := e.bulkHTTPRequest.GetRequestCount()
 	e.bulkHTTPRequest.CreateGenerator(reqURL)
 
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+	for e.bulkHTTPRequest.Next(reqURL) {
 		requestNumber++
+		result.Lock()
 		httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
 			p.Drop(remaining)
 		} else {
 			e.ratelimiter.Take()
 			// If the request was built correctly then execute it
 			format := "%s_" + strconv.Itoa(requestNumber)
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)
+			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format)
 			if err != nil {
 				result.Error = errors.Wrap(err, "could not handle http request")
 				p.Drop(remaining)
@@ -388,7 +411,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
 	return result
 }
 
-func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error {
+func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error {
 	// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
 	if e.randomAgent {
 		// nolint:errcheck // ignoring error
@@ -526,6 +549,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 
 	headers := headersToString(resp.Header)
 
+	var matchData map[string]interface{}
+	if payloads != nil {
+		matchData = generators.MergeMaps(result.historyData, payloads)
+	}
+
 	// store for internal purposes the DSL matcher data
 	// hardcode stopping storing data after defaultMaxHistorydata items
 	if len(result.historyData) < defaultMaxHistorydata {
@@ -536,13 +564,21 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 			result.historyData["Hostname"] = parsed.Host
 		}
 		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
+		if payloads == nil {
+			// merge them to history data
+			result.historyData = generators.MergeMaps(result.historyData, payloads)
+		}
+		result.historyData = generators.MergeMaps(result.historyData, dynamicvalues)
+
+		// complement match data with new one if necessary
+		matchData = generators.MergeMaps(matchData, result.historyData)
 		result.Unlock()
 	}
 
 	matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
 	for _, matcher := range e.bulkHTTPRequest.Matchers {
 		// Check if the matcher matched
-		if !matcher.Match(resp, body, headers, duration, result.historyData) {
+		if !matcher.Match(resp, body, headers, duration, matchData) {
 			// If the condition is AND we haven't matched, try next request.
 			if matcherCondition == matchers.ANDCondition {
 				return nil
@@ -557,7 +593,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 				result.Meta = request.Meta
 				result.GotResults = true
 				result.Unlock()
-				e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL)
+				e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL)
 			}
 		}
 	}
@@ -588,7 +624,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 	// Write a final string of output if matcher type is
 	// AND or if we have extractors for the mechanism too.
 	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL)
+		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL)
 		result.Lock()
 		result.GotResults = true
 		result.Unlock()
@@ -716,7 +752,6 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
 type Result struct {
 	sync.Mutex
 	GotResults  bool
-	Done        bool
 	Meta        map[string]interface{}
 	Matches     map[string]interface{}
 	Extractions map[string]interface{}

v2/pkg/executer/output_http.go

@@ -1,6 +1,7 @@
 package executer
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httputil"
 	"strings"
@@ -119,9 +120,8 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
 		builder.WriteString(" [")
 
 		var metas []string
-
 		for name, value := range req.Meta {
-			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String())
+			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String())
 		}
 
 		builder.WriteString(strings.Join(metas, ","))

v2/pkg/generators/dsl.go

@@ -35,112 +35,112 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 
 	// strings
 	functions["len"] = func(args ...interface{}) (interface{}, error) {
-		length := len(args[0].(string))
+		length := len(toString(args[0]))
 
 		return float64(length), nil
 	}
 
 	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToUpper(args[0].(string)), nil
+		return strings.ToUpper(toString(args[0])), nil
 	}
 
 	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToLower(args[0].(string)), nil
+		return strings.ToLower(toString(args[0])), nil
 	}
 
 	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil
+		return strings.ReplaceAll(toString(args[0]), toString(args[1]), toString(args[2])), nil
 	}
 
 	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[1].(string))
+		compiled, err := regexp.Compile(toString(args[1]))
 		if err != nil {
 			return nil, err
 		}
-		return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil
+		return compiled.ReplaceAllString(toString(args[0]), toString(args[2])), nil
 	}
 
 	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(args[0].(string), args[2].(string)), nil
+		return strings.Trim(toString(args[0]), toString(args[2])), nil
 	}
 
 	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
+		return strings.TrimLeft(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimRight(args[0].(string), args[1].(string)), nil
+		return strings.TrimRight(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSpace(args[0].(string)), nil
+		return strings.TrimSpace(toString(args[0])), nil
 	}
 
 	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
+		return strings.TrimPrefix(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
+		return strings.TrimSuffix(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
-		return reverseString(args[0].(string)), nil
+		return reverseString(toString(args[0])), nil
 	}
 
 	// encoding
 	functions["base64"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0])))
 
 		return sEnc, nil
 	}
 
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
 	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0])))
 
 		return insertInto(sEnc, 76, '\n'), nil
 	}
 
 	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
-		return base64.StdEncoding.DecodeString(args[0].(string))
+		return base64.StdEncoding.DecodeString(toString(args[0]))
 	}
 
 	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(args[0].(string)), nil
+		return url.PathEscape(toString(args[0])), nil
 	}
 
 	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(args[0].(string))
+		return url.PathUnescape(toString(args[0]))
 	}
 
 	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
-		return hex.EncodeToString([]byte(args[0].(string))), nil
+		return hex.EncodeToString([]byte(toString(args[0]))), nil
 	}
 
 	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
-		hx, _ := hex.DecodeString(args[0].(string))
+		hx, _ := hex.DecodeString(toString(args[0]))
 		return string(hx), nil
 	}
 
 	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
-		return html.EscapeString(args[0].(string)), nil
+		return html.EscapeString(toString(args[0])), nil
 	}
 
 	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
-		return html.UnescapeString(args[0].(string)), nil
+		return html.UnescapeString(toString(args[0])), nil
 	}
 
 	// hashing
 	functions["md5"] = func(args ...interface{}) (interface{}, error) {
-		hash := md5.Sum([]byte(args[0].(string)))
+		hash := md5.Sum([]byte(toString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
 	}
 
 	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		_, err := h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(toString(args[0])))
 
 		if err != nil {
 			return nil, err
@@ -151,7 +151,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 
 	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		_, err := h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(toString(args[0])))
 
 		if err != nil {
 			return nil, err
@@ -161,21 +161,21 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	}
 
 	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
-		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil
+		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(toString(args[0])), 0))), nil
 	}
 
 	// search
 	functions["contains"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Contains(args[0].(string), args[1].(string)), nil
+		return strings.Contains(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[0].(string))
+		compiled, err := regexp.Compile(toString(args[0]))
 		if err != nil {
 			return nil, err
 		}
 
-		return compiled.MatchString(args[1].(string)), nil
+		return compiled.MatchString(toString(args[1])), nil
 	}
 
 	// random generators
@@ -183,10 +183,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
-			chars = args[0].(string)
+			chars = toString(args[0])
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -203,10 +203,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 		if len(args) >= withBaseRandArgsSize {
-			base = args[2].(string)
+			base = toString(args[2])
 		}
 
 		base = TrimAll(base, bad)
@@ -223,7 +223,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -240,7 +240,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -257,7 +257,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -289,7 +289,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	// Collaborator
 	functions["collab"] = func(args ...interface{}) (interface{}, error) {
 		// check if collaborator contains a specific pattern
-		return collaborator.DefaultCollaborator.Has(args[0].(string)), nil
+		return collaborator.DefaultCollaborator.Has(toString(args[0])), nil
 	}
 
 	return functions

v2/pkg/generators/util.go

@@ -199,3 +199,7 @@ func insertInto(s string, interval int, sep rune) string {
 	buffer.WriteRune(sep)
 	return buffer.String()
 }
+
+func toString(v interface{}) string {
+	return fmt.Sprint(v)
+}

v2/pkg/matchers/match.go

@@ -199,7 +199,7 @@ func (m *Matcher) matchBinary(corpus string) bool {
 
 // matchDSL matches on a generic map result
 func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
-	// Iterate over all the regexes accepted as valid
+	// Iterate over all the expressions accepted as valid
 	for i, expression := range m.dslCompiled {
 		result, err := expression.Evaluate(mp)
 		if err != nil {

v2/pkg/requests/bulk-http-request.go

@@ -184,7 +184,12 @@ func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, d
 		r.gsfm.InitOrSkip(baseURL)
 		r.ReadOne(baseURL)
 
-		return r.handleRawWithPaylods(ctx, data, baseURL, values, r.gsfm.Value(baseURL))
+		payloads, err := r.GetPayloadsValues(baseURL)
+		if err != nil {
+			return nil, err
+		}
+
+		return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
 	}
 
 	// otherwise continue with normal flow
@@ -202,7 +207,7 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL
 
 	dynamicValues := make(map[string]interface{})
 	// find all potentials tokens between {{}}
-	var re = regexp.MustCompile(`(?m)\{\{.+}}`)
+	var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
 	for _, match := range re.FindAllString(raw, -1) {
 		// check if the match contains a dynamic variable
 		expr := generators.TrimDelimiters(match)
@@ -400,8 +405,15 @@ func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest,
 			value = p[1]
 		}
 
+		// in case of unsafe requests multiple headers should be accepted
+		// therefore use the full line as key
+		_, found := rawRequest.Headers[key]
+		if r.Unsafe && found {
+			rawRequest.Headers[line] = ""
+		} else {
 			rawRequest.Headers[key] = value
 		}
+	}
 
 	// Handle case with the full http url in path. In that case,
 	// ignore any host header that we encounter and use the path as request URL
@@ -482,3 +494,40 @@ func (r *BulkHTTPRequest) Total() int {
 func (r *BulkHTTPRequest) Increment(reqURL string) {
 	r.gsfm.Increment(reqURL)
 }
+
+// GetPayloadsValues for the specified URL
+func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface{}, error) {
+	payloadProcessedValues := make(map[string]interface{})
+	payloadsFromTemplate := r.gsfm.Value(reqURL)
+	for k, v := range payloadsFromTemplate {
+		kexp := v.(string)
+		// if it doesn't containg markups, we just continue
+		if !hasMarker(kexp) {
+			payloadProcessedValues[k] = v
+			continue
+		}
+		// attempts to expand expressions
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions())
+		if err != nil {
+			// it is a simple literal payload => proceed with literal value
+			payloadProcessedValues[k] = v
+			continue
+		}
+		// it is an expression - try to solve it
+		expValue, err := compiled.Evaluate(payloadsFromTemplate)
+		if err != nil {
+			// an error occurred => proceed with literal value
+			payloadProcessedValues[k] = v
+			continue
+		}
+		payloadProcessedValues[k] = fmt.Sprint(expValue)
+	}
+	var err error
+	if len(payloadProcessedValues) == 0 {
+		err = ErrNoPayload
+	}
+	return payloadProcessedValues, err
+}
+
+// ErrNoPayload error to avoid the additional base null request
+var ErrNoPayload = fmt.Errorf("No payload found")

v2/pkg/requests/util.go

@@ -71,3 +71,7 @@ func ExpandMapValues(m map[string]string) (m1 map[string][]string) {
 	}
 	return
 }
+
+func hasMarker(s string) bool {
+	return strings.Contains(s, markerParenthesisOpen) || strings.Contains(s, markerParenthesisClose) || strings.Contains(s, markerGeneral)
+}

v2/pkg/workflows/var.go

@@ -66,7 +66,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 			p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount())
 
 			for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
-				// apply externally supplied payloads if any
+				// apply externally supplied headers if any
 				request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
 				// apply externally supplied payloads if any
 				request.Payloads = generators.MergeMaps(request.Payloads, externalVars)