Merge pull request #448 from projectdiscovery/dev

Merging dev branch to master

.github/workflows/build.yaml

@@ -13,10 +13,10 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v1
+        uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.31
+          version: v1.33
           args: --timeout 5m
           working-directory: v2/
 

.golangci.yml

@@ -110,6 +110,6 @@ linters:
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration
 service:
-  golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
   prepare:
     - echo "here I can run custom commands, but no preparation needed for this repo"

README.md

@@ -94,6 +94,7 @@ This will display help for the tool. Here are all the switches it supports.
 |version        |Show version of nuclei               |nuclei -version|
 |proxy-url        |Proxy URL                      |nuclei -proxy-url hxxp://127.0.0.1:8080|
 |proxy-socks-url    |Socks proxyURL                   |nuclei -proxy-socks-url socks5://127.0.0.1:8080 |
+|random-agent            |Use random User-Agents |nuclei -random-agent |
 |H            |Custom Header                    |nuclei -H "x-bug-bounty: hacker"           |
 
 ## Installation Instructions
@@ -208,7 +209,7 @@ Remember to change `/path-to-nuclei-templates` to the real path on your host fil
 You can run the templates based on the specific severity of the template, single and multiple severity can be used for scan. 
 
 ```sh
-nuclei -l urls.txt -t cves/ -severity critical, medium
+nuclei -l urls.txt -t cves/ -severity critical,medium
 ```
 
 The above example will run all the templates under `cves` directory with `critical` and `medium` severity. 

v2/go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/corpix/uarand v0.1.1
 	github.com/d5/tengo/v2 v2.6.2
 	github.com/google/go-github/v32 v32.1.0
 	github.com/json-iterator/go v1.1.10
@@ -13,8 +14,8 @@ require (
 	github.com/miekg/dns v1.1.35
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.5
-	github.com/projectdiscovery/collaborator v0.0.1
+	github.com/projectdiscovery/collaborator v0.0.2
-	github.com/projectdiscovery/fastdialer v0.0.1
+	github.com/projectdiscovery/fastdialer v0.0.2
 	github.com/projectdiscovery/gologger v1.0.1
 	github.com/projectdiscovery/hmap v0.0.1
 	github.com/projectdiscovery/rawhttp v0.0.4
@@ -26,5 +27,5 @@ require (
 	github.com/stretchr/testify v1.5.1
 	go.uber.org/ratelimit v0.1.0
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -1,15 +1,23 @@
 github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
 github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
 github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
 github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
 github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -33,10 +41,12 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -48,8 +58,10 @@ github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F
 github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
 github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
 github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
+github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
-github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
+github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
+github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc=
+github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
 github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
@@ -112,5 +124,5 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

v2/internal/progress/progress.go

@@ -1,8 +1,13 @@
 package progress
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"net"
+	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -13,12 +18,13 @@ import (
 // Progress is a progress instance for showing program stats
 type Progress struct {
 	active       bool
-	stats        clistats.StatisticsClient
 	tickDuration time.Duration
+	stats        clistats.StatisticsClient
+	server       *http.Server
 }
 
 // NewProgress creates and returns a new progress tracking object.
-func NewProgress(active bool) *Progress {
+func NewProgress(active, metrics bool, port int) (*Progress, error) {
 	var tickDuration time.Duration
 	if active {
 		tickDuration = 5 * time.Second
@@ -26,29 +32,44 @@ func NewProgress(active bool) *Progress {
 		tickDuration = -1
 	}
 
-	var progress Progress
+	progress := &Progress{}
-	if active {
-		stats, err := clistats.New()
-		if err != nil {
-			gologger.Warningf("Couldn't create progress engine: %s\n", err)
-		}
-		progress.active = active
-		progress.stats = stats
-		progress.tickDuration = tickDuration
-	}
 
-	return &progress
+	stats, err := clistats.New()
+	if err != nil {
+		return nil, err
+	}
+	progress.active = active
+	progress.stats = stats
+	progress.tickDuration = tickDuration
+
+	if metrics {
+		http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
+			metrics := progress.getMetrics()
+			_ = json.NewEncoder(w).Encode(metrics)
+		})
+		progress.server = &http.Server{
+			Addr:    net.JoinHostPort("127.0.0.1", strconv.Itoa(port)),
+			Handler: http.DefaultServeMux,
+		}
+		go func() {
+			if err := progress.server.ListenAndServe(); err != nil {
+				gologger.Warningf("Could not serve metrics: %s\n", err)
+			}
+		}()
+	}
+	return progress, nil
 }
 
 // Init initializes the progress display mechanism by setting counters, etc.
 func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
+	p.stats.AddStatic("templates", rulesCount)
+	p.stats.AddStatic("hosts", hostCount)
+	p.stats.AddStatic("startedAt", time.Now())
+	p.stats.AddCounter("requests", uint64(0))
+	p.stats.AddCounter("errors", uint64(0))
+	p.stats.AddCounter("total", uint64(requestCount))
+
 	if p.active {
-		p.stats.AddStatic("templates", rulesCount)
-		p.stats.AddStatic("hosts", hostCount)
-		p.stats.AddStatic("startedAt", time.Now())
-		p.stats.AddCounter("requests", uint64(0))
-		p.stats.AddCounter("errors", uint64(0))
-		p.stats.AddCounter("total", uint64(requestCount))
 		if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
 			gologger.Warningf("Couldn't start statistics: %s\n", err)
 		}
@@ -57,25 +78,19 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
 
 // AddToTotal adds a value to the total request count
 func (p *Progress) AddToTotal(delta int64) {
-	if p.active {
+	p.stats.IncrementCounter("total", int(delta))
-		p.stats.IncrementCounter("total", int(delta))
-	}
 }
 
 // Update progress tracking information and increments the request counter by one unit.
 func (p *Progress) Update() {
-	if p.active {
+	p.stats.IncrementCounter("requests", 1)
-		p.stats.IncrementCounter("requests", 1)
-	}
 }
 
 // Drop drops the specified number of requests from the progress bar total.
 // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
 func (p *Progress) Drop(count int64) {
-	if p.active {
+	// mimic dropping by incrementing the completed requests
-		// mimic dropping by incrementing the completed requests
+	p.stats.IncrementCounter("errors", int(count))
-		p.stats.IncrementCounter("errors", int(count))
-	}
 }
 
 const bufferSize = 128
@@ -125,6 +140,34 @@ func makePrintCallback() func(stats clistats.StatisticsClient) {
 	}
 }
 
+// getMetrics returns a map of important metrics for client
+func (p *Progress) getMetrics() map[string]interface{} {
+	results := make(map[string]interface{})
+
+	startedAt, _ := p.stats.GetStatic("startedAt")
+	duration := time.Since(startedAt.(time.Time))
+
+	results["startedAt"] = startedAt.(time.Time)
+	results["duration"] = fmtDuration(duration)
+	templates, _ := p.stats.GetStatic("templates")
+	results["templates"] = clistats.String(templates)
+	hosts, _ := p.stats.GetStatic("hosts")
+	results["hosts"] = clistats.String(hosts)
+	requests, _ := p.stats.GetCounter("requests")
+	results["requests"] = clistats.String(requests)
+	total, _ := p.stats.GetCounter("total")
+	results["total"] = clistats.String(total)
+	results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds()))
+	errors, _ := p.stats.GetCounter("errors")
+	results["errors"] = clistats.String(errors)
+
+	//nolint:gomnd // this is not a magic number
+	percentData := (float64(requests) * float64(100)) / float64(total)
+	percent := clistats.String(uint64(percentData))
+	results["percent"] = percent
+	return results
+}
+
 // fmtDuration formats the duration for the time elapsed
 func fmtDuration(d time.Duration) string {
 	d = d.Round(time.Second)
@@ -143,4 +186,5 @@ func (p *Progress) Stop() {
 			gologger.Warningf("Couldn't stop statistics: %s\n", err)
 		}
 	}
+	_ = p.server.Shutdown(context.Background())
 }

v2/internal/runner/banner.go

@@ -7,11 +7,11 @@ const banner = `
      ____  __  _______/ /__  (_)
     / __ \/ / / / ___/ / _ \/ /
    / / / / /_/ / /__/ /  __/ /
-  /_/ /_/\__,_/\___/_/\___/_/   v2.2.0
+  /_/ /_/\__,_/\___/_/\___/_/   v2.2.1-dev
 `
 
 // Version is the current version of nuclei
-const Version = `2.2.0`
+const Version = `2.2.1-dev`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

v2/internal/runner/options.go

@@ -12,8 +12,10 @@ import (
 
 // Options contains the configuration options for tuning
 // the template requesting process.
-// nolint // false positive, options are allocated once and are necessary as is
 type Options struct {
+	RandomAgent          bool                   // Generate random User-Agent
+	Metrics              bool                   // Metrics enables display of metrics via an http endpoint
+	Sandbox              bool                   // Sandbox mode allows users to run isolated workflows with system commands disabled
 	Debug                bool                   // Debug mode allows debugging request/responses for the engine
 	Silent               bool                   // Silent suppresses any extra text and only writes found URLs on screen.
 	Version              bool                   // Version specifies if we should just show version and exit
@@ -28,13 +30,17 @@ type Options struct {
 	Stdin                bool                   // Stdin specifies whether stdin input was given to the process
 	StopAtFirstMatch     bool                   // Stop processing template at first full match (this may break chained requests)
 	NoMeta               bool                   // Don't display metadata for the matches
+	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
+	MetricsPort          int                    // MetricsPort is the port to show metrics on
+	MaxWorkflowDuration  int                    // MaxWorkflowDuration is the maximum time a workflow can run for a URL
 	BulkSize             int                    // Number of targets analyzed in parallel for each template
 	TemplateThreads      int                    // Number of templates executed in parallel
-	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
-	ProjectPath          string                 // Nuclei uses a user defined project folder
 	Timeout              int                    // Timeout is the seconds to wait for a response from the server.
 	Retries              int                    // Retries is the number of times to retry the request
 	RateLimit            int                    // Rate-Limit of requests per specified target
+	Threads              int                    // Thread controls the number of concurrent requests to make.
+	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
+	ProjectPath          string                 // Nuclei uses a user defined project folder
 	Severity             string                 // Filter templates based on their severity and only run the matching ones.
 	Target               string                 // Target is a single URL/Domain to scan usng a template
 	Targets              string                 // Targets specifies the targets to scan using templates.
@@ -46,8 +52,6 @@ type Options struct {
 	Templates            multiStringFlag        // Signature specifies the template/templates to use
 	ExcludedTemplates    multiStringFlag        // Signature specifies the template/templates to exclude
 	CustomHeaders        requests.CustomHeaders // Custom global headers
-	Threads              int                    // Thread controls the number of concurrent requests to make.
-	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
 }
 
 type multiStringFlag []string
@@ -65,6 +69,10 @@ func (m *multiStringFlag) Set(value string) error {
 func ParseOptions() *Options {
 	options := &Options{}
 
+	flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode")
+	flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
+	flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
+	flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes")
 	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
 	flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.")
 	flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.")
@@ -79,6 +87,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output")
 	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
 	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
+	flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value")
 	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
 	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
 	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")

v2/internal/runner/processor.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	tengo "github.com/d5/tengo/v2"
 	"github.com/d5/tengo/v2/stdlib"
@@ -28,6 +29,8 @@ type workflowTemplates struct {
 	Templates []*workflows.Template
 }
 
+var sandboxedModules = []string{"math", "text", "rand", "fmt", "json", "base64", "hex", "enum"}
+
 // processTemplateWithList processes a template and runs the enumeration on all the targets
 func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
 	var httpExecuter *executer.HTTPExecuter
@@ -62,6 +65,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
 			Retries:          r.options.Retries,
 			ProxyURL:         r.options.ProxyURL,
 			ProxySocksURL:    r.options.ProxySocksURL,
+			RandomAgent:      r.options.RandomAgent,
 			CustomHeaders:    r.options.CustomHeaders,
 			JSON:             r.options.JSON,
 			JSONRequests:     r.options.JSONRequests,
@@ -127,13 +131,11 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo
 	workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow)
 	if err != nil {
 		gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err)
-		return result
+		return false
 	}
-
 	logicBytes := []byte(workflow.Logic)
 
 	wg := sizedwaitgroup.New(r.options.BulkSize)
-
 	r.hm.Scan(func(k, _ []byte) error {
 		targetURL := string(k)
 		wg.Add()
@@ -142,10 +144,13 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo
 			defer wg.Done()
 
 			script := tengo.NewScript(logicBytes)
-			script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+			if !r.options.Sandbox {
+				script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+			} else {
+				script.SetImports(stdlib.GetModuleMap(sandboxedModules...))
+			}
 
 			variables := make(map[string]*workflows.NucleiVar)
-
 			for _, workflowTemplate := range *workflowTemplatesList {
 				name := workflowTemplate.Name
 				variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL}
@@ -157,7 +162,10 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo
 				variables[name] = variable
 			}
 
-			_, err := script.RunContext(context.Background())
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.options.MaxWorkflowDuration)*time.Minute)
+			defer cancel()
+
+			_, err := script.RunContext(ctx)
 			if err != nil {
 				gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
 			}
@@ -225,6 +233,7 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl
 					Retries:          r.options.Retries,
 					ProxyURL:         r.options.ProxyURL,
 					ProxySocksURL:    r.options.ProxySocksURL,
+					RandomAgent:      r.options.RandomAgent,
 					CustomHeaders:    r.options.CustomHeaders,
 					JSON:             r.options.JSON,
 					JSONRequests:     r.options.JSONRequests,
@@ -298,6 +307,7 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl
 						Retries:       r.options.Retries,
 						ProxyURL:      r.options.ProxyURL,
 						ProxySocksURL: r.options.ProxySocksURL,
+						RandomAgent:   r.options.RandomAgent,
 						CustomHeaders: r.options.CustomHeaders,
 						CookieJar:     jar,
 						TraceLog:      r.traceLog,

v2/internal/runner/runner.go

@@ -173,7 +173,11 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	// Creates the progress tracking object
-	runner.progress = progress.NewProgress(options.EnableProgressBar)
+	var progressErr error
+	runner.progress, progressErr = progress.NewProgress(options.EnableProgressBar, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
 
 	// create project file if requested or load existing one
 	if options.Project {

v2/pkg/collaborator/collaborator.go

@@ -2,6 +2,7 @@ package collaborator
 
 import (
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/projectdiscovery/collaborator"
@@ -17,6 +18,7 @@ var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
 var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()}
 
 type BurpCollaborator struct {
+	sync.RWMutex
 	options *Options // unused
 	Collab  *collaborator.BurpCollaborator
 }
@@ -41,19 +43,22 @@ func (b *BurpCollaborator) Poll() {
 	}
 }
 
-func (b *BurpCollaborator) Has(s string) bool {
+func (b *BurpCollaborator) Has(s string) (found bool) {
+	foundAt := 0
 	for _, r := range b.Collab.RespBuffer {
 		for i := 0; i < len(r.Responses); i++ {
-			// search in dns
+			// search in dns - http - smtp
-			if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) {
+			b.RLock()
-				return true
+			found = strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) || strings.Contains(r.Responses[i].Data.RequestDecoded, s) || strings.Contains(r.Responses[i].Data.MessageDecoded, s)
-			}
+			b.RUnlock()
-			// search in http
+			if found {
-			if strings.Contains(r.Responses[i].Data.RequestDecoded, s) {
+				b.Lock()
-				return true
+				r.Responses = removeMatch(r.Responses, foundAt)
+				b.Unlock()
+				break
 			}
 		}
 	}
 
-	return false
+	return
 }

v2/pkg/collaborator/util.go

@@ -0,0 +1,9 @@
+package collaborator
+
+import (
+	"github.com/projectdiscovery/collaborator"
+)
+
+func removeMatch(responses []collaborator.BurpResponse, index int) []collaborator.BurpResponse {
+	return append(responses[:index], responses[index+1:]...)
+}

v2/pkg/executer/executer_http.go

@@ -19,6 +19,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
@@ -59,6 +60,7 @@ type HTTPExecuter struct {
 	CookieJar        *cookiejar.Jar
 	traceLog         tracelog.Log
 	decolorizer      *regexp.Regexp
+	randomAgent      bool
 	coloredOutput    bool
 	debug            bool
 	Results          bool
@@ -71,18 +73,7 @@ type HTTPExecuter struct {
 
 // HTTPOptions contains configuration options for the HTTP executer.
 type HTTPOptions struct {
-	CustomHeaders    requests.CustomHeaders
+	RandomAgent      bool
-	ProxyURL         string
-	ProxySocksURL    string
-	Template         *templates.Template
-	BulkHTTPRequest  *requests.BulkHTTPRequest
-	Writer           *bufwriter.Writer
-	Timeout          int
-	Retries          int
-	CookieJar        *cookiejar.Jar
-	Colorizer        *colorizer.NucleiColorizer
-	Decolorizer      *regexp.Regexp
-	TraceLog         tracelog.Log
 	Debug            bool
 	JSON             bool
 	JSONRequests     bool
@@ -90,6 +81,18 @@ type HTTPOptions struct {
 	CookieReuse      bool
 	ColoredOutput    bool
 	StopAtFirstMatch bool
+	Timeout          int
+	Retries          int
+	ProxyURL         string
+	ProxySocksURL    string
+	Template         *templates.Template
+	BulkHTTPRequest  *requests.BulkHTTPRequest
+	Writer           *bufwriter.Writer
+	CustomHeaders    requests.CustomHeaders
+	CookieJar        *cookiejar.Jar
+	Colorizer        *colorizer.NucleiColorizer
+	Decolorizer      *regexp.Regexp
+	TraceLog         tracelog.Log
 	PF               *projetctfile.ProjectFile
 	RateLimiter      ratelimit.Limiter
 	Dialer           *fastdialer.Dialer
@@ -140,6 +143,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 		template:         options.Template,
 		bulkHTTPRequest:  options.BulkHTTPRequest,
 		writer:           options.Writer,
+		randomAgent:      options.RandomAgent,
 		customHeaders:    options.CustomHeaders,
 		CookieJar:        options.CookieJar,
 		coloredOutput:    options.ColoredOutput,
@@ -174,16 +178,21 @@ func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
 	for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
 		swg.Add()
 		// base request
+		result.Lock()
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
-			return result
 		}
 		go func(httpRequest *requests.HTTPRequest) {
 			defer swg.Done()
 
 			// If the request was built correctly then execute it
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
 			if err != nil {
 				result.Error = errors.Wrap(err, "could not handle http request")
 			}
@@ -214,9 +223,15 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
 	// Workers that keeps enqueuing new requests
 	maxWorkers := e.bulkHTTPRequest.Threads
 	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+	for e.bulkHTTPRequest.Next(reqURL) {
+		result.Lock()
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
 			p.Drop(remaining)
 		} else {
@@ -227,7 +242,7 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
 				e.ratelimiter.Take()
 
 				// If the request was built correctly then execute it
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
 				if err != nil {
 					e.traceLog.Request(e.template.ID, reqURL, "http", err)
 					result.Error = errors.Wrap(err, "could not handle http request")
@@ -284,9 +299,15 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
 		maxWorkers = pipeOptions.MaxPendingRequests
 	}
 	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+	for e.bulkHTTPRequest.Next(reqURL) {
+		result.Lock()
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
 		} else {
 			swg.Add()
@@ -297,7 +318,7 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
 				// If the request was built correctly then execute it
 				request.Pipeline = true
 				request.PipelineClient = pipeclient
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
 				if err != nil {
 					e.traceLog.Request(e.template.ID, reqURL, "http", err)
 					result.Error = errors.Wrap(err, "could not handle http request")
@@ -349,17 +370,23 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
 	remaining := e.bulkHTTPRequest.GetRequestCount()
 	e.bulkHTTPRequest.CreateGenerator(reqURL)
 
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+	for e.bulkHTTPRequest.Next(reqURL) {
 		requestNumber++
+		result.Lock()
 		httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
+		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
+		result.Unlock()
+		// ignore the error due to the base request having null paylods
+		if err == requests.ErrNoPayload {
+			// pass through
+		} else if err != nil {
 			result.Error = err
 			p.Drop(remaining)
 		} else {
 			e.ratelimiter.Take()
 			// If the request was built correctly then execute it
 			format := "%s_" + strconv.Itoa(requestNumber)
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)
+			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format)
 			if err != nil {
 				result.Error = errors.Wrap(err, "could not handle http request")
 				p.Drop(remaining)
@@ -384,7 +411,13 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
 	return result
 }
 
-func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error {
+func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error {
+	// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
+	if e.randomAgent {
+		// nolint:errcheck // ignoring error
+		e.customHeaders.Set("User-Agent: " + uarand.GetRandom())
+	}
+
 	e.setCustomHeaders(request)
 
 	var (
@@ -516,18 +549,36 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 
 	headers := headersToString(resp.Header)
 
+	var matchData map[string]interface{}
+	if payloads != nil {
+		matchData = generators.MergeMaps(result.historyData, payloads)
+	}
+
 	// store for internal purposes the DSL matcher data
 	// hardcode stopping storing data after defaultMaxHistorydata items
 	if len(result.historyData) < defaultMaxHistorydata {
 		result.Lock()
+		// update history data with current reqURL and hostname
+		result.historyData["reqURL"] = reqURL
+		if parsed, err := url.Parse(reqURL); err == nil {
+			result.historyData["Hostname"] = parsed.Host
+		}
 		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
+		if payloads == nil {
+			// merge them to history data
+			result.historyData = generators.MergeMaps(result.historyData, payloads)
+		}
+		result.historyData = generators.MergeMaps(result.historyData, dynamicvalues)
+
+		// complement match data with new one if necessary
+		matchData = generators.MergeMaps(matchData, result.historyData)
 		result.Unlock()
 	}
 
 	matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
 	for _, matcher := range e.bulkHTTPRequest.Matchers {
 		// Check if the matcher matched
-		if !matcher.Match(resp, body, headers, duration, result.historyData) {
+		if !matcher.Match(resp, body, headers, duration, matchData) {
 			// If the condition is AND we haven't matched, try next request.
 			if matcherCondition == matchers.ANDCondition {
 				return nil
@@ -542,7 +593,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 				result.Meta = request.Meta
 				result.GotResults = true
 				result.Unlock()
-				e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL)
+				e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL)
 			}
 		}
 	}
@@ -573,7 +624,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 	// Write a final string of output if matcher type is
 	// AND or if we have extractors for the mechanism too.
 	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL)
+		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL)
 		result.Lock()
 		result.GotResults = true
 		result.Unlock()
@@ -701,7 +752,6 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
 type Result struct {
 	sync.Mutex
 	GotResults  bool
-	Done        bool
 	Meta        map[string]interface{}
 	Matches     map[string]interface{}
 	Extractions map[string]interface{}

v2/pkg/executer/output_http.go

@@ -1,6 +1,7 @@
 package executer
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httputil"
 	"strings"
@@ -119,9 +120,8 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
 		builder.WriteString(" [")
 
 		var metas []string
-
 		for name, value := range req.Meta {
-			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String())
+			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String())
 		}
 
 		builder.WriteString(strings.Join(metas, ","))

v2/pkg/generators/dsl.go

@@ -35,112 +35,112 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 
 	// strings
 	functions["len"] = func(args ...interface{}) (interface{}, error) {
-		length := len(args[0].(string))
+		length := len(toString(args[0]))
 
 		return float64(length), nil
 	}
 
 	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToUpper(args[0].(string)), nil
+		return strings.ToUpper(toString(args[0])), nil
 	}
 
 	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToLower(args[0].(string)), nil
+		return strings.ToLower(toString(args[0])), nil
 	}
 
 	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil
+		return strings.ReplaceAll(toString(args[0]), toString(args[1]), toString(args[2])), nil
 	}
 
 	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[1].(string))
+		compiled, err := regexp.Compile(toString(args[1]))
 		if err != nil {
 			return nil, err
 		}
-		return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil
+		return compiled.ReplaceAllString(toString(args[0]), toString(args[2])), nil
 	}
 
 	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(args[0].(string), args[2].(string)), nil
+		return strings.Trim(toString(args[0]), toString(args[2])), nil
 	}
 
 	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
+		return strings.TrimLeft(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimRight(args[0].(string), args[1].(string)), nil
+		return strings.TrimRight(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSpace(args[0].(string)), nil
+		return strings.TrimSpace(toString(args[0])), nil
 	}
 
 	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
+		return strings.TrimPrefix(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
+		return strings.TrimSuffix(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
-		return reverseString(args[0].(string)), nil
+		return reverseString(toString(args[0])), nil
 	}
 
 	// encoding
 	functions["base64"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0])))
 
 		return sEnc, nil
 	}
 
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
 	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0])))
 
 		return insertInto(sEnc, 76, '\n'), nil
 	}
 
 	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
-		return base64.StdEncoding.DecodeString(args[0].(string))
+		return base64.StdEncoding.DecodeString(toString(args[0]))
 	}
 
 	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(args[0].(string)), nil
+		return url.PathEscape(toString(args[0])), nil
 	}
 
 	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(args[0].(string))
+		return url.PathUnescape(toString(args[0]))
 	}
 
 	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
-		return hex.EncodeToString([]byte(args[0].(string))), nil
+		return hex.EncodeToString([]byte(toString(args[0]))), nil
 	}
 
 	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
-		hx, _ := hex.DecodeString(args[0].(string))
+		hx, _ := hex.DecodeString(toString(args[0]))
 		return string(hx), nil
 	}
 
 	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
-		return html.EscapeString(args[0].(string)), nil
+		return html.EscapeString(toString(args[0])), nil
 	}
 
 	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
-		return html.UnescapeString(args[0].(string)), nil
+		return html.UnescapeString(toString(args[0])), nil
 	}
 
 	// hashing
 	functions["md5"] = func(args ...interface{}) (interface{}, error) {
-		hash := md5.Sum([]byte(args[0].(string)))
+		hash := md5.Sum([]byte(toString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
 	}
 
 	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		_, err := h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(toString(args[0])))
 
 		if err != nil {
 			return nil, err
@@ -151,7 +151,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 
 	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		_, err := h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(toString(args[0])))
 
 		if err != nil {
 			return nil, err
@@ -161,21 +161,21 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	}
 
 	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
-		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil
+		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(toString(args[0])), 0))), nil
 	}
 
 	// search
 	functions["contains"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Contains(args[0].(string), args[1].(string)), nil
+		return strings.Contains(toString(args[0]), toString(args[1])), nil
 	}
 
 	functions["regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[0].(string))
+		compiled, err := regexp.Compile(toString(args[0]))
 		if err != nil {
 			return nil, err
 		}
 
-		return compiled.MatchString(args[1].(string)), nil
+		return compiled.MatchString(toString(args[1])), nil
 	}
 
 	// random generators
@@ -183,10 +183,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
-			chars = args[0].(string)
+			chars = toString(args[0])
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -203,10 +203,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 		if len(args) >= withBaseRandArgsSize {
-			base = args[2].(string)
+			base = toString(args[2])
 		}
 
 		base = TrimAll(base, bad)
@@ -223,7 +223,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -240,7 +240,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -257,7 +257,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = toString(args[1])
 		}
 
 		chars = TrimAll(chars, bad)
@@ -289,7 +289,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	// Collaborator
 	functions["collab"] = func(args ...interface{}) (interface{}, error) {
 		// check if collaborator contains a specific pattern
-		return collaborator.DefaultCollaborator.Has(args[0].(string)), nil
+		return collaborator.DefaultCollaborator.Has(toString(args[0])), nil
 	}
 
 	return functions

v2/pkg/generators/util.go

@@ -199,3 +199,7 @@ func insertInto(s string, interval int, sep rune) string {
 	buffer.WriteRune(sep)
 	return buffer.String()
 }
+
+func toString(v interface{}) string {
+	return fmt.Sprint(v)
+}

v2/pkg/matchers/match.go

@@ -199,7 +199,7 @@ func (m *Matcher) matchBinary(corpus string) bool {
 
 // matchDSL matches on a generic map result
 func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
-	// Iterate over all the regexes accepted as valid
+	// Iterate over all the expressions accepted as valid
 	for i, expression := range m.dslCompiled {
 		result, err := expression.Evaluate(mp)
 		if err != nil {

v2/pkg/requests/bulk-http-request.go

@@ -184,7 +184,12 @@ func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, d
 		r.gsfm.InitOrSkip(baseURL)
 		r.ReadOne(baseURL)
 
-		return r.handleRawWithPaylods(ctx, data, baseURL, values, r.gsfm.Value(baseURL))
+		payloads, err := r.GetPayloadsValues(baseURL)
+		if err != nil {
+			return nil, err
+		}
+
+		return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
 	}
 
 	// otherwise continue with normal flow
@@ -202,7 +207,7 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL
 
 	dynamicValues := make(map[string]interface{})
 	// find all potentials tokens between {{}}
-	var re = regexp.MustCompile(`(?m)\{\{.+}}`)
+	var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
 	for _, match := range re.FindAllString(raw, -1) {
 		// check if the match contains a dynamic variable
 		expr := generators.TrimDelimiters(match)
@@ -400,7 +405,14 @@ func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest,
 			value = p[1]
 		}
 
-		rawRequest.Headers[key] = value
+		// in case of unsafe requests multiple headers should be accepted
+		// therefore use the full line as key
+		_, found := rawRequest.Headers[key]
+		if r.Unsafe && found {
+			rawRequest.Headers[line] = ""
+		} else {
+			rawRequest.Headers[key] = value
+		}
 	}
 
 	// Handle case with the full http url in path. In that case,
@@ -482,3 +494,40 @@ func (r *BulkHTTPRequest) Total() int {
 func (r *BulkHTTPRequest) Increment(reqURL string) {
 	r.gsfm.Increment(reqURL)
 }
+
+// GetPayloadsValues for the specified URL
+func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface{}, error) {
+	payloadProcessedValues := make(map[string]interface{})
+	payloadsFromTemplate := r.gsfm.Value(reqURL)
+	for k, v := range payloadsFromTemplate {
+		kexp := v.(string)
+		// if it doesn't containing markups, we just continue
+		if !hasMarker(kexp) {
+			payloadProcessedValues[k] = v
+			continue
+		}
+		// attempts to expand expressions
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions())
+		if err != nil {
+			// it is a simple literal payload => proceed with literal value
+			payloadProcessedValues[k] = v
+			continue
+		}
+		// it is an expression - try to solve it
+		expValue, err := compiled.Evaluate(payloadsFromTemplate)
+		if err != nil {
+			// an error occurred => proceed with literal value
+			payloadProcessedValues[k] = v
+			continue
+		}
+		payloadProcessedValues[k] = fmt.Sprint(expValue)
+	}
+	var err error
+	if len(payloadProcessedValues) == 0 {
+		err = ErrNoPayload
+	}
+	return payloadProcessedValues, err
+}
+
+// ErrNoPayload error to avoid the additional base null request
+var ErrNoPayload = fmt.Errorf("no payload found")

v2/pkg/requests/util.go

@@ -71,3 +71,7 @@ func ExpandMapValues(m map[string]string) (m1 map[string][]string) {
 	}
 	return
 }
+
+func hasMarker(s string) bool {
+	return strings.Contains(s, markerParenthesisOpen) || strings.Contains(s, markerParenthesisClose) || strings.Contains(s, markerGeneral)
+}

v2/pkg/workflows/var.go

@@ -66,7 +66,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 			p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount())
 
 			for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
-				// apply externally supplied payloads if any
+				// apply externally supplied headers if any
 				request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
 				// apply externally supplied payloads if any
 				request.Payloads = generators.MergeMaps(request.Payloads, externalVars)