diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ead969e43..5e6435e01 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v1 + uses: golangci/golangci-lint-action@v2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.31 + version: v1.33 args: --timeout 5m working-directory: v2/ diff --git a/.golangci.yml b/.golangci.yml index 917ad0fec..694cea254 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -110,6 +110,6 @@ linters: # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly prepare: - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/README.md b/README.md index 609e58052..30cdc56ec 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ This will display help for the tool. Here are all the switches it supports. |version |Show version of nuclei |nuclei -version| |proxy-url |Proxy URL |nuclei -proxy-url hxxp://127.0.0.1:8080| |proxy-socks-url |Socks proxyURL |nuclei -proxy-socks-url socks5://127.0.0.1:8080 | +|random-agent |Use random User-Agents |nuclei -random-agent | |H |Custom Header |nuclei -H "x-bug-bounty: hacker" | ## Installation Instructions @@ -208,7 +209,7 @@ Remember to change `/path-to-nuclei-templates` to the real path on your host fil You can run the templates based on the specific severity of the template, single and multiple severity can be used for scan. ```sh -nuclei -l urls.txt -t cves/ -severity critical, medium +nuclei -l urls.txt -t cves/ -severity critical,medium ``` The above example will run all the templates under `cves` directory with `critical` and `medium` severity. diff --git a/v2/go.mod b/v2/go.mod index 1e8218a80..1f63ac369 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible + github.com/corpix/uarand v0.1.1 github.com/d5/tengo/v2 v2.6.2 github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 @@ -13,8 +14,8 @@ require ( github.com/miekg/dns v1.1.35 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.5 - github.com/projectdiscovery/collaborator v0.0.1 - github.com/projectdiscovery/fastdialer v0.0.1 + github.com/projectdiscovery/collaborator v0.0.2 + github.com/projectdiscovery/fastdialer v0.0.2 github.com/projectdiscovery/gologger v1.0.1 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/rawhttp v0.0.4 @@ -26,5 +27,5 @@ require ( github.com/stretchr/testify v1.5.1 go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/v2/go.sum b/v2/go.sum index e336c0eb0..ac2df7df3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,15 +1,23 @@ github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -33,10 +41,12 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -48,8 +58,10 @@ github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI= github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= -github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg= -github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc= +github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s= +github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= +github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc= +github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= @@ -112,5 +124,5 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 6c12be52f..a9b5c2ead 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -1,8 +1,13 @@ package progress import ( + "context" + "encoding/json" "fmt" + "net" + "net/http" "os" + "strconv" "strings" "time" @@ -13,12 +18,13 @@ import ( // Progress is a progress instance for showing program stats type Progress struct { active bool - stats clistats.StatisticsClient tickDuration time.Duration + stats clistats.StatisticsClient + server *http.Server } // NewProgress creates and returns a new progress tracking object. -func NewProgress(active bool) *Progress { +func NewProgress(active, metrics bool, port int) (*Progress, error) { var tickDuration time.Duration if active { tickDuration = 5 * time.Second @@ -26,29 +32,44 @@ func NewProgress(active bool) *Progress { tickDuration = -1 } - var progress Progress - if active { - stats, err := clistats.New() - if err != nil { - gologger.Warningf("Couldn't create progress engine: %s\n", err) - } - progress.active = active - progress.stats = stats - progress.tickDuration = tickDuration - } + progress := &Progress{} - return &progress + stats, err := clistats.New() + if err != nil { + return nil, err + } + progress.active = active + progress.stats = stats + progress.tickDuration = tickDuration + + if metrics { + http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { + metrics := progress.getMetrics() + _ = json.NewEncoder(w).Encode(metrics) + }) + progress.server = &http.Server{ + Addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), + Handler: http.DefaultServeMux, + } + go func() { + if err := progress.server.ListenAndServe(); err != nil { + gologger.Warningf("Could not serve metrics: %s\n", err) + } + }() + } + return progress, nil } // Init initializes the progress display mechanism by setting counters, etc. func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { + p.stats.AddStatic("templates", rulesCount) + p.stats.AddStatic("hosts", hostCount) + p.stats.AddStatic("startedAt", time.Now()) + p.stats.AddCounter("requests", uint64(0)) + p.stats.AddCounter("errors", uint64(0)) + p.stats.AddCounter("total", uint64(requestCount)) + if p.active { - p.stats.AddStatic("templates", rulesCount) - p.stats.AddStatic("hosts", hostCount) - p.stats.AddStatic("startedAt", time.Now()) - p.stats.AddCounter("requests", uint64(0)) - p.stats.AddCounter("errors", uint64(0)) - p.stats.AddCounter("total", uint64(requestCount)) if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil { gologger.Warningf("Couldn't start statistics: %s\n", err) } @@ -57,25 +78,19 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { // AddToTotal adds a value to the total request count func (p *Progress) AddToTotal(delta int64) { - if p.active { - p.stats.IncrementCounter("total", int(delta)) - } + p.stats.IncrementCounter("total", int(delta)) } // Update progress tracking information and increments the request counter by one unit. func (p *Progress) Update() { - if p.active { - p.stats.IncrementCounter("requests", 1) - } + p.stats.IncrementCounter("requests", 1) } // Drop drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. func (p *Progress) Drop(count int64) { - if p.active { - // mimic dropping by incrementing the completed requests - p.stats.IncrementCounter("errors", int(count)) - } + // mimic dropping by incrementing the completed requests + p.stats.IncrementCounter("errors", int(count)) } const bufferSize = 128 @@ -125,6 +140,34 @@ func makePrintCallback() func(stats clistats.StatisticsClient) { } } +// getMetrics returns a map of important metrics for client +func (p *Progress) getMetrics() map[string]interface{} { + results := make(map[string]interface{}) + + startedAt, _ := p.stats.GetStatic("startedAt") + duration := time.Since(startedAt.(time.Time)) + + results["startedAt"] = startedAt.(time.Time) + results["duration"] = fmtDuration(duration) + templates, _ := p.stats.GetStatic("templates") + results["templates"] = clistats.String(templates) + hosts, _ := p.stats.GetStatic("hosts") + results["hosts"] = clistats.String(hosts) + requests, _ := p.stats.GetCounter("requests") + results["requests"] = clistats.String(requests) + total, _ := p.stats.GetCounter("total") + results["total"] = clistats.String(total) + results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds())) + errors, _ := p.stats.GetCounter("errors") + results["errors"] = clistats.String(errors) + + //nolint:gomnd // this is not a magic number + percentData := (float64(requests) * float64(100)) / float64(total) + percent := clistats.String(uint64(percentData)) + results["percent"] = percent + return results +} + // fmtDuration formats the duration for the time elapsed func fmtDuration(d time.Duration) string { d = d.Round(time.Second) @@ -143,4 +186,5 @@ func (p *Progress) Stop() { gologger.Warningf("Couldn't stop statistics: %s\n", err) } } + _ = p.server.Shutdown(context.Background()) } diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index f0e360a5f..88a1de78d 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,11 +7,11 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.2.0 + /_/ /_/\__,_/\___/_/\___/_/ v2.2.1-dev ` // Version is the current version of nuclei -const Version = `2.2.0` +const Version = `2.2.1-dev` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 197f880e0..b00bb485f 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -12,8 +12,10 @@ import ( // Options contains the configuration options for tuning // the template requesting process. -// nolint // false positive, options are allocated once and are necessary as is type Options struct { + RandomAgent bool // Generate random User-Agent + Metrics bool // Metrics enables display of metrics via an http endpoint + Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled Debug bool // Debug mode allows debugging request/responses for the engine Silent bool // Silent suppresses any extra text and only writes found URLs on screen. Version bool // Version specifies if we should just show version and exit @@ -28,13 +30,17 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests) NoMeta bool // Don't display metadata for the matches + Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times + MetricsPort int // MetricsPort is the port to show metrics on + MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL BulkSize int // Number of targets analyzed in parallel for each template TemplateThreads int // Number of templates executed in parallel - Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times - ProjectPath string // Nuclei uses a user defined project folder Timeout int // Timeout is the seconds to wait for a response from the server. Retries int // Retries is the number of times to retry the request RateLimit int // Rate-Limit of requests per specified target + Threads int // Thread controls the number of concurrent requests to make. + BurpCollaboratorBiid string // Burp Collaborator BIID for polling + ProjectPath string // Nuclei uses a user defined project folder Severity string // Filter templates based on their severity and only run the matching ones. Target string // Target is a single URL/Domain to scan usng a template Targets string // Targets specifies the targets to scan using templates. @@ -46,8 +52,6 @@ type Options struct { Templates multiStringFlag // Signature specifies the template/templates to use ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude CustomHeaders requests.CustomHeaders // Custom global headers - Threads int // Thread controls the number of concurrent requests to make. - BurpCollaboratorBiid string // Burp Collaborator BIID for polling } type multiStringFlag []string @@ -65,6 +69,10 @@ func (m *multiStringFlag) Set(value string) error { func ParseOptions() *Options { options := &Options{} + flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") + flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") + flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") + flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes") flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.") @@ -79,6 +87,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output") flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") + flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value") flag.Var(&options.CustomHeaders, "H", "Custom Header.") flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 7b2c1126f..2cd288613 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -8,6 +8,7 @@ import ( "path" "path/filepath" "strings" + "time" tengo "github.com/d5/tengo/v2" "github.com/d5/tengo/v2/stdlib" @@ -28,6 +29,8 @@ type workflowTemplates struct { Templates []*workflows.Template } +var sandboxedModules = []string{"math", "text", "rand", "fmt", "json", "base64", "hex", "enum"} + // processTemplateWithList processes a template and runs the enumeration on all the targets func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { var httpExecuter *executer.HTTPExecuter @@ -62,6 +65,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, + RandomAgent: r.options.RandomAgent, CustomHeaders: r.options.CustomHeaders, JSON: r.options.JSON, JSONRequests: r.options.JSONRequests, @@ -127,13 +131,11 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow) if err != nil { gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err) - return result + return false } - logicBytes := []byte(workflow.Logic) wg := sizedwaitgroup.New(r.options.BulkSize) - r.hm.Scan(func(k, _ []byte) error { targetURL := string(k) wg.Add() @@ -142,10 +144,13 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo defer wg.Done() script := tengo.NewScript(logicBytes) - script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) + if !r.options.Sandbox { + script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) + } else { + script.SetImports(stdlib.GetModuleMap(sandboxedModules...)) + } variables := make(map[string]*workflows.NucleiVar) - for _, workflowTemplate := range *workflowTemplatesList { name := workflowTemplate.Name variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL} @@ -157,7 +162,10 @@ func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflo variables[name] = variable } - _, err := script.RunContext(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.options.MaxWorkflowDuration)*time.Minute) + defer cancel() + + _, err := script.RunContext(ctx) if err != nil { gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err) } @@ -225,6 +233,7 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, + RandomAgent: r.options.RandomAgent, CustomHeaders: r.options.CustomHeaders, JSON: r.options.JSON, JSONRequests: r.options.JSONRequests, @@ -298,6 +307,7 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, + RandomAgent: r.options.RandomAgent, CustomHeaders: r.options.CustomHeaders, CookieJar: jar, TraceLog: r.traceLog, diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 46c7d4e88..79ded4bf0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -173,7 +173,11 @@ func New(options *Options) (*Runner, error) { } // Creates the progress tracking object - runner.progress = progress.NewProgress(options.EnableProgressBar) + var progressErr error + runner.progress, progressErr = progress.NewProgress(options.EnableProgressBar, options.Metrics, options.MetricsPort) + if progressErr != nil { + return nil, progressErr + } // create project file if requested or load existing one if options.Project { diff --git a/v2/pkg/collaborator/collaborator.go b/v2/pkg/collaborator/collaborator.go index f4d34c9e3..4b6383b23 100644 --- a/v2/pkg/collaborator/collaborator.go +++ b/v2/pkg/collaborator/collaborator.go @@ -2,6 +2,7 @@ package collaborator import ( "strings" + "sync" "time" "github.com/projectdiscovery/collaborator" @@ -17,6 +18,7 @@ var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds) var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()} type BurpCollaborator struct { + sync.RWMutex options *Options // unused Collab *collaborator.BurpCollaborator } @@ -41,19 +43,22 @@ func (b *BurpCollaborator) Poll() { } } -func (b *BurpCollaborator) Has(s string) bool { +func (b *BurpCollaborator) Has(s string) (found bool) { + foundAt := 0 for _, r := range b.Collab.RespBuffer { for i := 0; i < len(r.Responses); i++ { - // search in dns - if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) { - return true - } - // search in http - if strings.Contains(r.Responses[i].Data.RequestDecoded, s) { - return true + // search in dns - http - smtp + b.RLock() + found = strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) || strings.Contains(r.Responses[i].Data.RequestDecoded, s) || strings.Contains(r.Responses[i].Data.MessageDecoded, s) + b.RUnlock() + if found { + b.Lock() + r.Responses = removeMatch(r.Responses, foundAt) + b.Unlock() + break } } } - return false + return } diff --git a/v2/pkg/collaborator/util.go b/v2/pkg/collaborator/util.go new file mode 100644 index 000000000..a6e35675b --- /dev/null +++ b/v2/pkg/collaborator/util.go @@ -0,0 +1,9 @@ +package collaborator + +import ( + "github.com/projectdiscovery/collaborator" +) + +func removeMatch(responses []collaborator.BurpResponse, index int) []collaborator.BurpResponse { + return append(responses[:index], responses[index+1:]...) +} diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index cd7c05847..c8b58b403 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" @@ -59,6 +60,7 @@ type HTTPExecuter struct { CookieJar *cookiejar.Jar traceLog tracelog.Log decolorizer *regexp.Regexp + randomAgent bool coloredOutput bool debug bool Results bool @@ -71,18 +73,7 @@ type HTTPExecuter struct { // HTTPOptions contains configuration options for the HTTP executer. type HTTPOptions struct { - CustomHeaders requests.CustomHeaders - ProxyURL string - ProxySocksURL string - Template *templates.Template - BulkHTTPRequest *requests.BulkHTTPRequest - Writer *bufwriter.Writer - Timeout int - Retries int - CookieJar *cookiejar.Jar - Colorizer *colorizer.NucleiColorizer - Decolorizer *regexp.Regexp - TraceLog tracelog.Log + RandomAgent bool Debug bool JSON bool JSONRequests bool @@ -90,6 +81,18 @@ type HTTPOptions struct { CookieReuse bool ColoredOutput bool StopAtFirstMatch bool + Timeout int + Retries int + ProxyURL string + ProxySocksURL string + Template *templates.Template + BulkHTTPRequest *requests.BulkHTTPRequest + Writer *bufwriter.Writer + CustomHeaders requests.CustomHeaders + CookieJar *cookiejar.Jar + Colorizer *colorizer.NucleiColorizer + Decolorizer *regexp.Regexp + TraceLog tracelog.Log PF *projetctfile.ProjectFile RateLimiter ratelimit.Limiter Dialer *fastdialer.Dialer @@ -140,6 +143,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { template: options.Template, bulkHTTPRequest: options.BulkHTTPRequest, writer: options.Writer, + randomAgent: options.RandomAgent, customHeaders: options.CustomHeaders, CookieJar: options.CookieJar, coloredOutput: options.ColoredOutput, @@ -174,16 +178,21 @@ func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result { for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ { swg.Add() // base request + result.Lock() request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { result.Error = err - return result } go func(httpRequest *requests.HTTPRequest) { defer swg.Done() // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "") + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") if err != nil { result.Error = errors.Wrap(err, "could not handle http request") } @@ -214,9 +223,15 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) // Workers that keeps enqueuing new requests maxWorkers := e.bulkHTTPRequest.Threads swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) && !result.Done { + for e.bulkHTTPRequest.Next(reqURL) { + result.Lock() request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { result.Error = err p.Drop(remaining) } else { @@ -227,7 +242,7 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) e.ratelimiter.Take() // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "") + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") if err != nil { e.traceLog.Request(e.template.ID, reqURL, "http", err) result.Error = errors.Wrap(err, "could not handle http request") @@ -284,9 +299,15 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { maxWorkers = pipeOptions.MaxPendingRequests } swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) && !result.Done { + for e.bulkHTTPRequest.Next(reqURL) { + result.Lock() request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { result.Error = err } else { swg.Add() @@ -297,7 +318,7 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { // If the request was built correctly then execute it request.Pipeline = true request.PipelineClient = pipeclient - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "") + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") if err != nil { e.traceLog.Request(e.template.ID, reqURL, "http", err) result.Error = errors.Wrap(err, "could not handle http request") @@ -349,17 +370,23 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result remaining := e.bulkHTTPRequest.GetRequestCount() e.bulkHTTPRequest.CreateGenerator(reqURL) - for e.bulkHTTPRequest.Next(reqURL) && !result.Done { + for e.bulkHTTPRequest.Next(reqURL) { requestNumber++ + result.Lock() httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - if err != nil { + payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { result.Error = err p.Drop(remaining) } else { e.ratelimiter.Take() // If the request was built correctly then execute it format := "%s_" + strconv.Itoa(requestNumber) - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format) + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format) if err != nil { result.Error = errors.Wrap(err, "could not handle http request") p.Drop(remaining) @@ -384,7 +411,13 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result return result } -func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error { +func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error { + // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given + if e.randomAgent { + // nolint:errcheck // ignoring error + e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) + } + e.setCustomHeaders(request) var ( @@ -516,18 +549,36 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, headers := headersToString(resp.Header) + var matchData map[string]interface{} + if payloads != nil { + matchData = generators.MergeMaps(result.historyData, payloads) + } + // store for internal purposes the DSL matcher data // hardcode stopping storing data after defaultMaxHistorydata items if len(result.historyData) < defaultMaxHistorydata { result.Lock() + // update history data with current reqURL and hostname + result.historyData["reqURL"] = reqURL + if parsed, err := url.Parse(reqURL); err == nil { + result.historyData["Hostname"] = parsed.Host + } result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) + if payloads == nil { + // merge them to history data + result.historyData = generators.MergeMaps(result.historyData, payloads) + } + result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) + + // complement match data with new one if necessary + matchData = generators.MergeMaps(matchData, result.historyData) result.Unlock() } matcherCondition := e.bulkHTTPRequest.GetMatchersCondition() for _, matcher := range e.bulkHTTPRequest.Matchers { // Check if the matcher matched - if !matcher.Match(resp, body, headers, duration, result.historyData) { + if !matcher.Match(resp, body, headers, duration, matchData) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { return nil @@ -542,7 +593,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, result.Meta = request.Meta result.GotResults = true result.Unlock() - e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL) + e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL) } } } @@ -573,7 +624,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL) + e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL) result.Lock() result.GotResults = true result.Unlock() @@ -701,7 +752,6 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { type Result struct { sync.Mutex GotResults bool - Done bool Meta map[string]interface{} Matches map[string]interface{} Extractions map[string]interface{} diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go index c40ead5a5..7ce421136 100644 --- a/v2/pkg/executer/output_http.go +++ b/v2/pkg/executer/output_http.go @@ -1,6 +1,7 @@ package executer import ( + "fmt" "net/http" "net/http/httputil" "strings" @@ -119,9 +120,8 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res builder.WriteString(" [") var metas []string - for name, value := range req.Meta { - metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String()) + metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String()) } builder.WriteString(strings.Join(metas, ",")) diff --git a/v2/pkg/generators/dsl.go b/v2/pkg/generators/dsl.go index c8b6792bc..5899ff652 100644 --- a/v2/pkg/generators/dsl.go +++ b/v2/pkg/generators/dsl.go @@ -35,112 +35,112 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { // strings functions["len"] = func(args ...interface{}) (interface{}, error) { - length := len(args[0].(string)) + length := len(toString(args[0])) return float64(length), nil } functions["toupper"] = func(args ...interface{}) (interface{}, error) { - return strings.ToUpper(args[0].(string)), nil + return strings.ToUpper(toString(args[0])), nil } functions["tolower"] = func(args ...interface{}) (interface{}, error) { - return strings.ToLower(args[0].(string)), nil + return strings.ToLower(toString(args[0])), nil } functions["replace"] = func(args ...interface{}) (interface{}, error) { - return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil + return strings.ReplaceAll(toString(args[0]), toString(args[1]), toString(args[2])), nil } functions["replace_regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(args[1].(string)) + compiled, err := regexp.Compile(toString(args[1])) if err != nil { return nil, err } - return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil + return compiled.ReplaceAllString(toString(args[0]), toString(args[2])), nil } functions["trim"] = func(args ...interface{}) (interface{}, error) { - return strings.Trim(args[0].(string), args[2].(string)), nil + return strings.Trim(toString(args[0]), toString(args[2])), nil } functions["trimleft"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimLeft(args[0].(string), args[1].(string)), nil + return strings.TrimLeft(toString(args[0]), toString(args[1])), nil } functions["trimright"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimRight(args[0].(string), args[1].(string)), nil + return strings.TrimRight(toString(args[0]), toString(args[1])), nil } functions["trimspace"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSpace(args[0].(string)), nil + return strings.TrimSpace(toString(args[0])), nil } functions["trimprefix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimPrefix(args[0].(string), args[1].(string)), nil + return strings.TrimPrefix(toString(args[0]), toString(args[1])), nil } functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSuffix(args[0].(string), args[1].(string)), nil + return strings.TrimSuffix(toString(args[0]), toString(args[1])), nil } functions["reverse"] = func(args ...interface{}) (interface{}, error) { - return reverseString(args[0].(string)), nil + return reverseString(toString(args[0])), nil } // encoding functions["base64"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string))) + sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0]))) return sEnc, nil } // python encodes to base64 with lines of 76 bytes terminated by new line "\n" functions["base64_py"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string))) + sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0]))) return insertInto(sEnc, 76, '\n'), nil } functions["base64_decode"] = func(args ...interface{}) (interface{}, error) { - return base64.StdEncoding.DecodeString(args[0].(string)) + return base64.StdEncoding.DecodeString(toString(args[0])) } functions["url_encode"] = func(args ...interface{}) (interface{}, error) { - return url.PathEscape(args[0].(string)), nil + return url.PathEscape(toString(args[0])), nil } functions["url_decode"] = func(args ...interface{}) (interface{}, error) { - return url.PathUnescape(args[0].(string)) + return url.PathUnescape(toString(args[0])) } functions["hex_encode"] = func(args ...interface{}) (interface{}, error) { - return hex.EncodeToString([]byte(args[0].(string))), nil + return hex.EncodeToString([]byte(toString(args[0]))), nil } functions["hex_decode"] = func(args ...interface{}) (interface{}, error) { - hx, _ := hex.DecodeString(args[0].(string)) + hx, _ := hex.DecodeString(toString(args[0])) return string(hx), nil } functions["html_escape"] = func(args ...interface{}) (interface{}, error) { - return html.EscapeString(args[0].(string)), nil + return html.EscapeString(toString(args[0])), nil } functions["html_unescape"] = func(args ...interface{}) (interface{}, error) { - return html.UnescapeString(args[0].(string)), nil + return html.UnescapeString(toString(args[0])), nil } // hashing functions["md5"] = func(args ...interface{}) (interface{}, error) { - hash := md5.Sum([]byte(args[0].(string))) + hash := md5.Sum([]byte(toString(args[0]))) return hex.EncodeToString(hash[:]), nil } functions["sha256"] = func(args ...interface{}) (interface{}, error) { h := sha256.New() - _, err := h.Write([]byte(args[0].(string))) + _, err := h.Write([]byte(toString(args[0]))) if err != nil { return nil, err @@ -151,7 +151,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { functions["sha1"] = func(args ...interface{}) (interface{}, error) { h := sha1.New() - _, err := h.Write([]byte(args[0].(string))) + _, err := h.Write([]byte(toString(args[0]))) if err != nil { return nil, err @@ -161,21 +161,21 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { } functions["mmh3"] = func(args ...interface{}) (interface{}, error) { - return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil + return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(toString(args[0])), 0))), nil } // search functions["contains"] = func(args ...interface{}) (interface{}, error) { - return strings.Contains(args[0].(string), args[1].(string)), nil + return strings.Contains(toString(args[0]), toString(args[1])), nil } functions["regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(args[0].(string)) + compiled, err := regexp.Compile(toString(args[0])) if err != nil { return nil, err } - return compiled.MatchString(args[1].(string)), nil + return compiled.MatchString(toString(args[1])), nil } // random generators @@ -183,10 +183,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { chars := letters + numbers bad := "" if len(args) >= 1 { - chars = args[0].(string) + chars = toString(args[0]) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -203,10 +203,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } if len(args) >= withBaseRandArgsSize { - base = args[2].(string) + base = toString(args[2]) } base = TrimAll(base, bad) @@ -223,7 +223,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -240,7 +240,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -257,7 +257,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = args[1].(string) + bad = toString(args[1]) } chars = TrimAll(chars, bad) @@ -289,7 +289,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { // Collaborator functions["collab"] = func(args ...interface{}) (interface{}, error) { // check if collaborator contains a specific pattern - return collaborator.DefaultCollaborator.Has(args[0].(string)), nil + return collaborator.DefaultCollaborator.Has(toString(args[0])), nil } return functions diff --git a/v2/pkg/generators/util.go b/v2/pkg/generators/util.go index 3446c1a2a..b7761a546 100644 --- a/v2/pkg/generators/util.go +++ b/v2/pkg/generators/util.go @@ -199,3 +199,7 @@ func insertInto(s string, interval int, sep rune) string { buffer.WriteRune(sep) return buffer.String() } + +func toString(v interface{}) string { + return fmt.Sprint(v) +} diff --git a/v2/pkg/matchers/match.go b/v2/pkg/matchers/match.go index e33262e4b..a2a258409 100644 --- a/v2/pkg/matchers/match.go +++ b/v2/pkg/matchers/match.go @@ -199,7 +199,7 @@ func (m *Matcher) matchBinary(corpus string) bool { // matchDSL matches on a generic map result func (m *Matcher) matchDSL(mp map[string]interface{}) bool { - // Iterate over all the regexes accepted as valid + // Iterate over all the expressions accepted as valid for i, expression := range m.dslCompiled { result, err := expression.Evaluate(mp) if err != nil { diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index b90a0836f..907e62418 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -184,7 +184,12 @@ func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, d r.gsfm.InitOrSkip(baseURL) r.ReadOne(baseURL) - return r.handleRawWithPaylods(ctx, data, baseURL, values, r.gsfm.Value(baseURL)) + payloads, err := r.GetPayloadsValues(baseURL) + if err != nil { + return nil, err + } + + return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) } // otherwise continue with normal flow @@ -202,7 +207,7 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL dynamicValues := make(map[string]interface{}) // find all potentials tokens between {{}} - var re = regexp.MustCompile(`(?m)\{\{.+}}`) + var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) for _, match := range re.FindAllString(raw, -1) { // check if the match contains a dynamic variable expr := generators.TrimDelimiters(match) @@ -400,7 +405,14 @@ func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest, value = p[1] } - rawRequest.Headers[key] = value + // in case of unsafe requests multiple headers should be accepted + // therefore use the full line as key + _, found := rawRequest.Headers[key] + if r.Unsafe && found { + rawRequest.Headers[line] = "" + } else { + rawRequest.Headers[key] = value + } } // Handle case with the full http url in path. In that case, @@ -482,3 +494,40 @@ func (r *BulkHTTPRequest) Total() int { func (r *BulkHTTPRequest) Increment(reqURL string) { r.gsfm.Increment(reqURL) } + +// GetPayloadsValues for the specified URL +func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface{}, error) { + payloadProcessedValues := make(map[string]interface{}) + payloadsFromTemplate := r.gsfm.Value(reqURL) + for k, v := range payloadsFromTemplate { + kexp := v.(string) + // if it doesn't containing markups, we just continue + if !hasMarker(kexp) { + payloadProcessedValues[k] = v + continue + } + // attempts to expand expressions + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions()) + if err != nil { + // it is a simple literal payload => proceed with literal value + payloadProcessedValues[k] = v + continue + } + // it is an expression - try to solve it + expValue, err := compiled.Evaluate(payloadsFromTemplate) + if err != nil { + // an error occurred => proceed with literal value + payloadProcessedValues[k] = v + continue + } + payloadProcessedValues[k] = fmt.Sprint(expValue) + } + var err error + if len(payloadProcessedValues) == 0 { + err = ErrNoPayload + } + return payloadProcessedValues, err +} + +// ErrNoPayload error to avoid the additional base null request +var ErrNoPayload = fmt.Errorf("no payload found") diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go index bdd633a83..ce8ba795b 100644 --- a/v2/pkg/requests/util.go +++ b/v2/pkg/requests/util.go @@ -71,3 +71,7 @@ func ExpandMapValues(m map[string]string) (m1 map[string][]string) { } return } + +func hasMarker(s string) bool { + return strings.Contains(s, markerParenthesisOpen) || strings.Contains(s, markerParenthesisClose) || strings.Contains(s, markerGeneral) +} diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 9af658896..02765564b 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -66,7 +66,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount()) for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP { - // apply externally supplied payloads if any + // apply externally supplied headers if any request.Headers = generators.MergeMapsWithStrings(request.Headers, headers) // apply externally supplied payloads if any request.Payloads = generators.MergeMaps(request.Payloads, externalVars)