diff --git a/v2/go.mod b/v2/go.mod index cb2bc84cd..1f63ac369 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -24,6 +24,7 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/segmentio/ksuid v1.0.3 github.com/spaolacci/murmur3 v1.1.0 + github.com/stretchr/testify v1.5.1 go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b gopkg.in/yaml.v2 v2.4.0 diff --git a/v2/go.sum b/v2/go.sum index e33e59472..ac2df7df3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -14,6 +14,7 @@ github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= @@ -51,6 +52,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0= github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= @@ -78,6 +80,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 6c12be52f..a9b5c2ead 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -1,8 +1,13 @@ package progress import ( + "context" + "encoding/json" "fmt" + "net" + "net/http" "os" + "strconv" "strings" "time" @@ -13,12 +18,13 @@ import ( // Progress is a progress instance for showing program stats type Progress struct { active bool - stats clistats.StatisticsClient tickDuration time.Duration + stats clistats.StatisticsClient + server *http.Server } // NewProgress creates and returns a new progress tracking object. -func NewProgress(active bool) *Progress { +func NewProgress(active, metrics bool, port int) (*Progress, error) { var tickDuration time.Duration if active { tickDuration = 5 * time.Second @@ -26,29 +32,44 @@ func NewProgress(active bool) *Progress { tickDuration = -1 } - var progress Progress - if active { - stats, err := clistats.New() - if err != nil { - gologger.Warningf("Couldn't create progress engine: %s\n", err) - } - progress.active = active - progress.stats = stats - progress.tickDuration = tickDuration - } + progress := &Progress{} - return &progress + stats, err := clistats.New() + if err != nil { + return nil, err + } + progress.active = active + progress.stats = stats + progress.tickDuration = tickDuration + + if metrics { + http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { + metrics := progress.getMetrics() + _ = json.NewEncoder(w).Encode(metrics) + }) + progress.server = &http.Server{ + Addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), + Handler: http.DefaultServeMux, + } + go func() { + if err := progress.server.ListenAndServe(); err != nil { + gologger.Warningf("Could not serve metrics: %s\n", err) + } + }() + } + return progress, nil } // Init initializes the progress display mechanism by setting counters, etc. func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { + p.stats.AddStatic("templates", rulesCount) + p.stats.AddStatic("hosts", hostCount) + p.stats.AddStatic("startedAt", time.Now()) + p.stats.AddCounter("requests", uint64(0)) + p.stats.AddCounter("errors", uint64(0)) + p.stats.AddCounter("total", uint64(requestCount)) + if p.active { - p.stats.AddStatic("templates", rulesCount) - p.stats.AddStatic("hosts", hostCount) - p.stats.AddStatic("startedAt", time.Now()) - p.stats.AddCounter("requests", uint64(0)) - p.stats.AddCounter("errors", uint64(0)) - p.stats.AddCounter("total", uint64(requestCount)) if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil { gologger.Warningf("Couldn't start statistics: %s\n", err) } @@ -57,25 +78,19 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { // AddToTotal adds a value to the total request count func (p *Progress) AddToTotal(delta int64) { - if p.active { - p.stats.IncrementCounter("total", int(delta)) - } + p.stats.IncrementCounter("total", int(delta)) } // Update progress tracking information and increments the request counter by one unit. func (p *Progress) Update() { - if p.active { - p.stats.IncrementCounter("requests", 1) - } + p.stats.IncrementCounter("requests", 1) } // Drop drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. func (p *Progress) Drop(count int64) { - if p.active { - // mimic dropping by incrementing the completed requests - p.stats.IncrementCounter("errors", int(count)) - } + // mimic dropping by incrementing the completed requests + p.stats.IncrementCounter("errors", int(count)) } const bufferSize = 128 @@ -125,6 +140,34 @@ func makePrintCallback() func(stats clistats.StatisticsClient) { } } +// getMetrics returns a map of important metrics for client +func (p *Progress) getMetrics() map[string]interface{} { + results := make(map[string]interface{}) + + startedAt, _ := p.stats.GetStatic("startedAt") + duration := time.Since(startedAt.(time.Time)) + + results["startedAt"] = startedAt.(time.Time) + results["duration"] = fmtDuration(duration) + templates, _ := p.stats.GetStatic("templates") + results["templates"] = clistats.String(templates) + hosts, _ := p.stats.GetStatic("hosts") + results["hosts"] = clistats.String(hosts) + requests, _ := p.stats.GetCounter("requests") + results["requests"] = clistats.String(requests) + total, _ := p.stats.GetCounter("total") + results["total"] = clistats.String(total) + results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds())) + errors, _ := p.stats.GetCounter("errors") + results["errors"] = clistats.String(errors) + + //nolint:gomnd // this is not a magic number + percentData := (float64(requests) * float64(100)) / float64(total) + percent := clistats.String(uint64(percentData)) + results["percent"] = percent + return results +} + // fmtDuration formats the duration for the time elapsed func fmtDuration(d time.Duration) string { d = d.Round(time.Second) @@ -143,4 +186,5 @@ func (p *Progress) Stop() { gologger.Warningf("Couldn't stop statistics: %s\n", err) } } + _ = p.server.Shutdown(context.Background()) } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 8dfa7f190..b00bb485f 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -12,9 +12,9 @@ import ( // Options contains the configuration options for tuning // the template requesting process. -// nolint // false positive, options are allocated once and are necessary as is type Options struct { - MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL + RandomAgent bool // Generate random User-Agent + Metrics bool // Metrics enables display of metrics via an http endpoint Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled Debug bool // Debug mode allows debugging request/responses for the engine Silent bool // Silent suppresses any extra text and only writes found URLs on screen. @@ -30,13 +30,17 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests) NoMeta bool // Don't display metadata for the matches + Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times + MetricsPort int // MetricsPort is the port to show metrics on + MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL BulkSize int // Number of targets analyzed in parallel for each template TemplateThreads int // Number of templates executed in parallel - Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times - ProjectPath string // Nuclei uses a user defined project folder Timeout int // Timeout is the seconds to wait for a response from the server. Retries int // Retries is the number of times to retry the request RateLimit int // Rate-Limit of requests per specified target + Threads int // Thread controls the number of concurrent requests to make. + BurpCollaboratorBiid string // Burp Collaborator BIID for polling + ProjectPath string // Nuclei uses a user defined project folder Severity string // Filter templates based on their severity and only run the matching ones. Target string // Target is a single URL/Domain to scan usng a template Targets string // Targets specifies the targets to scan using templates. @@ -47,10 +51,7 @@ type Options struct { TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests Templates multiStringFlag // Signature specifies the template/templates to use ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude - RandomAgent bool // Generate random User-Agent CustomHeaders requests.CustomHeaders // Custom global headers - Threads int // Thread controls the number of concurrent requests to make. - BurpCollaboratorBiid string // Burp Collaborator BIID for polling } type multiStringFlag []string @@ -69,6 +70,8 @@ func ParseOptions() *Options { options := &Options{} flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") + flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") + flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes") flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 46c7d4e88..79ded4bf0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -173,7 +173,11 @@ func New(options *Options) (*Runner, error) { } // Creates the progress tracking object - runner.progress = progress.NewProgress(options.EnableProgressBar) + var progressErr error + runner.progress, progressErr = progress.NewProgress(options.EnableProgressBar, options.Metrics, options.MetricsPort) + if progressErr != nil { + return nil, progressErr + } // create project file if requested or load existing one if options.Project { diff --git a/v2/pkg/collaborator/collaborator.go b/v2/pkg/collaborator/collaborator.go index cfa434402..4b6383b23 100644 --- a/v2/pkg/collaborator/collaborator.go +++ b/v2/pkg/collaborator/collaborator.go @@ -57,7 +57,6 @@ func (b *BurpCollaborator) Has(s string) (found bool) { b.Unlock() break } - } } diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 2e1475a6f..c8b58b403 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -73,19 +73,7 @@ type HTTPExecuter struct { // HTTPOptions contains configuration options for the HTTP executer. type HTTPOptions struct { - CustomHeaders requests.CustomHeaders RandomAgent bool - ProxyURL string - ProxySocksURL string - Template *templates.Template - BulkHTTPRequest *requests.BulkHTTPRequest - Writer *bufwriter.Writer - Timeout int - Retries int - CookieJar *cookiejar.Jar - Colorizer *colorizer.NucleiColorizer - Decolorizer *regexp.Regexp - TraceLog tracelog.Log Debug bool JSON bool JSONRequests bool @@ -93,6 +81,18 @@ type HTTPOptions struct { CookieReuse bool ColoredOutput bool StopAtFirstMatch bool + Timeout int + Retries int + ProxyURL string + ProxySocksURL string + Template *templates.Template + BulkHTTPRequest *requests.BulkHTTPRequest + Writer *bufwriter.Writer + CustomHeaders requests.CustomHeaders + CookieJar *cookiejar.Jar + Colorizer *colorizer.NucleiColorizer + Decolorizer *regexp.Regexp + TraceLog tracelog.Log PF *projetctfile.ProjectFile RateLimiter ratelimit.Limiter Dialer *fastdialer.Dialer diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index fac066036..907e62418 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -501,7 +501,7 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface payloadsFromTemplate := r.gsfm.Value(reqURL) for k, v := range payloadsFromTemplate { kexp := v.(string) - // if it doesn't containg markups, we just continue + // if it doesn't containing markups, we just continue if !hasMarker(kexp) { payloadProcessedValues[k] = v continue @@ -530,4 +530,4 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface } // ErrNoPayload error to avoid the additional base null request -var ErrNoPayload = fmt.Errorf("No payload found") +var ErrNoPayload = fmt.Errorf("no payload found")