Merge pull request #447 from projectdiscovery/iceman-add-metrics

Added simple json based http metrics support

v2/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/segmentio/ksuid v1.0.3
 	github.com/spaolacci/murmur3 v1.1.0
+	github.com/stretchr/testify v1.5.1
 	go.uber.org/ratelimit v0.1.0
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	gopkg.in/yaml.v2 v2.4.0

v2/go.sum

@@ -14,6 +14,7 @@ github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
 github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
 github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
@@ -51,6 +52,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
 github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
@@ -78,6 +80,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=

v2/internal/progress/progress.go

@@ -1,8 +1,13 @@
 package progress
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"net"
+	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -13,12 +18,13 @@ import (
 // Progress is a progress instance for showing program stats
 type Progress struct {
 	active       bool
-	stats        clistats.StatisticsClient
 	tickDuration time.Duration
+	stats        clistats.StatisticsClient
+	server       *http.Server
 }
 
 // NewProgress creates and returns a new progress tracking object.
-func NewProgress(active bool) *Progress {
+func NewProgress(active, metrics bool, port int) (*Progress, error) {
 	var tickDuration time.Duration
 	if active {
 		tickDuration = 5 * time.Second
@@ -26,29 +32,44 @@ func NewProgress(active bool) *Progress {
 		tickDuration = -1
 	}
 
-	var progress Progress
+	progress := &Progress{}
-	if active {
-		stats, err := clistats.New()
-		if err != nil {
-			gologger.Warningf("Couldn't create progress engine: %s\n", err)
-		}
-		progress.active = active
-		progress.stats = stats
-		progress.tickDuration = tickDuration
-	}
 
-	return &progress
+	stats, err := clistats.New()
+	if err != nil {
+		return nil, err
+	}
+	progress.active = active
+	progress.stats = stats
+	progress.tickDuration = tickDuration
+
+	if metrics {
+		http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
+			metrics := progress.getMetrics()
+			_ = json.NewEncoder(w).Encode(metrics)
+		})
+		progress.server = &http.Server{
+			Addr:    net.JoinHostPort("127.0.0.1", strconv.Itoa(port)),
+			Handler: http.DefaultServeMux,
+		}
+		go func() {
+			if err := progress.server.ListenAndServe(); err != nil {
+				gologger.Warningf("Could not serve metrics: %s\n", err)
+			}
+		}()
+	}
+	return progress, nil
 }
 
 // Init initializes the progress display mechanism by setting counters, etc.
 func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
+	p.stats.AddStatic("templates", rulesCount)
+	p.stats.AddStatic("hosts", hostCount)
+	p.stats.AddStatic("startedAt", time.Now())
+	p.stats.AddCounter("requests", uint64(0))
+	p.stats.AddCounter("errors", uint64(0))
+	p.stats.AddCounter("total", uint64(requestCount))
+
 	if p.active {
-		p.stats.AddStatic("templates", rulesCount)
-		p.stats.AddStatic("hosts", hostCount)
-		p.stats.AddStatic("startedAt", time.Now())
-		p.stats.AddCounter("requests", uint64(0))
-		p.stats.AddCounter("errors", uint64(0))
-		p.stats.AddCounter("total", uint64(requestCount))
 		if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
 			gologger.Warningf("Couldn't start statistics: %s\n", err)
 		}
@@ -57,25 +78,19 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
 
 // AddToTotal adds a value to the total request count
 func (p *Progress) AddToTotal(delta int64) {
-	if p.active {
+	p.stats.IncrementCounter("total", int(delta))
-		p.stats.IncrementCounter("total", int(delta))
-	}
 }
 
 // Update progress tracking information and increments the request counter by one unit.
 func (p *Progress) Update() {
-	if p.active {
+	p.stats.IncrementCounter("requests", 1)
-		p.stats.IncrementCounter("requests", 1)
-	}
 }
 
 // Drop drops the specified number of requests from the progress bar total.
 // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
 func (p *Progress) Drop(count int64) {
-	if p.active {
+	// mimic dropping by incrementing the completed requests
-		// mimic dropping by incrementing the completed requests
+	p.stats.IncrementCounter("errors", int(count))
-		p.stats.IncrementCounter("errors", int(count))
-	}
 }
 
 const bufferSize = 128
@@ -125,6 +140,34 @@ func makePrintCallback() func(stats clistats.StatisticsClient) {
 	}
 }
 
+// getMetrics returns a map of important metrics for client
+func (p *Progress) getMetrics() map[string]interface{} {
+	results := make(map[string]interface{})
+
+	startedAt, _ := p.stats.GetStatic("startedAt")
+	duration := time.Since(startedAt.(time.Time))
+
+	results["startedAt"] = startedAt.(time.Time)
+	results["duration"] = fmtDuration(duration)
+	templates, _ := p.stats.GetStatic("templates")
+	results["templates"] = clistats.String(templates)
+	hosts, _ := p.stats.GetStatic("hosts")
+	results["hosts"] = clistats.String(hosts)
+	requests, _ := p.stats.GetCounter("requests")
+	results["requests"] = clistats.String(requests)
+	total, _ := p.stats.GetCounter("total")
+	results["total"] = clistats.String(total)
+	results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds()))
+	errors, _ := p.stats.GetCounter("errors")
+	results["errors"] = clistats.String(errors)
+
+	//nolint:gomnd // this is not a magic number
+	percentData := (float64(requests) * float64(100)) / float64(total)
+	percent := clistats.String(uint64(percentData))
+	results["percent"] = percent
+	return results
+}
+
 // fmtDuration formats the duration for the time elapsed
 func fmtDuration(d time.Duration) string {
 	d = d.Round(time.Second)
@@ -143,4 +186,5 @@ func (p *Progress) Stop() {
 			gologger.Warningf("Couldn't stop statistics: %s\n", err)
 		}
 	}
+	_ = p.server.Shutdown(context.Background())
 }

v2/internal/runner/options.go

@@ -12,9 +12,9 @@ import (
 
 // Options contains the configuration options for tuning
 // the template requesting process.
-// nolint // false positive, options are allocated once and are necessary as is
 type Options struct {
-	MaxWorkflowDuration  int                    // MaxWorkflowDuration is the maximum time a workflow can run for a URL
+	RandomAgent          bool                   // Generate random User-Agent
+	Metrics              bool                   // Metrics enables display of metrics via an http endpoint
 	Sandbox              bool                   // Sandbox mode allows users to run isolated workflows with system commands disabled
 	Debug                bool                   // Debug mode allows debugging request/responses for the engine
 	Silent               bool                   // Silent suppresses any extra text and only writes found URLs on screen.
@@ -30,13 +30,17 @@ type Options struct {
 	Stdin                bool                   // Stdin specifies whether stdin input was given to the process
 	StopAtFirstMatch     bool                   // Stop processing template at first full match (this may break chained requests)
 	NoMeta               bool                   // Don't display metadata for the matches
+	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
+	MetricsPort          int                    // MetricsPort is the port to show metrics on
+	MaxWorkflowDuration  int                    // MaxWorkflowDuration is the maximum time a workflow can run for a URL
 	BulkSize             int                    // Number of targets analyzed in parallel for each template
 	TemplateThreads      int                    // Number of templates executed in parallel
-	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
-	ProjectPath          string                 // Nuclei uses a user defined project folder
 	Timeout              int                    // Timeout is the seconds to wait for a response from the server.
 	Retries              int                    // Retries is the number of times to retry the request
 	RateLimit            int                    // Rate-Limit of requests per specified target
+	Threads              int                    // Thread controls the number of concurrent requests to make.
+	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
+	ProjectPath          string                 // Nuclei uses a user defined project folder
 	Severity             string                 // Filter templates based on their severity and only run the matching ones.
 	Target               string                 // Target is a single URL/Domain to scan usng a template
 	Targets              string                 // Targets specifies the targets to scan using templates.
@@ -47,10 +51,7 @@ type Options struct {
 	TraceLogFile         string                 // TraceLogFile specifies a file to write with the trace of all requests
 	Templates            multiStringFlag        // Signature specifies the template/templates to use
 	ExcludedTemplates    multiStringFlag        // Signature specifies the template/templates to exclude
-	RandomAgent          bool                   // Generate random User-Agent
 	CustomHeaders        requests.CustomHeaders // Custom global headers
-	Threads              int                    // Thread controls the number of concurrent requests to make.
-	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
 }
 
 type multiStringFlag []string
@@ -69,6 +70,8 @@ func ParseOptions() *Options {
 	options := &Options{}
 
 	flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode")
+	flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
+	flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
 	flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes")
 	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
 	flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.")

v2/internal/runner/runner.go

@@ -173,7 +173,11 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	// Creates the progress tracking object
-	runner.progress = progress.NewProgress(options.EnableProgressBar)
+	var progressErr error
+	runner.progress, progressErr = progress.NewProgress(options.EnableProgressBar, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
 
 	// create project file if requested or load existing one
 	if options.Project {

v2/pkg/collaborator/collaborator.go

@@ -57,7 +57,6 @@ func (b *BurpCollaborator) Has(s string) (found bool) {
 				b.Unlock()
 				break
 			}
-
 		}
 	}
 

v2/pkg/executer/executer_http.go

@@ -73,19 +73,7 @@ type HTTPExecuter struct {
 
 // HTTPOptions contains configuration options for the HTTP executer.
 type HTTPOptions struct {
-	CustomHeaders    requests.CustomHeaders
 	RandomAgent      bool
-	ProxyURL         string
-	ProxySocksURL    string
-	Template         *templates.Template
-	BulkHTTPRequest  *requests.BulkHTTPRequest
-	Writer           *bufwriter.Writer
-	Timeout          int
-	Retries          int
-	CookieJar        *cookiejar.Jar
-	Colorizer        *colorizer.NucleiColorizer
-	Decolorizer      *regexp.Regexp
-	TraceLog         tracelog.Log
 	Debug            bool
 	JSON             bool
 	JSONRequests     bool
@@ -93,6 +81,18 @@ type HTTPOptions struct {
 	CookieReuse      bool
 	ColoredOutput    bool
 	StopAtFirstMatch bool
+	Timeout          int
+	Retries          int
+	ProxyURL         string
+	ProxySocksURL    string
+	Template         *templates.Template
+	BulkHTTPRequest  *requests.BulkHTTPRequest
+	Writer           *bufwriter.Writer
+	CustomHeaders    requests.CustomHeaders
+	CookieJar        *cookiejar.Jar
+	Colorizer        *colorizer.NucleiColorizer
+	Decolorizer      *regexp.Regexp
+	TraceLog         tracelog.Log
 	PF               *projetctfile.ProjectFile
 	RateLimiter      ratelimit.Limiter
 	Dialer           *fastdialer.Dialer

v2/pkg/requests/bulk-http-request.go

@@ -501,7 +501,7 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface
 	payloadsFromTemplate := r.gsfm.Value(reqURL)
 	for k, v := range payloadsFromTemplate {
 		kexp := v.(string)
-		// if it doesn't containg markups, we just continue
+		// if it doesn't containing markups, we just continue
 		if !hasMarker(kexp) {
 			payloadProcessedValues[k] = v
 			continue
@@ -530,4 +530,4 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface
 }
 
 // ErrNoPayload error to avoid the additional base null request
-var ErrNoPayload = fmt.Errorf("No payload found")
+var ErrNoPayload = fmt.Errorf("no payload found")