External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:05:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #447 from projectdiscovery/iceman-add-metrics

Browse Source 
Added simple json based http metrics support
This commit is contained in:
Ice3man 2020-12-17 10:10:37 -08:00 committed by GitHub
commit b7475e84f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 106 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
v2/go.mod
View File

@ -24,6 +24,7 @@ require (
github.com/remeh/sizedwaitgroup v1.0.0
github.com/segmentio/ksuid v1.0.3
github.com/spaolacci/murmur3 v1.1.0
github.com/stretchr/testify v1.5.1
go.uber.org/ratelimit v0.1.0
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
gopkg.in/yaml.v2 v2.4.0

3
v2/go.sum
View File

@ -14,6 +14,7 @@ github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
@ -51,6 +52,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
@ -78,6 +80,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=

102
v2/internal/progress/progress.go
View File

@ -1,8 +1,13 @@
package progress
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"strconv"
"strings"
"time"
@ -13,12 +18,13 @@ import (
// Progress is a progress instance for showing program stats
type Progress struct {
active bool
stats clistats.StatisticsClient
tickDuration time.Duration
stats clistats.StatisticsClient
server *http.Server
}
// NewProgress creates and returns a new progress tracking object.
func NewProgress(active bool) *Progress {
func NewProgress(active, metrics bool, port int) (*Progress, error) {
var tickDuration time.Duration
if active {
tickDuration = 5 * time.Second
@ -26,29 +32,44 @@ func NewProgress(active bool) *Progress {
tickDuration = -1
}
var progress Progress
if active {
stats, err := clistats.New()
if err != nil {
gologger.Warningf("Couldn't create progress engine: %s\n", err)
}
progress.active = active
progress.stats = stats
progress.tickDuration = tickDuration
}
progress := &Progress{}
return &progress
stats, err := clistats.New()
if err != nil {
return nil, err
}
progress.active = active
progress.stats = stats
progress.tickDuration = tickDuration
if metrics {
http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
metrics := progress.getMetrics()
_ = json.NewEncoder(w).Encode(metrics)
})
progress.server = &http.Server{
Addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)),
Handler: http.DefaultServeMux,
}
go func() {
if err := progress.server.ListenAndServe(); err != nil {
gologger.Warningf("Could not serve metrics: %s\n", err)
}
}()
}
return progress, nil
}
// Init initializes the progress display mechanism by setting counters, etc.
func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
p.stats.AddStatic("templates", rulesCount)
p.stats.AddStatic("hosts", hostCount)
p.stats.AddStatic("startedAt", time.Now())
p.stats.AddCounter("requests", uint64(0))
p.stats.AddCounter("errors", uint64(0))
p.stats.AddCounter("total", uint64(requestCount))
if p.active {
p.stats.AddStatic("templates", rulesCount)
p.stats.AddStatic("hosts", hostCount)
p.stats.AddStatic("startedAt", time.Now())
p.stats.AddCounter("requests", uint64(0))
p.stats.AddCounter("errors", uint64(0))
p.stats.AddCounter("total", uint64(requestCount))
if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
gologger.Warningf("Couldn't start statistics: %s\n", err)
}
@ -57,25 +78,19 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
// AddToTotal adds a value to the total request count
func (p *Progress) AddToTotal(delta int64) {
if p.active {
p.stats.IncrementCounter("total", int(delta))
}
p.stats.IncrementCounter("total", int(delta))
}
// Update progress tracking information and increments the request counter by one unit.
func (p *Progress) Update() {
if p.active {
p.stats.IncrementCounter("requests", 1)
}
p.stats.IncrementCounter("requests", 1)
}
// Drop drops the specified number of requests from the progress bar total.
// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
func (p *Progress) Drop(count int64) {
if p.active {
// mimic dropping by incrementing the completed requests
p.stats.IncrementCounter("errors", int(count))
}
// mimic dropping by incrementing the completed requests
p.stats.IncrementCounter("errors", int(count))
}
const bufferSize = 128
@ -125,6 +140,34 @@ func makePrintCallback() func(stats clistats.StatisticsClient) {
}
}
// getMetrics returns a map of important metrics for client
func (p *Progress) getMetrics() map[string]interface{} {
results := make(map[string]interface{})
startedAt, _ := p.stats.GetStatic("startedAt")
duration := time.Since(startedAt.(time.Time))
results["startedAt"] = startedAt.(time.Time)
results["duration"] = fmtDuration(duration)
templates, _ := p.stats.GetStatic("templates")
results["templates"] = clistats.String(templates)
hosts, _ := p.stats.GetStatic("hosts")
results["hosts"] = clistats.String(hosts)
requests, _ := p.stats.GetCounter("requests")
results["requests"] = clistats.String(requests)
total, _ := p.stats.GetCounter("total")
results["total"] = clistats.String(total)
results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds()))
errors, _ := p.stats.GetCounter("errors")
results["errors"] = clistats.String(errors)
//nolint:gomnd // this is not a magic number
percentData := (float64(requests) * float64(100)) / float64(total)
percent := clistats.String(uint64(percentData))
results["percent"] = percent
return results
}
// fmtDuration formats the duration for the time elapsed
func fmtDuration(d time.Duration) string {
d = d.Round(time.Second)
@ -143,4 +186,5 @@ func (p *Progress) Stop() {
gologger.Warningf("Couldn't stop statistics: %s\n", err)
}
}
_ = p.server.Shutdown(context.Background())
}

17
v2/internal/runner/options.go
View File

@ -12,9 +12,9 @@ import (
// Options contains the configuration options for tuning
// the template requesting process.
// nolint // false positive, options are allocated once and are necessary as is
type Options struct {
MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL
RandomAgent bool // Generate random User-Agent
Metrics bool // Metrics enables display of metrics via an http endpoint
Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled
Debug bool // Debug mode allows debugging request/responses for the engine
Silent bool // Silent suppresses any extra text and only writes found URLs on screen.
@ -30,13 +30,17 @@ type Options struct {
Stdin bool // Stdin specifies whether stdin input was given to the process
StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests)
NoMeta bool // Don't display metadata for the matches
Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times
MetricsPort int // MetricsPort is the port to show metrics on
MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL
BulkSize int // Number of targets analyzed in parallel for each template
TemplateThreads int // Number of templates executed in parallel
Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times
ProjectPath string // Nuclei uses a user defined project folder
Timeout int // Timeout is the seconds to wait for a response from the server.
Retries int // Retries is the number of times to retry the request
RateLimit int // Rate-Limit of requests per specified target
Threads int // Thread controls the number of concurrent requests to make.
BurpCollaboratorBiid string // Burp Collaborator BIID for polling
ProjectPath string // Nuclei uses a user defined project folder
Severity string // Filter templates based on their severity and only run the matching ones.
Target string // Target is a single URL/Domain to scan usng a template
Targets string // Targets specifies the targets to scan using templates.
@ -47,10 +51,7 @@ type Options struct {
TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests
Templates multiStringFlag // Signature specifies the template/templates to use
ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude
RandomAgent bool // Generate random User-Agent
CustomHeaders requests.CustomHeaders // Custom global headers
Threads int // Thread controls the number of concurrent requests to make.
BurpCollaboratorBiid string // Burp Collaborator BIID for polling
}
type multiStringFlag []string
@ -69,6 +70,8 @@ func ParseOptions() *Options {
options := &Options{}
flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode")
flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes")
flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.")

6
v2/internal/runner/runner.go
View File

@ -173,7 +173,11 @@ func New(options *Options) (*Runner, error) {
}
// Creates the progress tracking object
runner.progress = progress.NewProgress(options.EnableProgressBar)
var progressErr error
runner.progress, progressErr = progress.NewProgress(options.EnableProgressBar, options.Metrics, options.MetricsPort)
if progressErr != nil {
return nil, progressErr
}
// create project file if requested or load existing one
if options.Project {

1
v2/pkg/collaborator/collaborator.go
View File

@ -57,7 +57,6 @@ func (b *BurpCollaborator) Has(s string) (found bool) {
b.Unlock()
break
}
}
}

24
v2/pkg/executer/executer_http.go
View File

@ -73,19 +73,7 @@ type HTTPExecuter struct {
// HTTPOptions contains configuration options for the HTTP executer.
type HTTPOptions struct {
CustomHeaders requests.CustomHeaders
RandomAgent bool
ProxyURL string
ProxySocksURL string
Template *templates.Template
BulkHTTPRequest *requests.BulkHTTPRequest
Writer *bufwriter.Writer
Timeout int
Retries int
CookieJar *cookiejar.Jar
Colorizer *colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
TraceLog tracelog.Log
Debug bool
JSON bool
JSONRequests bool
@ -93,6 +81,18 @@ type HTTPOptions struct {
CookieReuse bool
ColoredOutput bool
StopAtFirstMatch bool
Timeout int
Retries int
ProxyURL string
ProxySocksURL string
Template *templates.Template
BulkHTTPRequest *requests.BulkHTTPRequest
Writer *bufwriter.Writer
CustomHeaders requests.CustomHeaders
CookieJar *cookiejar.Jar
Colorizer *colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
TraceLog tracelog.Log
PF *projetctfile.ProjectFile
RateLimiter ratelimit.Limiter
Dialer *fastdialer.Dialer

4
v2/pkg/requests/bulk-http-request.go
View File

@ -501,7 +501,7 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface
payloadsFromTemplate := r.gsfm.Value(reqURL)
for k, v := range payloadsFromTemplate {
kexp := v.(string)
// if it doesn't containg markups, we just continue
// if it doesn't containing markups, we just continue
if !hasMarker(kexp) {
payloadProcessedValues[k] = v
continue
@ -530,4 +530,4 @@ func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface
}
// ErrNoPayload error to avoid the additional base null request
var ErrNoPayload = fmt.Errorf("No payload found")
var ErrNoPayload = fmt.Errorf("no payload found")