nuclei 'stats' build : scan events + chart utils (#5032)

* prototype new scan events

* scan-event: improvements + conditional build

* add scan charts server: make scan-charts

* scan-charts: bug fix

.gitignore

@@ -36,5 +36,8 @@ pkg/protocols/headless/engine/.cache
 /fuzzplayground
 integration_tests/fuzzplayground
 /dsl.md
+/nuclei-stats
+/nuclei-stats-*
+/scan-charts
 
 

Makefile

@@ -11,9 +11,18 @@ ifneq ($(shell go env GOOS),darwin)
 LDFLAGS := -extldflags "-static"
 endif
     
+.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
+
 all: build
 build:
+	rm -f nuclei 2>/dev/null
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
+build-stats:
+	rm -f nuclei-stats 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
+scan-charts:
+	rm -f scan-charts 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
 docs:
 	if ! which dstdocgen > /dev/null; then
 		echo -e "Command not found! Install? (y/n) \c"

cmd/scan-charts/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/charts"
+)
+
+var (
+	dir     string
+	address string
+	output  string
+)
+
+func main() {
+	flag.StringVar(&dir, "dir", "", "directory to scan")
+	flag.StringVar(&address, "address", ":9000", "address to run the server on")
+	flag.StringVar(&output, "output", "", "output filename of generated html file")
+	flag.Parse()
+
+	if dir == "" {
+		flag.Usage()
+		return
+	}
+
+	server, err := charts.NewScanEventsCharts(dir)
+	if err != nil {
+		panic(err)
+	}
+	server.PrintInfo()
+
+	if output != "" {
+		if err = server.GenerateHTML(output); err != nil {
+			panic(err)
+		}
+		return
+	}
+
+	server.Start(address)
+}

go.mod

@@ -329,6 +329,7 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562
 	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-echarts/go-echarts/v2 v2.3.3
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
@@ -348,3 +349,5 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
+
+replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1

go.sum

@@ -341,6 +341,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg=
+github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
 github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -1013,6 +1015,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
+github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=

internal/runner/runner.go

@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
@@ -553,6 +554,20 @@ func (r *Runner) RunEnumeration() error {
 		executorOpts.InputHelper.InputsHTTP = inputHelpers
 	}
 
+	// initialize stats worker ( this is no-op unless nuclei is built with stats build tag)
+	// during execution a directory with 2 files will be created in the current directory
+	// config.json - containing below info
+	// events.jsonl - containing all start and end times of all templates
+	events.InitWithConfig(&events.ScanConfig{
+		Name:                "nuclei-stats", // make this configurable
+		TargetCount:         int(r.inputProvider.Count()),
+		TemplatesCount:      len(store.Templates()) + len(store.Workflows()),
+		TemplateConcurrency: r.options.TemplateThreads,
+		PayloadConcurrency:  r.options.PayloadConcurrency,
+		JsConcurrency:       r.options.JsConcurrency,
+		Retries:             r.options.Retries,
+	}, "")
+
 	enumeration := false
 	var results *atomic.Bool
 	results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)

pkg/scan/charts/charts.go

@@ -0,0 +1,87 @@
+package charts
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
+	fileutil "github.com/projectdiscovery/utils/file"
+)
+
+// ScanEventsCharts is a struct for nuclei event charts
+type ScanEventsCharts struct {
+	eventsDir string
+	config    *events.ScanConfig
+	data      []events.ScanEvent
+}
+
+func (sc *ScanEventsCharts) PrintInfo() {
+	fmt.Printf("[+] Scan Info\n")
+	fmt.Printf("  - Name: %s\n", sc.config.Name)
+	fmt.Printf("  - Target Count: %d\n", sc.config.TargetCount)
+	fmt.Printf("  - Template Count: %d\n", sc.config.TemplatesCount)
+	fmt.Printf("  - Template Concurrency: %d\n", sc.config.TemplateConcurrency)
+	fmt.Printf("  - Payload Concurrency: %d\n", sc.config.PayloadConcurrency)
+	fmt.Printf("  - Retries: %v\n", sc.config.Retries)
+	fmt.Printf("  - Total Events: %d\n", len(sc.data))
+	fmt.Println()
+}
+
+// NewScanEventsCharts creates a new nuclei event charts
+func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) {
+	sc := &ScanEventsCharts{eventsDir: eventsDir}
+	if !fileutil.FolderExists(eventsDir) {
+		return nil, fmt.Errorf("events directory does not exist")
+	}
+	// open two files
+	// config.json
+	bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile))
+	if err != nil {
+		return nil, err
+	}
+	var config events.ScanConfig
+	err = json.Unmarshal(bin, &config)
+	if err != nil {
+		return nil, err
+	}
+	sc.config = &config
+
+	// events.jsonl
+	f, err := os.Open(filepath.Join(eventsDir, events.EventsFile))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data := []events.ScanEvent{}
+	dec := json.NewDecoder(f)
+	for {
+		var event events.ScanEvent
+		if err := dec.Decode(&event); err != nil {
+			break
+		}
+		data = append(data, event)
+	}
+	sc.data = data
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("no events found in the events file")
+	}
+
+	return sc, nil
+}
+
+// Start starts the nuclei event charts server
+func (sc *ScanEventsCharts) Start(addr string) {
+	e := echo.New()
+	e.HideBanner = true
+	e.GET("/concurrency", sc.ConcurrencyVsTime)
+	e.GET("/requests", sc.TotalRequestsOverTime)
+	e.GET("/slow", sc.TopSlowTemplates)
+	e.GET("/rps", sc.RequestsVSInterval)
+	e.GET("/", sc.AllCharts)
+	e.Logger.Fatal(e.Start(addr))
+}

pkg/scan/charts/echarts.go

@@ -0,0 +1,351 @@
+package charts
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/go-echarts/go-echarts/v2/charts"
+	"github.com/go-echarts/go-echarts/v2/components"
+	"github.com/go-echarts/go-echarts/v2/opts"
+	"github.com/labstack/echo/v4"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
+	sliceutil "github.com/projectdiscovery/utils/slice"
+)
+
+const (
+	TopK         = 50
+	SpacerHeight = "50px"
+)
+
+func (s *ScanEventsCharts) AllCharts(c echo.Context) error {
+	page := s.allCharts(c)
+	return page.Render(c.Response().Writer)
+}
+
+func (s *ScanEventsCharts) GenerateHTML(filePath string) error {
+	page := s.allCharts(nil)
+	output, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	return page.Render(output)
+}
+
+// AllCharts generates all the charts for the scan events and returns a page component
+func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
+	page := components.NewPage()
+	page.PageTitle = "Nuclei Charts"
+	line1 := s.totalRequestsOverTime(c)
+	line1.SetSpacerHeight(SpacerHeight)
+	kline := s.topSlowTemplates(c)
+	kline.SetSpacerHeight(SpacerHeight)
+	line2 := s.requestsVSInterval(c)
+	line2.SetSpacerHeight(SpacerHeight)
+	line3 := s.concurrencyVsTime(c)
+	line3.SetSpacerHeight(SpacerHeight)
+	page.AddCharts(line1, kline, line2, line3)
+	page.Validate()
+	page.SetLayout(components.PageCenterLayout)
+	return page
+}
+
+func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
+	line := s.totalRequestsOverTime(c)
+	return line.Render(c.Response().Writer)
+}
+
+// totalRequestsOverTime generates a line chart showing total requests count over time
+func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
+
+	var startTime time.Time = time.Now()
+	var endTime time.Time
+
+	for _, event := range s.data {
+		if event.Time.Before(startTime) {
+			startTime = event.Time
+		}
+		if event.Time.After(endTime) {
+			endTime = event.Time
+		}
+	}
+	data := getCategoryRequestCount(s.data)
+	max := 0
+	for _, v := range data {
+		if len(v) > max {
+			max = len(v)
+		}
+	}
+	line.SetXAxis(time.Now().Format(time.RFC3339))
+	for k, v := range data {
+		lineData := make([]opts.LineData, 0)
+		temp := 0
+		for _, scanEvent := range v {
+			temp += scanEvent.MaxRequests
+			val := scanEvent.Time.Sub(startTime)
+			lineData = append(lineData, opts.LineData{
+				Value: []interface{}{val.Milliseconds(), temp},
+				Name:  scanEvent.TemplateID,
+			})
+		}
+		line.AddSeries(k, lineData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+	}
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: total-req vs time"}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time", Type: "time", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "Requests Sent", Type: "value"}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
+	kline := s.topSlowTemplates(c)
+	return kline.Render(c.Response().Writer)
+}
+
+// topSlowTemplates generates a Kline chart showing the top slow templates by time taken
+func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
+	kline := charts.NewKLine()
+	kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
+
+	ids := map[string][]int64{}
+	var startTime time.Time = time.Now()
+	for _, event := range s.data {
+		if event.Time.Before(startTime) {
+			startTime = event.Time
+		}
+	}
+	for _, event := range s.data {
+		ids[event.TemplateID] = append(ids[event.TemplateID], event.Time.Sub(startTime).Milliseconds())
+	}
+
+	type entry struct {
+		ID        string
+		KlineData opts.KlineData
+		start     int64
+		end       int64
+	}
+	data := []entry{}
+
+	for a, b := range ids {
+		if len(b) < 2 {
+			continue // Prevents index out of range error
+		}
+		d := entry{
+			ID:        a,
+			KlineData: opts.KlineData{Value: []int64{b[0], b[len(b)-1], b[0], b[len(b)-1]}}, // Adjusted to prevent index out of range error
+			start:     b[0],
+			end:       b[len(b)-1],
+		}
+		data = append(data, d)
+	}
+
+	sort.Slice(data, func(i, j int) bool {
+		return data[i].end-data[i].start > data[j].end-data[j].start
+	})
+
+	x := make([]string, 0)
+	y := make([]opts.KlineData, 0)
+	for _, event := range data[:TopK] {
+		x = append(x, event.ID)
+		y = append(y, event.KlineData)
+	}
+
+	kline.SetXAxis(x).AddSeries("templates", y)
+	kline.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: fmt.Sprintf("Nuclei: Top %v Slow Templates", TopK)}),
+		charts.WithXAxisOpts(opts.XAxis{
+			Type:      "category",
+			Show:      true,
+			AxisLabel: &opts.AxisLabel{Rotate: 90, Show: true, ShowMinLabel: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (value) { return value; }`)},
+		}),
+		charts.WithYAxisOpts(opts.YAxis{
+			Scale:     true,
+			Type:      "value",
+			Show:      true,
+			AxisLabel: &opts.AxisLabel{Show: true, Formatter: opts.FuncOpts(`function (ms) {  return Math.floor(ms/60000) + 'm' + Math.floor((ms/60000 - Math.floor(ms/60000))*60) + 's'; }`)},
+		}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "40%", Top: "10%"}),
+		charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "events.ScanEvent", TriggerOn: "mousemove|click", Enterable: true, Formatter: opts.FuncOpts(`function (params) { return params.name ; }`)}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	return kline
+}
+
+func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
+	line := s.requestsVSInterval(c)
+	return line.Render(c.Response().Writer)
+}
+
+// requestsVSInterval generates a line chart showing requests per second over time
+func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
+
+	sort.Slice(s.data, func(i, j int) bool {
+		return s.data[i].Time.Before(s.data[j].Time)
+	})
+
+	var interval time.Duration
+
+	if c != nil {
+		interval, _ = time.ParseDuration(c.QueryParam("interval"))
+	}
+	if interval <= 3 {
+		interval = 5 * time.Second
+	}
+
+	data := []opts.LineData{}
+	temp := 0
+	if len(s.data) > 0 {
+		orig := s.data[0].Time
+		startTime := orig
+		xaxisData := []int64{}
+		for _, v := range s.data {
+			if v.Time.Sub(startTime) > interval {
+				millisec := v.Time.Sub(orig).Milliseconds()
+				xaxisData = append(xaxisData, millisec)
+				data = append(data, opts.LineData{Value: temp, Name: v.Time.Sub(orig).String()})
+				temp = 0
+				startTime = v.Time
+			}
+			temp += 1
+		}
+		// Handle last interval if exists
+		if temp > 0 {
+			millisec := s.data[len(s.data)-1].Time.Sub(orig).Milliseconds()
+			xaxisData = append(xaxisData, millisec)
+			data = append(data, opts.LineData{Value: temp, Name: s.data[len(s.data)-1].Time.Sub(orig).String()})
+		}
+		line.SetXAxis(xaxisData)
+		line.AddSeries("RPS", data, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+	}
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: Template Execution", Subtitle: "Time Interval: " + interval.String()}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "RPS Value", Type: "value", Show: true}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
+	line := s.concurrencyVsTime(c)
+	return line.Render(c.Response().Writer)
+}
+
+// concurrencyVsTime generates a line chart showing concurrency (total workers) over time
+func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
+
+	dataset := sliceutil.Clone(s.data)
+
+	sort.Slice(dataset, func(i, j int) bool {
+		return dataset[i].Time.Before(dataset[j].Time)
+	})
+
+	var interval time.Duration
+	if c != nil {
+		interval, _ = time.ParseDuration(c.QueryParam("interval"))
+	}
+	if interval <= 3 {
+		interval = 5 * time.Second
+	}
+
+	// create array with time interval as x-axis and worker count as y-axis
+	// entry is a struct with time and poolsize
+	type entry struct {
+		Time     time.Duration
+		poolsize int
+	}
+	allEntries := []entry{}
+
+	dataIndex := 0
+	maxIndex := len(dataset) - 1
+	currEntry := entry{}
+
+	lastTime := dataset[0].Time
+	for dataIndex <= maxIndex {
+		currTime := dataset[dataIndex].Time
+		if currTime.Sub(lastTime) > interval {
+			// next batch
+			currEntry.Time = interval
+			allEntries = append(allEntries, currEntry)
+			lastTime = dataset[dataIndex-1].Time
+		}
+		if dataset[dataIndex].EventType == events.ScanStarted {
+			currEntry.poolsize += 1
+		} else {
+			currEntry.poolsize -= 1
+		}
+		dataIndex += 1
+	}
+
+	plotData := []opts.LineData{}
+	xaxisData := []int64{}
+	tempTime := time.Duration(0)
+	for _, v := range allEntries {
+		tempTime += v.Time
+		plotData = append(plotData, opts.LineData{Value: v.poolsize, Name: tempTime.String()})
+		xaxisData = append(xaxisData, tempTime.Milliseconds())
+	}
+	line.SetXAxis(xaxisData)
+	line.AddSeries("Concurrency", plotData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: WorkerPool", Subtitle: "Time Interval: " + interval.String()}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "Total Workers", Type: "value", Show: true}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+// getCategoryRequestCount returns a map of category and request count
+func getCategoryRequestCount(values []events.ScanEvent) map[string][]events.ScanEvent {
+	mx := make(map[string][]events.ScanEvent)
+	for _, event := range values {
+		mx[event.TemplateType] = append(mx[event.TemplateType], event)
+	}
+	return mx
+}

pkg/scan/events/scan_noop.go

@@ -0,0 +1,11 @@
+//go:build !stats
+// +build !stats
+
+package events
+
+// AddScanEvent is a no-op function
+func AddScanEvent(event ScanEvent) {
+}
+
+func InitWithConfig(config *ScanConfig, statsDirectory string) {
+}

pkg/scan/events/stats_build.go

@@ -0,0 +1,80 @@
+//go:build stats
+// +build stats
+
+package events
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+var _ ScanEventWorker = &ScanStatsWorker{}
+
+var defaultWorker = &ScanStatsWorker{}
+
+// ScanStatsWorker is a worker for scanning stats
+// This tracks basic stats in jsonlines format
+// in given directory or a default directory with name stats_{timestamp} in the current directory
+type ScanStatsWorker struct {
+	config    *ScanConfig
+	m         *sync.Mutex
+	directory string
+	enc       *json.Encoder
+}
+
+// Init initializes the scan stats worker
+func InitWithConfig(config *ScanConfig, statsDirectory string) {
+	currentTime := time.Now().Format("20060102150405")
+	dirName := fmt.Sprintf("nuclei-stats-%s", currentTime)
+	err := os.Mkdir(dirName, 0755)
+	if err != nil {
+		panic(err)
+	}
+	// save the config to the directory
+	bin, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	err = os.WriteFile(filepath.Join(dirName, ConfigFile), bin, 0755)
+	if err != nil {
+		panic(err)
+	}
+	defaultWorker = &ScanStatsWorker{config: config, m: &sync.Mutex{}, directory: dirName}
+	err = defaultWorker.initEventsFile()
+	if err != nil {
+		panic(err)
+	}
+}
+
+// initEventsFile initializes the events file for the worker
+func (s *ScanStatsWorker) initEventsFile() error {
+	f, err := os.Create(filepath.Join(s.directory, EventsFile))
+	if err != nil {
+		return err
+	}
+	s.enc = json.NewEncoder(f)
+	return nil
+}
+
+// AddScanEvent adds a scan event to the worker
+func (s *ScanStatsWorker) AddScanEvent(event ScanEvent) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	err := s.enc.Encode(event)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// AddScanEvent adds a scan event to the worker
+func AddScanEvent(event ScanEvent) {
+	if defaultWorker == nil {
+		return
+	}
+	defaultWorker.AddScanEvent(event)
+}

pkg/scan/events/utils.go

@@ -0,0 +1,45 @@
+package events
+
+import (
+	"time"
+)
+
+type ScanEventWorker interface {
+	// AddScanEvent adds a scan event to the worker
+	AddScanEvent(event ScanEvent)
+}
+
+// Track scan start / finish status
+type ScanStatus string
+
+const (
+	ScanStarted  ScanStatus = "scan_start"
+	ScanFinished ScanStatus = "scan_end"
+)
+
+const (
+	ConfigFile = "config.json"
+	EventsFile = "events.jsonl"
+)
+
+// ScanEvent represents a single scan event with its metadata
+type ScanEvent struct {
+	Target       string     `json:"target" yaml:"target"`
+	TemplateType string     `json:"template_type" yaml:"template_type"`
+	TemplateID   string     `json:"template_id" yaml:"template_id"`
+	TemplatePath string     `json:"template_path" yaml:"template_path"`
+	MaxRequests  int        `json:"max_requests" yaml:"max_requests"`
+	Time         time.Time  `json:"time" yaml:"time"`
+	EventType    ScanStatus `json:"event_type" yaml:"event_type"`
+}
+
+// ScanConfig is only in context of scan event analysis
+type ScanConfig struct {
+	Name                string `json:"name" yaml:"name"`
+	TargetCount         int    `json:"target_count" yaml:"target_count"`
+	TemplatesCount      int    `json:"templates_count" yaml:"templates_count"`
+	TemplateConcurrency int    `json:"template_concurrency" yaml:"template_concurrency"`
+	PayloadConcurrency  int    `json:"payload_concurrency" yaml:"payload_concurrency"`
+	JsConcurrency       int    `json:"js_concurrency" yaml:"js_concurrency"`
+	Retries             int    `json:"retries" yaml:"retries"`
+}

pkg/scan/scan_context.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 )
 
+
 type ScanContextOption func(*ScanContext)
 
 func WithEvents() ScanContextOption {

pkg/tmplexec/exec.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 	"sync/atomic"
+	"time"
 
 	"github.com/dop251/goja"
 	"github.com/projectdiscovery/gologger"
@@ -14,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
@@ -92,6 +94,31 @@ func (e *TemplateExecuter) Requests() int {
 
 // Execute executes the protocol group and returns true or false if results were found.
 func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
+
+	// === when nuclei is built with -tags=stats ===
+	// Note: this is no-op (empty functions) when nuclei is built in normal or without -tags=stats
+	events.AddScanEvent(events.ScanEvent{
+		Target:       ctx.Input.MetaInput.Input,
+		Time:         time.Now(),
+		EventType:    events.ScanStarted,
+		TemplateType: e.getTemplateType(),
+		TemplateID:   e.options.TemplateID,
+		TemplatePath: e.options.TemplatePath,
+		MaxRequests:  e.Requests(),
+	})
+	defer func() {
+		events.AddScanEvent(events.ScanEvent{
+			Target:       ctx.Input.MetaInput.Input,
+			Time:         time.Now(),
+			EventType:    events.ScanFinished,
+			TemplateType: e.getTemplateType(),
+			TemplateID:   e.options.TemplateID,
+			TemplatePath: e.options.TemplatePath,
+			MaxRequests:  e.Requests(),
+		})
+	}()
+	// ==== end of stats ====
+
 	// executed contains status of execution if it was successfully executed or not
 	// doesn't matter if it was matched or not
 	executed := &atomic.Bool{}
@@ -182,3 +209,17 @@ func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.
 	ctx.LogError(err)
 	return ctx.GenerateResult(), err
 }
+
+// getTemplateType returns the template type of the template
+func (e *TemplateExecuter) getTemplateType() string {
+	if len(e.requests) == 0 {
+		return "null"
+	}
+	if e.options.Flow != "" {
+		return "flow"
+	}
+	if len(e.requests) > 1 {
+		return "multiprotocol"
+	}
+	return e.requests[0].Type().String()
+}

pkg/types/types.go

@@ -419,7 +419,9 @@ func DefaultOptions() *Options {
 		BulkSize:                25,
 		TemplateThreads:         25,
 		HeadlessBulkSize:        10,
+		PayloadConcurrency:      25,
 		HeadlessTemplateThreads: 10,
+		ProbeConcurrency:        50,
 		Timeout:                 5,
 		Retries:                 1,
 		MaxHostError:            30,