mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-17 15:55:26 +00:00
nuclei 'stats' build : scan events + chart utils (#5032)
* prototype new scan events * scan-event: improvements + conditional build * add scan charts server: make scan-charts * scan-charts: bug fix
This commit is contained in:
Tarun Koyalwar
GitHub
parent
bec7cb273a
commit
ea2e13a4aa
3
.gitignore
vendored
3
.gitignore
vendored
@ -36,5 +36,8 @@ pkg/protocols/headless/engine/.cache
|
||||
/fuzzplayground
|
||||
integration_tests/fuzzplayground
|
||||
/dsl.md
|
||||
/nuclei-stats
|
||||
/nuclei-stats-*
|
||||
/scan-charts
|
||||
|
||||
|
||||
|
||||
9
Makefile
9
Makefile
@ -11,9 +11,18 @@ ifneq ($(shell go env GOOS),darwin)
|
||||
LDFLAGS := -extldflags "-static"
|
||||
endif
|
||||
|
||||
.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
|
||||
|
||||
all: build
|
||||
build:
|
||||
rm -f nuclei 2>/dev/null
|
||||
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
|
||||
build-stats:
|
||||
rm -f nuclei-stats 2>/dev/null
|
||||
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
|
||||
scan-charts:
|
||||
rm -f scan-charts 2>/dev/null
|
||||
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
|
||||
docs:
|
||||
if ! which dstdocgen > /dev/null; then
|
||||
echo -e "Command not found! Install? (y/n) \c"
|
||||
|
||||
40
cmd/scan-charts/main.go
Normal file
40
cmd/scan-charts/main.go
Normal file
@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan/charts"
|
||||
)
|
||||
|
||||
var (
|
||||
dir string
|
||||
address string
|
||||
output string
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.StringVar(&dir, "dir", "", "directory to scan")
|
||||
flag.StringVar(&address, "address", ":9000", "address to run the server on")
|
||||
flag.StringVar(&output, "output", "", "output filename of generated html file")
|
||||
flag.Parse()
|
||||
|
||||
if dir == "" {
|
||||
flag.Usage()
|
||||
return
|
||||
}
|
||||
|
||||
server, err := charts.NewScanEventsCharts(dir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
server.PrintInfo()
|
||||
|
||||
if output != "" {
|
||||
if err = server.GenerateHTML(output); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
server.Start(address)
|
||||
}
|
||||
3
go.mod
3
go.mod
@ -329,6 +329,7 @@ require (
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/go-echarts/go-echarts/v2 v2.3.3
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
@ -348,3 +349,5 @@ require (
|
||||
|
||||
// https://go.dev/ref/mod#go-mod-file-retract
|
||||
retract v3.2.0 // retract due to broken js protocol issue
|
||||
|
||||
replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1
|
||||
|
||||
4
go.sum
4
go.sum
@ -341,6 +341,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
|
||||
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg=
|
||||
github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
|
||||
github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
|
||||
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
@ -1013,6 +1015,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
|
||||
github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
|
||||
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
|
||||
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
|
||||
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
|
||||
|
||||
@ -18,6 +18,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/installer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
|
||||
uncoverlib "github.com/projectdiscovery/uncover"
|
||||
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
|
||||
"github.com/projectdiscovery/utils/env"
|
||||
@ -553,6 +554,20 @@ func (r *Runner) RunEnumeration() error {
|
||||
executorOpts.InputHelper.InputsHTTP = inputHelpers
|
||||
}
|
||||
|
||||
// initialize stats worker ( this is no-op unless nuclei is built with stats build tag)
|
||||
// during execution a directory with 2 files will be created in the current directory
|
||||
// config.json - containing below info
|
||||
// events.jsonl - containing all start and end times of all templates
|
||||
events.InitWithConfig(&events.ScanConfig{
|
||||
Name: "nuclei-stats", // make this configurable
|
||||
TargetCount: int(r.inputProvider.Count()),
|
||||
TemplatesCount: len(store.Templates()) + len(store.Workflows()),
|
||||
TemplateConcurrency: r.options.TemplateThreads,
|
||||
PayloadConcurrency: r.options.PayloadConcurrency,
|
||||
JsConcurrency: r.options.JsConcurrency,
|
||||
Retries: r.options.Retries,
|
||||
}, "")
|
||||
|
||||
enumeration := false
|
||||
var results *atomic.Bool
|
||||
results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)
|
||||
|
||||
87
pkg/scan/charts/charts.go
Normal file
87
pkg/scan/charts/charts.go
Normal file
@ -0,0 +1,87 @@
|
||||
package charts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
)
|
||||
|
||||
// ScanEventsCharts is a struct for nuclei event charts
|
||||
type ScanEventsCharts struct {
|
||||
eventsDir string
|
||||
config *events.ScanConfig
|
||||
data []events.ScanEvent
|
||||
}
|
||||
|
||||
func (sc *ScanEventsCharts) PrintInfo() {
|
||||
fmt.Printf("[+] Scan Info\n")
|
||||
fmt.Printf(" - Name: %s\n", sc.config.Name)
|
||||
fmt.Printf(" - Target Count: %d\n", sc.config.TargetCount)
|
||||
fmt.Printf(" - Template Count: %d\n", sc.config.TemplatesCount)
|
||||
fmt.Printf(" - Template Concurrency: %d\n", sc.config.TemplateConcurrency)
|
||||
fmt.Printf(" - Payload Concurrency: %d\n", sc.config.PayloadConcurrency)
|
||||
fmt.Printf(" - Retries: %v\n", sc.config.Retries)
|
||||
fmt.Printf(" - Total Events: %d\n", len(sc.data))
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// NewScanEventsCharts creates a new nuclei event charts
|
||||
func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) {
|
||||
sc := &ScanEventsCharts{eventsDir: eventsDir}
|
||||
if !fileutil.FolderExists(eventsDir) {
|
||||
return nil, fmt.Errorf("events directory does not exist")
|
||||
}
|
||||
// open two files
|
||||
// config.json
|
||||
bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var config events.ScanConfig
|
||||
err = json.Unmarshal(bin, &config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sc.config = &config
|
||||
|
||||
// events.jsonl
|
||||
f, err := os.Open(filepath.Join(eventsDir, events.EventsFile))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data := []events.ScanEvent{}
|
||||
dec := json.NewDecoder(f)
|
||||
for {
|
||||
var event events.ScanEvent
|
||||
if err := dec.Decode(&event); err != nil {
|
||||
break
|
||||
}
|
||||
data = append(data, event)
|
||||
}
|
||||
sc.data = data
|
||||
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("no events found in the events file")
|
||||
}
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
// Start starts the nuclei event charts server
|
||||
func (sc *ScanEventsCharts) Start(addr string) {
|
||||
e := echo.New()
|
||||
e.HideBanner = true
|
||||
e.GET("/concurrency", sc.ConcurrencyVsTime)
|
||||
e.GET("/requests", sc.TotalRequestsOverTime)
|
||||
e.GET("/slow", sc.TopSlowTemplates)
|
||||
e.GET("/rps", sc.RequestsVSInterval)
|
||||
e.GET("/", sc.AllCharts)
|
||||
e.Logger.Fatal(e.Start(addr))
|
||||
}
|
||||
351
pkg/scan/charts/echarts.go
Normal file
351
pkg/scan/charts/echarts.go
Normal file
@ -0,0 +1,351 @@
|
||||
package charts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/go-echarts/go-echarts/v2/charts"
|
||||
"github.com/go-echarts/go-echarts/v2/components"
|
||||
"github.com/go-echarts/go-echarts/v2/opts"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
)
|
||||
|
||||
const (
|
||||
TopK = 50
|
||||
SpacerHeight = "50px"
|
||||
)
|
||||
|
||||
func (s *ScanEventsCharts) AllCharts(c echo.Context) error {
|
||||
page := s.allCharts(c)
|
||||
return page.Render(c.Response().Writer)
|
||||
}
|
||||
|
||||
func (s *ScanEventsCharts) GenerateHTML(filePath string) error {
|
||||
page := s.allCharts(nil)
|
||||
output, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return page.Render(output)
|
||||
}
|
||||
|
||||
// AllCharts generates all the charts for the scan events and returns a page component
|
||||
func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
|
||||
page := components.NewPage()
|
||||
page.PageTitle = "Nuclei Charts"
|
||||
line1 := s.totalRequestsOverTime(c)
|
||||
line1.SetSpacerHeight(SpacerHeight)
|
||||
kline := s.topSlowTemplates(c)
|
||||
kline.SetSpacerHeight(SpacerHeight)
|
||||
line2 := s.requestsVSInterval(c)
|
||||
line2.SetSpacerHeight(SpacerHeight)
|
||||
line3 := s.concurrencyVsTime(c)
|
||||
line3.SetSpacerHeight(SpacerHeight)
|
||||
page.AddCharts(line1, kline, line2, line3)
|
||||
page.Validate()
|
||||
page.SetLayout(components.PageCenterLayout)
|
||||
return page
|
||||
}
|
||||
|
||||
func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
|
||||
line := s.totalRequestsOverTime(c)
|
||||
return line.Render(c.Response().Writer)
|
||||
}
|
||||
|
||||
// totalRequestsOverTime generates a line chart showing total requests count over time
|
||||
func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
|
||||
line := charts.NewLine()
|
||||
line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
|
||||
|
||||
var startTime time.Time = time.Now()
|
||||
var endTime time.Time
|
||||
|
||||
for _, event := range s.data {
|
||||
if event.Time.Before(startTime) {
|
||||
startTime = event.Time
|
||||
}
|
||||
if event.Time.After(endTime) {
|
||||
endTime = event.Time
|
||||
}
|
||||
}
|
||||
data := getCategoryRequestCount(s.data)
|
||||
max := 0
|
||||
for _, v := range data {
|
||||
if len(v) > max {
|
||||
max = len(v)
|
||||
}
|
||||
}
|
||||
line.SetXAxis(time.Now().Format(time.RFC3339))
|
||||
for k, v := range data {
|
||||
lineData := make([]opts.LineData, 0)
|
||||
temp := 0
|
||||
for _, scanEvent := range v {
|
||||
temp += scanEvent.MaxRequests
|
||||
val := scanEvent.Time.Sub(startTime)
|
||||
lineData = append(lineData, opts.LineData{
|
||||
Value: []interface{}{val.Milliseconds(), temp},
|
||||
Name: scanEvent.TemplateID,
|
||||
})
|
||||
}
|
||||
line.AddSeries(k, lineData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
|
||||
}
|
||||
|
||||
line.SetGlobalOptions(
|
||||
charts.WithTitleOpts(opts.Title{Title: "Nuclei: total-req vs time"}),
|
||||
charts.WithXAxisOpts(opts.XAxis{Name: "Time", Type: "time", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
|
||||
charts.WithYAxisOpts(opts.YAxis{Name: "Requests Sent", Type: "value"}),
|
||||
charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
|
||||
charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
|
||||
charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
|
||||
charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
|
||||
SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
|
||||
DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
|
||||
DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
|
||||
}}),
|
||||
)
|
||||
|
||||
line.Validate()
|
||||
return line
|
||||
}
|
||||
|
||||
func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
|
||||
kline := s.topSlowTemplates(c)
|
||||
return kline.Render(c.Response().Writer)
|
||||
}
|
||||
|
||||
// topSlowTemplates generates a Kline chart showing the top slow templates by time taken
|
||||
func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
|
||||
kline := charts.NewKLine()
|
||||
kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
|
||||
|
||||
ids := map[string][]int64{}
|
||||
var startTime time.Time = time.Now()
|
||||
for _, event := range s.data {
|
||||
if event.Time.Before(startTime) {
|
||||
startTime = event.Time
|
||||
}
|
||||
}
|
||||
for _, event := range s.data {
|
||||
ids[event.TemplateID] = append(ids[event.TemplateID], event.Time.Sub(startTime).Milliseconds())
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
ID string
|
||||
KlineData opts.KlineData
|
||||
start int64
|
||||
end int64
|
||||
}
|
||||
data := []entry{}
|
||||
|
||||
for a, b := range ids {
|
||||
if len(b) < 2 {
|
||||
continue // Prevents index out of range error
|
||||
}
|
||||
d := entry{
|
||||
ID: a,
|
||||
KlineData: opts.KlineData{Value: []int64{b[0], b[len(b)-1], b[0], b[len(b)-1]}}, // Adjusted to prevent index out of range error
|
||||
start: b[0],
|
||||
end: b[len(b)-1],
|
||||
}
|
||||
data = append(data, d)
|
||||
}
|
||||
|
||||
sort.Slice(data, func(i, j int) bool {
|
||||
return data[i].end-data[i].start > data[j].end-data[j].start
|
||||
})
|
||||
|
||||
x := make([]string, 0)
|
||||
y := make([]opts.KlineData, 0)
|
||||
for _, event := range data[:TopK] {
|
||||
x = append(x, event.ID)
|
||||
y = append(y, event.KlineData)
|
||||
}
|
||||
|
||||
kline.SetXAxis(x).AddSeries("templates", y)
|
||||
kline.SetGlobalOptions(
|
||||
charts.WithTitleOpts(opts.Title{Title: fmt.Sprintf("Nuclei: Top %v Slow Templates", TopK)}),
|
||||
charts.WithXAxisOpts(opts.XAxis{
|
||||
Type: "category",
|
||||
Show: true,
|
||||
AxisLabel: &opts.AxisLabel{Rotate: 90, Show: true, ShowMinLabel: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (value) { return value; }`)},
|
||||
}),
|
||||
charts.WithYAxisOpts(opts.YAxis{
|
||||
Scale: true,
|
||||
Type: "value",
|
||||
Show: true,
|
||||
AxisLabel: &opts.AxisLabel{Show: true, Formatter: opts.FuncOpts(`function (ms) { return Math.floor(ms/60000) + 'm' + Math.floor((ms/60000 - Math.floor(ms/60000))*60) + 's'; }`)},
|
||||
}),
|
||||
charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
|
||||
charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "40%", Top: "10%"}),
|
||||
charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "events.ScanEvent", TriggerOn: "mousemove|click", Enterable: true, Formatter: opts.FuncOpts(`function (params) { return params.name ; }`)}),
|
||||
charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
|
||||
SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
|
||||
DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
|
||||
DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
|
||||
}}),
|
||||
)
|
||||
|
||||
return kline
|
||||
}
|
||||
|
||||
func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
|
||||
line := s.requestsVSInterval(c)
|
||||
return line.Render(c.Response().Writer)
|
||||
}
|
||||
|
||||
// requestsVSInterval generates a line chart showing requests per second over time
|
||||
func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
|
||||
line := charts.NewLine()
|
||||
line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
|
||||
|
||||
sort.Slice(s.data, func(i, j int) bool {
|
||||
return s.data[i].Time.Before(s.data[j].Time)
|
||||
})
|
||||
|
||||
var interval time.Duration
|
||||
|
||||
if c != nil {
|
||||
interval, _ = time.ParseDuration(c.QueryParam("interval"))
|
||||
}
|
||||
if interval <= 3 {
|
||||
interval = 5 * time.Second
|
||||
}
|
||||
|
||||
data := []opts.LineData{}
|
||||
temp := 0
|
||||
if len(s.data) > 0 {
|
||||
orig := s.data[0].Time
|
||||
startTime := orig
|
||||
xaxisData := []int64{}
|
||||
for _, v := range s.data {
|
||||
if v.Time.Sub(startTime) > interval {
|
||||
millisec := v.Time.Sub(orig).Milliseconds()
|
||||
xaxisData = append(xaxisData, millisec)
|
||||
data = append(data, opts.LineData{Value: temp, Name: v.Time.Sub(orig).String()})
|
||||
temp = 0
|
||||
startTime = v.Time
|
||||
}
|
||||
temp += 1
|
||||
}
|
||||
// Handle last interval if exists
|
||||
if temp > 0 {
|
||||
millisec := s.data[len(s.data)-1].Time.Sub(orig).Milliseconds()
|
||||
xaxisData = append(xaxisData, millisec)
|
||||
data = append(data, opts.LineData{Value: temp, Name: s.data[len(s.data)-1].Time.Sub(orig).String()})
|
||||
}
|
||||
line.SetXAxis(xaxisData)
|
||||
line.AddSeries("RPS", data, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
|
||||
}
|
||||
|
||||
line.SetGlobalOptions(
|
||||
charts.WithTitleOpts(opts.Title{Title: "Nuclei: Template Execution", Subtitle: "Time Interval: " + interval.String()}),
|
||||
charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
|
||||
charts.WithYAxisOpts(opts.YAxis{Name: "RPS Value", Type: "value", Show: true}),
|
||||
charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
|
||||
charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
|
||||
charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
|
||||
charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
|
||||
SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
|
||||
DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
|
||||
DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
|
||||
}}),
|
||||
)
|
||||
|
||||
line.Validate()
|
||||
return line
|
||||
}
|
||||
|
||||
func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
|
||||
line := s.concurrencyVsTime(c)
|
||||
return line.Render(c.Response().Writer)
|
||||
}
|
||||
|
||||
// concurrencyVsTime generates a line chart showing concurrency (total workers) over time
|
||||
func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
|
||||
line := charts.NewLine()
|
||||
line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
|
||||
|
||||
dataset := sliceutil.Clone(s.data)
|
||||
|
||||
sort.Slice(dataset, func(i, j int) bool {
|
||||
return dataset[i].Time.Before(dataset[j].Time)
|
||||
})
|
||||
|
||||
var interval time.Duration
|
||||
if c != nil {
|
||||
interval, _ = time.ParseDuration(c.QueryParam("interval"))
|
||||
}
|
||||
if interval <= 3 {
|
||||
interval = 5 * time.Second
|
||||
}
|
||||
|
||||
// create array with time interval as x-axis and worker count as y-axis
|
||||
// entry is a struct with time and poolsize
|
||||
type entry struct {
|
||||
Time time.Duration
|
||||
poolsize int
|
||||
}
|
||||
allEntries := []entry{}
|
||||
|
||||
dataIndex := 0
|
||||
maxIndex := len(dataset) - 1
|
||||
currEntry := entry{}
|
||||
|
||||
lastTime := dataset[0].Time
|
||||
for dataIndex <= maxIndex {
|
||||
currTime := dataset[dataIndex].Time
|
||||
if currTime.Sub(lastTime) > interval {
|
||||
// next batch
|
||||
currEntry.Time = interval
|
||||
allEntries = append(allEntries, currEntry)
|
||||
lastTime = dataset[dataIndex-1].Time
|
||||
}
|
||||
if dataset[dataIndex].EventType == events.ScanStarted {
|
||||
currEntry.poolsize += 1
|
||||
} else {
|
||||
currEntry.poolsize -= 1
|
||||
}
|
||||
dataIndex += 1
|
||||
}
|
||||
|
||||
plotData := []opts.LineData{}
|
||||
xaxisData := []int64{}
|
||||
tempTime := time.Duration(0)
|
||||
for _, v := range allEntries {
|
||||
tempTime += v.Time
|
||||
plotData = append(plotData, opts.LineData{Value: v.poolsize, Name: tempTime.String()})
|
||||
xaxisData = append(xaxisData, tempTime.Milliseconds())
|
||||
}
|
||||
line.SetXAxis(xaxisData)
|
||||
line.AddSeries("Concurrency", plotData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
|
||||
|
||||
line.SetGlobalOptions(
|
||||
charts.WithTitleOpts(opts.Title{Title: "Nuclei: WorkerPool", Subtitle: "Time Interval: " + interval.String()}),
|
||||
charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
|
||||
charts.WithYAxisOpts(opts.YAxis{Name: "Total Workers", Type: "value", Show: true}),
|
||||
charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
|
||||
charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
|
||||
charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
|
||||
charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
|
||||
SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
|
||||
DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
|
||||
DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
|
||||
}}),
|
||||
)
|
||||
|
||||
line.Validate()
|
||||
return line
|
||||
}
|
||||
|
||||
// getCategoryRequestCount returns a map of category and request count
|
||||
func getCategoryRequestCount(values []events.ScanEvent) map[string][]events.ScanEvent {
|
||||
mx := make(map[string][]events.ScanEvent)
|
||||
for _, event := range values {
|
||||
mx[event.TemplateType] = append(mx[event.TemplateType], event)
|
||||
}
|
||||
return mx
|
||||
}
|
||||
11
pkg/scan/events/scan_noop.go
Normal file
11
pkg/scan/events/scan_noop.go
Normal file
@ -0,0 +1,11 @@
|
||||
//go:build !stats
|
||||
// +build !stats
|
||||
|
||||
package events
|
||||
|
||||
// AddScanEvent is a no-op function
|
||||
func AddScanEvent(event ScanEvent) {
|
||||
}
|
||||
|
||||
func InitWithConfig(config *ScanConfig, statsDirectory string) {
|
||||
}
|
||||
80
pkg/scan/events/stats_build.go
Normal file
80
pkg/scan/events/stats_build.go
Normal file
@ -0,0 +1,80 @@
|
||||
//go:build stats
|
||||
// +build stats
|
||||
|
||||
package events
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ ScanEventWorker = &ScanStatsWorker{}
|
||||
|
||||
var defaultWorker = &ScanStatsWorker{}
|
||||
|
||||
// ScanStatsWorker is a worker for scanning stats
|
||||
// This tracks basic stats in jsonlines format
|
||||
// in given directory or a default directory with name stats_{timestamp} in the current directory
|
||||
type ScanStatsWorker struct {
|
||||
config *ScanConfig
|
||||
m *sync.Mutex
|
||||
directory string
|
||||
enc *json.Encoder
|
||||
}
|
||||
|
||||
// Init initializes the scan stats worker
|
||||
func InitWithConfig(config *ScanConfig, statsDirectory string) {
|
||||
currentTime := time.Now().Format("20060102150405")
|
||||
dirName := fmt.Sprintf("nuclei-stats-%s", currentTime)
|
||||
err := os.Mkdir(dirName, 0755)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// save the config to the directory
|
||||
bin, err := json.MarshalIndent(config, "", " ")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = os.WriteFile(filepath.Join(dirName, ConfigFile), bin, 0755)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defaultWorker = &ScanStatsWorker{config: config, m: &sync.Mutex{}, directory: dirName}
|
||||
err = defaultWorker.initEventsFile()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// initEventsFile initializes the events file for the worker
|
||||
func (s *ScanStatsWorker) initEventsFile() error {
|
||||
f, err := os.Create(filepath.Join(s.directory, EventsFile))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.enc = json.NewEncoder(f)
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddScanEvent adds a scan event to the worker
|
||||
func (s *ScanStatsWorker) AddScanEvent(event ScanEvent) {
|
||||
s.m.Lock()
|
||||
defer s.m.Unlock()
|
||||
|
||||
err := s.enc.Encode(event)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// AddScanEvent adds a scan event to the worker
|
||||
func AddScanEvent(event ScanEvent) {
|
||||
if defaultWorker == nil {
|
||||
return
|
||||
}
|
||||
defaultWorker.AddScanEvent(event)
|
||||
}
|
||||
45
pkg/scan/events/utils.go
Normal file
45
pkg/scan/events/utils.go
Normal file
@ -0,0 +1,45 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type ScanEventWorker interface {
|
||||
// AddScanEvent adds a scan event to the worker
|
||||
AddScanEvent(event ScanEvent)
|
||||
}
|
||||
|
||||
// Track scan start / finish status
|
||||
type ScanStatus string
|
||||
|
||||
const (
|
||||
ScanStarted ScanStatus = "scan_start"
|
||||
ScanFinished ScanStatus = "scan_end"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfigFile = "config.json"
|
||||
EventsFile = "events.jsonl"
|
||||
)
|
||||
|
||||
// ScanEvent represents a single scan event with its metadata
|
||||
type ScanEvent struct {
|
||||
Target string `json:"target" yaml:"target"`
|
||||
TemplateType string `json:"template_type" yaml:"template_type"`
|
||||
TemplateID string `json:"template_id" yaml:"template_id"`
|
||||
TemplatePath string `json:"template_path" yaml:"template_path"`
|
||||
MaxRequests int `json:"max_requests" yaml:"max_requests"`
|
||||
Time time.Time `json:"time" yaml:"time"`
|
||||
EventType ScanStatus `json:"event_type" yaml:"event_type"`
|
||||
}
|
||||
|
||||
// ScanConfig is only in context of scan event analysis
|
||||
type ScanConfig struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
TargetCount int `json:"target_count" yaml:"target_count"`
|
||||
TemplatesCount int `json:"templates_count" yaml:"templates_count"`
|
||||
TemplateConcurrency int `json:"template_concurrency" yaml:"template_concurrency"`
|
||||
PayloadConcurrency int `json:"payload_concurrency" yaml:"payload_concurrency"`
|
||||
JsConcurrency int `json:"js_concurrency" yaml:"js_concurrency"`
|
||||
Retries int `json:"retries" yaml:"retries"`
|
||||
}
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
)
|
||||
|
||||
|
||||
type ScanContextOption func(*ScanContext)
|
||||
|
||||
func WithEvents() ScanContextOption {
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
@ -14,6 +15,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
|
||||
@ -92,6 +94,31 @@ func (e *TemplateExecuter) Requests() int {
|
||||
|
||||
// Execute executes the protocol group and returns true or false if results were found.
|
||||
func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
|
||||
|
||||
// === when nuclei is built with -tags=stats ===
|
||||
// Note: this is no-op (empty functions) when nuclei is built in normal or without -tags=stats
|
||||
events.AddScanEvent(events.ScanEvent{
|
||||
Target: ctx.Input.MetaInput.Input,
|
||||
Time: time.Now(),
|
||||
EventType: events.ScanStarted,
|
||||
TemplateType: e.getTemplateType(),
|
||||
TemplateID: e.options.TemplateID,
|
||||
TemplatePath: e.options.TemplatePath,
|
||||
MaxRequests: e.Requests(),
|
||||
})
|
||||
defer func() {
|
||||
events.AddScanEvent(events.ScanEvent{
|
||||
Target: ctx.Input.MetaInput.Input,
|
||||
Time: time.Now(),
|
||||
EventType: events.ScanFinished,
|
||||
TemplateType: e.getTemplateType(),
|
||||
TemplateID: e.options.TemplateID,
|
||||
TemplatePath: e.options.TemplatePath,
|
||||
MaxRequests: e.Requests(),
|
||||
})
|
||||
}()
|
||||
// ==== end of stats ====
|
||||
|
||||
// executed contains status of execution if it was successfully executed or not
|
||||
// doesn't matter if it was matched or not
|
||||
executed := &atomic.Bool{}
|
||||
@ -182,3 +209,17 @@ func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.
|
||||
ctx.LogError(err)
|
||||
return ctx.GenerateResult(), err
|
||||
}
|
||||
|
||||
// getTemplateType returns the template type of the template
|
||||
func (e *TemplateExecuter) getTemplateType() string {
|
||||
if len(e.requests) == 0 {
|
||||
return "null"
|
||||
}
|
||||
if e.options.Flow != "" {
|
||||
return "flow"
|
||||
}
|
||||
if len(e.requests) > 1 {
|
||||
return "multiprotocol"
|
||||
}
|
||||
return e.requests[0].Type().String()
|
||||
}
|
||||
|
||||
@ -419,7 +419,9 @@ func DefaultOptions() *Options {
|
||||
BulkSize: 25,
|
||||
TemplateThreads: 25,
|
||||
HeadlessBulkSize: 10,
|
||||
PayloadConcurrency: 25,
|
||||
HeadlessTemplateThreads: 10,
|
||||
ProbeConcurrency: 50,
|
||||
Timeout: 5,
|
||||
Retries: 1,
|
||||
MaxHostError: 30,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user