diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..13602bfcd 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -110,6 +110,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), + flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), + flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "hc", 10, "maximum number of headless templates to be executed in parallel"), ) createGroup(flagSet, "optimization", "Optimizations", diff --git a/v2/go.mod b/v2/go.mod index 4ada6683a..9d3d5017a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,65 +5,57 @@ go 1.17 require ( github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible - github.com/akrylysov/pogreb v0.10.1 // indirect - github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c + github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.14.0 - github.com/antchfx/htmlquery v1.2.3 + github.com/antchfx/htmlquery v1.2.4 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 - github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 - github.com/go-rod/rod v0.101.7 + github.com/go-rod/rod v0.101.8 + github.com/gobwas/ws v1.1.0 github.com/google/go-github v17.0.0+incompatible - github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect - github.com/itchyny/gojq v0.12.4 + github.com/itchyny/gojq v0.12.5 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/miekg/dns v1.1.43 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/fastdialer v0.0.13 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 - github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 + github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/interactsh v0.0.6 - github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d + github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.21.7 + github.com/shirou/gopsutil/v3 v3.21.9 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 - github.com/xanzy/go-gitlab v0.50.3 - github.com/ysmood/gson v0.6.4 // indirect - github.com/ysmood/leakless v0.7.0 // indirect + github.com/xanzy/go-gitlab v0.51.1 go.uber.org/atomic v1.9.0 go.uber.org/multierr v1.7.0 go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 - golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect + golang.org/x/net v0.0.0-20211020060615-d418f374d309 + golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/text v0.3.7 - google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) @@ -72,12 +64,13 @@ require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect + github.com/akrylysov/pogreb v0.10.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect - github.com/antchfx/xpath v1.1.6 // indirect - github.com/aymerick/douceur v0.2.0 // indirect + github.com/antchfx/xpath v1.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect + github.com/c4milo/unpackit v0.1.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -87,15 +80,14 @@ require ( github.com/go-ole/go-ole v1.2.5 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.1.0 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/css v1.0.0 // indirect + github.com/gosuri/uilive v0.0.4 // indirect + github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect @@ -105,7 +97,7 @@ require ( github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect - github.com/microcosm-cc/bluemonday v1.0.15 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -115,17 +107,20 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect - github.com/tklauser/go-sysconf v0.3.7 // indirect - github.com/tklauser/numcpus v0.2.3 // indirect + github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/tklauser/numcpus v0.3.0 // indirect github.com/trivago/tgo v1.0.7 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.3.0 // indirect + github.com/ysmood/gson v0.6.4 // indirect + github.com/ysmood/leakless v0.7.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/v2/go.sum b/v2/go.sum index 82e27443a..d4a21e8b5 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -67,8 +67,9 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA= +github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -79,10 +80,12 @@ github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5z github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= -github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -96,14 +99,11 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -232,20 +232,20 @@ github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= -github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= -github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= -github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= -github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU= +github.com/go-rod/rod v0.101.8/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= @@ -331,15 +331,13 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= @@ -391,8 +389,9 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= -github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= +github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0= +github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= @@ -485,8 +484,6 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= -github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -579,11 +576,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb h1:h+HvVw51KUvcO4Tww1QCd95D6MWV/6wpXuSbmFpPQSI= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= -github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= +github.com/projectdiscovery/fastdialer v0.0.13 h1:BCe7JsFxRk1kAUQcy4X+9lqEuT7Y6LRSlHXfia03XOo= +github.com/projectdiscovery/fastdialer v0.0.13/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= @@ -591,8 +585,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= -github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= -github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk= +github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -600,7 +594,6 @@ github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8q github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54= -github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= @@ -618,8 +611,8 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= @@ -632,9 +625,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= -github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -673,8 +665,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -683,8 +673,9 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -735,10 +726,12 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs= github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo= -github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg= github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= -github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= @@ -761,8 +754,9 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= -github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xanzy/go-gitlab v0.51.1 h1:wWKLalwx4omxFoHh3PLs9zDgAD4GXDP/uoxwMRCSiWM= +github.com/xanzy/go-gitlab v0.51.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -931,16 +925,18 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= +golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA= golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1016,7 +1012,9 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 59791fe63..557821745 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -47,13 +47,6 @@ func ParseOptions(options *types.Options) { gologger.Fatal().Msgf("Program exiting: %s\n", err) } - // Auto adjust rate limits when using headless mode if the user - // hasn't specified any custom limits. - if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 { - options.BulkSize = 2 - options.TemplateThreads = 2 - } - // Load the resolvers if user asked for them loadResolvers(options) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go deleted file mode 100644 index 6b14b5f91..000000000 --- a/v2/internal/runner/processor.go +++ /dev/null @@ -1,81 +0,0 @@ -package runner - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" -) - -// processSelfContainedTemplates execute a self-contained template. -func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool { - match, err := template.Executer.Execute("") - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } - return match -} - -// processTemplateWithList execute a template against the list of user provided targets -func (r *Runner) processTemplateWithList(template *templates.Template) bool { - results := &atomic.Bool{} - wg := sizedwaitgroup.New(r.options.BulkSize) - processItem := func(k, _ []byte) error { - URL := string(k) - - // Skip if the host has had errors - if r.hostErrors != nil && r.hostErrors.Check(URL) { - return nil - } - wg.Add() - go func(URL string) { - defer wg.Done() - - match, err := template.Executer.Execute(URL) - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } - results.CAS(false, match) - }(URL) - return nil - } - if r.options.Stream { - _ = r.hostMapStream.Scan(processItem) - } else { - r.hostMap.Scan(processItem) - } - - wg.Wait() - return results.Load() -} - -// processTemplateWithList process a template on the URL list -func (r *Runner) processWorkflowWithList(template *templates.Template) bool { - results := &atomic.Bool{} - wg := sizedwaitgroup.New(r.options.BulkSize) - - processItem := func(k, _ []byte) error { - URL := string(k) - - // Skip if the host has had errors - if r.hostErrors != nil && r.hostErrors.Check(URL) { - return nil - } - wg.Add() - go func(URL string) { - defer wg.Done() - match := template.CompiledWorkflow.RunWorkflow(URL) - results.CAS(false, match) - }(URL) - return nil - } - - if r.options.Stream { - _ = r.hostMapStream.Scan(processItem) - } else { - r.hostMap.Scan(processItem) - } - - wg.Wait() - return results.Load() -} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f806952..d27713a35 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -2,7 +2,6 @@ package runner import ( "bufio" - "fmt" "os" "path/filepath" "strings" @@ -10,27 +9,22 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" - "github.com/rs/xid" - "go.uber.org/atomic" "go.uber.org/ratelimit" "gopkg.in/yaml.v2" - "github.com/projectdiscovery/filekv" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" @@ -38,7 +32,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" @@ -46,22 +39,20 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - hostMap *hybrid.HybridMap - hostMapStream *filekv.FileDB - output output.Writer - interactsh *interactsh.Client - inputCount int64 - templatesConfig *config.Config - options *types.Options - projectFile *projectfile.ProjectFile - catalog *catalog.Catalog - progress progress.Progress - colorizer aurora.Aurora - issuesClient *reporting.Client - addColor func(severity.Severity) string - browser *engine.Browser - ratelimiter ratelimit.Limiter - hostErrors *hosterrorscache.Cache + output output.Writer + interactsh *interactsh.Client + templatesConfig *config.Config + options *types.Options + projectFile *projectfile.ProjectFile + catalog *catalog.Catalog + progress progress.Progress + colorizer aurora.Aurora + issuesClient *reporting.Client + addColor func(severity.Severity) string + hmapInputProvider *hybrid.Input + browser *engine.Browser + ratelimiter ratelimit.Limiter + hostErrors *hosterrorscache.Cache } // New creates a new client for running enumeration process. @@ -116,103 +107,13 @@ func New(options *types.Options) (*Runner, error) { if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates { os.Exit(0) } - hm, err := hybrid.New(hybrid.DefaultDiskOptions) + + // Initialize the input source + hmapInput, err := hybrid.New(options) if err != nil { - return nil, errors.Wrap(err, "could not create temporary input file") - } - runner.hostMap = hm - - if options.Stream { - fkvOptions := filekv.DefaultOptions - if tmpFileName, err := fileutil.GetTempFileName(); err != nil { - return nil, errors.Wrap(err, "could not create temporary input file") - } else { - fkvOptions.Path = tmpFileName - } - fkv, err := filekv.Open(fkvOptions) - if err != nil { - return nil, errors.Wrap(err, "could not create temporary unsorted input file") - } - runner.hostMapStream = fkv - } - - runner.inputCount = 0 - dupeCount := 0 - - // Handle multiple targets - if len(options.Targets) != 0 { - for _, target := range options.Targets { - url := strings.TrimSpace(target) - if url == "" { - continue - } - - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - } - - // Handle stdin - if options.Stdin { - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - url := strings.TrimSpace(scanner.Text()) - if url == "" { - continue - } - - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - } - - // Handle target file - if options.TargetsFilePath != "" { - input, inputErr := os.Open(options.TargetsFilePath) - if inputErr != nil { - return nil, errors.Wrap(inputErr, "could not open targets file") - } - scanner := bufio.NewScanner(input) - for scanner.Scan() { - url := strings.TrimSpace(scanner.Text()) - if url == "" { - continue - } - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - input.Close() - } - - if dupeCount > 0 { - gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount) + return nil, errors.Wrap(err, "could not create input provider") } + runner.hmapInputProvider = hmapInput // Create the output file if asked outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile) @@ -312,13 +213,10 @@ func (r *Runner) Close() { if r.output != nil { r.output.Close() } - r.hostMap.Close() if r.projectFile != nil { r.projectFile.Close() } - if r.options.Stream { - r.hostMapStream.Close() - } + r.hmapInputProvider.Close() protocolinit.Close() } @@ -344,6 +242,9 @@ func (r *Runner) RunEnumeration() error { cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) } r.hostErrors = cache + + // Create the executer options which will be used throughout the execution + // stage by the nuclei engine modules. executerOpts := protocols.ExecuterOptions{ Output: r.output, Options: r.options, @@ -355,31 +256,18 @@ func (r *Runner) RunEnumeration() error { ProjectFile: r.projectFile, Browser: r.browser, HostErrorsCache: cache, + Colorizer: r.colorizer, } + engine := core.New(r.options) + engine.SetExecuterOptions(executerOpts) workflowLoader, err := parsers.NewLoader(&executerOpts) if err != nil { return errors.Wrap(err, "Could not create loader.") } - executerOpts.WorkflowLoader = workflowLoader - loaderConfig := loader.Config{ - Templates: r.options.Templates, - Workflows: r.options.Workflows, - ExcludeTemplates: r.options.ExcludedTemplates, - Tags: r.options.Tags, - ExcludeTags: r.options.ExcludeTags, - IncludeTemplates: r.options.IncludeTemplates, - Authors: r.options.Author, - Severities: r.options.Severities, - ExcludeSeverities: r.options.ExcludeSeverities, - IncludeTags: r.options.IncludeTags, - TemplatesDirectory: r.options.TemplatesDirectory, - Catalog: r.catalog, - ExecutorOptions: executerOpts, - } - store, err := loader.New(&loaderConfig) + store, err := loader.New(loader.NewConfig(r.options, r.catalog, executerOpts)) if err != nil { return errors.Wrap(err, "could not load templates from config") } @@ -397,6 +285,78 @@ func (r *Runner) RunEnumeration() error { return nil // exit } + r.displayExecutionInfo(store) + + var unclusteredRequests int64 + for _, template := range store.Templates() { + // workflows will dynamically adjust the totals while running, as + // it can't be known in advance which requests will be called + if len(template.Workflows) > 0 { + continue + } + unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count() + } + + if r.options.VerboseVerbose { + for _, template := range store.Templates() { + r.logAvailableTemplate(template.Path) + } + for _, template := range store.Workflows() { + r.logAvailableTemplate(template.Path) + } + } + + // Cluster the templates first because we want info on how many + // templates did we cluster for showing to user in CLI + originalTemplatesCount := len(store.Templates()) + finalTemplates, clusterCount := engine.ClusterTemplates(store.Templates()) + finalTemplates = append(finalTemplates, store.Workflows()...) + + var totalRequests int64 + for _, t := range finalTemplates { + if len(t.Workflows) > 0 { + continue + } + totalRequests += int64(t.TotalRequests) * r.hmapInputProvider.Count() + } + if totalRequests < unclusteredRequests { + gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests) + } + workflowCount := len(store.Workflows()) + templateCount := originalTemplatesCount + workflowCount + + // 0 matches means no templates were found in directory + if templateCount == 0 { + return errors.New("no valid templates were found") + } + + // tracks global progress and captures stdout/stderr until p.Wait finishes + r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests) + + results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true) + + if r.interactsh != nil { + matched := r.interactsh.Close() + if matched { + results.CAS(false, true) + } + } + r.progress.Stop() + + if r.issuesClient != nil { + r.issuesClient.Close() + } + if !results.Load() { + gologger.Info().Msgf("No results found. Better luck next time!") + } + if r.browser != nil { + r.browser.Close() + } + return nil +} + +// displayExecutionInfo displays misc info about the nuclei engine execution +func (r *Runner) displayExecutionInfo(store *loader.Store) { // Display stats for any loaded templates' syntax warnings or errors stats.Display(parsers.SyntaxWarningStats) stats.Display(parsers.SyntaxErrorStats) @@ -445,128 +405,6 @@ func (r *Runner) RunEnumeration() error { if len(store.Workflows()) > 0 { gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) } - - // pre-parse all the templates, apply filters - finalTemplates := []*templates.Template{} - - var unclusteredRequests int64 - for _, template := range store.Templates() { - // workflows will dynamically adjust the totals while running, as - // it can't be known in advance which requests will be called - if len(template.Workflows) > 0 { - continue - } - unclusteredRequests += int64(template.TotalRequests) * r.inputCount - } - - if r.options.VerboseVerbose { - for _, template := range store.Templates() { - r.logAvailableTemplate(template.Path) - } - for _, template := range store.Workflows() { - r.logAvailableTemplate(template.Path) - } - } - templatesMap := make(map[string]*templates.Template) - for _, v := range store.Templates() { - templatesMap[v.Path] = v - } - originalTemplatesCount := len(store.Templates()) - clusterCount := 0 - clusters := clusterer.Cluster(templatesMap) - for _, cluster := range clusters { - if len(cluster) > 1 && !r.options.OfflineHTTP { - executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - RateLimiter: r.ratelimiter, - IssuesClient: r.issuesClient, - Browser: r.browser, - ProjectFile: r.projectFile, - Interactsh: r.interactsh, - HostErrorsCache: cache, - } - clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) - - finalTemplates = append(finalTemplates, &templates.Template{ - ID: clusterID, - RequestsHTTP: cluster[0].RequestsHTTP, - Executer: clusterer.NewExecuter(cluster, &executerOpts), - TotalRequests: len(cluster[0].RequestsHTTP), - }) - clusterCount += len(cluster) - } else { - finalTemplates = append(finalTemplates, cluster...) - } - } - - finalTemplates = append(finalTemplates, store.Workflows()...) - - var totalRequests int64 - for _, t := range finalTemplates { - if len(t.Workflows) > 0 { - continue - } - totalRequests += int64(t.TotalRequests) * r.inputCount - } - if totalRequests < unclusteredRequests { - gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests) - } - workflowCount := len(store.Workflows()) - templateCount := originalTemplatesCount + workflowCount - - // 0 matches means no templates were found in directory - if templateCount == 0 { - return errors.New("no valid templates were found") - } - - /* - TODO does it make sense to run the logic below if there are no targets specified? - Can we safely assume the user is just experimenting with the template/workflow filters before running them? - */ - - results := &atomic.Bool{} - wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) - - // tracks global progress and captures stdout/stderr until p.Wait finishes - r.progress.Init(r.inputCount, templateCount, totalRequests) - - for _, t := range finalTemplates { - wgtemplates.Add() - go func(template *templates.Template) { - defer wgtemplates.Done() - - if template.SelfContained { - results.CAS(false, r.processSelfContainedTemplates(template)) - } else if len(template.Workflows) > 0 { - results.CAS(false, r.processWorkflowWithList(template)) - } else { - results.CAS(false, r.processTemplateWithList(template)) - } - }(t) - } - wgtemplates.Wait() - - if r.interactsh != nil { - matched := r.interactsh.Close() - if matched { - results.CAS(false, true) - } - } - r.progress.Stop() - - if r.issuesClient != nil { - r.issuesClient.Close() - } - if !results.Load() { - gologger.Info().Msgf("No results found. Better luck next time!") - } - if r.browser != nil { - r.browser.Close() - } - return nil } // readNewTemplatesFile reads newly added templates from directory if it exists diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d630..e58c70791 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -1,7 +1,6 @@ package testutils import ( - "github.com/logrusorgru/aurora" "go.uber.org/ratelimit" "github.com/projectdiscovery/gologger/levels" @@ -60,41 +59,6 @@ var DefaultOptions = &types.Options{ CustomHeaders: []string{}, } -// MockOutputWriter is a mocked output writer. -type MockOutputWriter struct { - aurora aurora.Aurora - RequestCallback func(templateID, url, requestType string, err error) - WriteCallback func(o *output.ResultEvent) -} - -// NewMockOutputWriter creates a new mock output writer -func NewMockOutputWriter() *MockOutputWriter { - return &MockOutputWriter{aurora: aurora.NewAurora(false)} -} - -// Close closes the output writer interface -func (m *MockOutputWriter) Close() {} - -// Colorizer returns the colorizer instance for writer -func (m *MockOutputWriter) Colorizer() aurora.Aurora { - return m.aurora -} - -// Write writes the event to file and/or screen. -func (m *MockOutputWriter) Write(result *output.ResultEvent) error { - if m.WriteCallback != nil { - m.WriteCallback(result) - } - return nil -} - -// Request writes a log the requests trace log -func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { - if m.RequestCallback != nil { - m.RequestCallback(templateID, url, requestType, err) - } -} - // TemplateInfo contains info for a mock executed template. type TemplateInfo struct { ID string @@ -109,7 +73,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco TemplateID: info.ID, TemplateInfo: info.Info, TemplatePath: info.Path, - Output: NewMockOutputWriter(), + Output: output.NewMockOutputWriter(), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4ae..386898b12 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Config contains the configuration options for the loader @@ -44,6 +45,26 @@ type Store struct { preprocessor templates.Preprocessor } +// NewConfig returns a new loader config +func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { + loaderConfig := Config{ + Templates: options.Templates, + Workflows: options.Workflows, + ExcludeTemplates: options.ExcludedTemplates, + Tags: options.Tags, + ExcludeTags: options.ExcludeTags, + IncludeTemplates: options.IncludeTemplates, + Authors: options.Author, + Severities: options.Severities, + ExcludeSeverities: options.ExcludeSeverities, + IncludeTags: options.IncludeTags, + TemplatesDirectory: options.TemplatesDirectory, + Catalog: catalog, + ExecutorOptions: executerOpts, + } + return &loaderConfig +} + // New creates a new template store based on provided configuration func New(config *Config) (*Store, error) { // Create a tag filter based on provided configuration diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go new file mode 100644 index 000000000..455546244 --- /dev/null +++ b/v2/pkg/core/engine.go @@ -0,0 +1,57 @@ +package core + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Engine is an engine for running Nuclei Templates/Workflows. +// +// The engine contains multiple thread pools which allow using different +// concurrency values per protocol executed. +// +// The engine does most of the heavy lifting of execution, from clustering +// templates to leading to the final execution by the workpool, it is +// handled by the engine. +type Engine struct { + workPool *WorkPool + options *types.Options + executerOpts protocols.ExecuterOptions +} + +// InputProvider is an input provider interface for the nuclei execution +// engine. +// +// An example InputProvider is provided in form of hmap input provider. +type InputProvider interface { + // Count returns the number of items for input provider + Count() int64 + // Scan calls a callback function till the input provider is exhausted + Scan(callback func(value string)) +} + +// New returns a new Engine instance +func New(options *types.Options) *Engine { + workPool := NewWorkPool(WorkPoolConfig{ + InputConcurrency: options.BulkSize, + TypeConcurrency: options.TemplateThreads, + HeadlessInputConcurrency: options.HeadlessBulkSize, + HeadlessTypeConcurrency: options.HeadlessTemplateThreads, + }) + engine := &Engine{ + options: options, + workPool: workPool, + } + return engine +} + +// SetExecuterOptions sets the executer options for the engine. This is required +// before using the engine to perform any execution. +func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) { + e.executerOpts = options +} + +// ExecuterOptions returns protocols.ExecuterOptions for nuclei engine. +func (e *Engine) ExecuterOptions() protocols.ExecuterOptions { + return e.executerOpts +} diff --git a/v2/pkg/core/engine_test.go b/v2/pkg/core/engine_test.go new file mode 100644 index 000000000..9a8bc9592 --- /dev/null +++ b/v2/pkg/core/engine_test.go @@ -0,0 +1 @@ +package core diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go new file mode 100644 index 000000000..883707875 --- /dev/null +++ b/v2/pkg/core/execute.go @@ -0,0 +1,130 @@ +package core + +import ( + "fmt" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" + "github.com/rs/xid" + "go.uber.org/atomic" +) + +// Execute takes a list of templates/workflows that have been compiled +// and executes them based on provided concurrency options. +// +// All the execution logic for the templates/workflows happens in this part +// of the engine. +func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool { + return e.ExecuteWithOpts(templates, input, false) +} + +// ExecuteWithOpts is execute with the full options +func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool { + var finalTemplates []*templates.Template + if !noCluster { + finalTemplates, _ = e.ClusterTemplates(templatesList) + } else { + finalTemplates = templatesList + } + + results := &atomic.Bool{} + for _, template := range finalTemplates { + templateType := template.Type() + + var wg *sizedwaitgroup.SizedWaitGroup + if templateType == "headless" { + wg = e.workPool.Headless + } else { + wg = e.workPool.Default + } + + wg.Add() + switch { + case template.SelfContained: + // Self Contained requests are executed here separately + e.executeSelfContainedTemplateWithInput(template, results) + default: + // All other request types are executed here + e.executeModelWithInput(templateType, template, input, results) + } + wg.Done() + } + e.workPool.Wait() + return results +} + +// processSelfContainedTemplates execute a self-contained template. +func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) { + match, err := template.Executer.Execute("") + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) + } + results.CAS(false, match) +} + +// executeModelWithInput executes a type of template with input +func (e *Engine) executeModelWithInput(templateType string, template *templates.Template, input InputProvider, results *atomic.Bool) { + wg := e.workPool.InputPool(templateType) + + input.Scan(func(scannedValue string) { + // Skip if the host has had errors + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { + return + } + + wg.Waitgroup.Add() + go func(value string) { + defer wg.Waitgroup.Done() + + var match bool + var err error + switch templateType { + case "workflow": + match = e.executeWorkflow(value, template.CompiledWorkflow) + default: + match, err = template.Executer.Execute(value) + } + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) + } + results.CAS(false, match) + }(scannedValue) + }) + wg.Waitgroup.Wait() +} + +// ClusterTemplates performs identical http requests clustering for a list of templates +func (e *Engine) ClusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { + if e.options.OfflineHTTP { + return templatesList, 0 + } + + templatesMap := make(map[string]*templates.Template) + for _, v := range templatesList { + templatesMap[v.Path] = v + } + clusterCount := 0 + + finalTemplatesList := make([]*templates.Template, 0, len(templatesList)) + clusters := clusterer.Cluster(templatesMap) + for _, cluster := range clusters { + if len(cluster) > 1 { + executerOpts := e.ExecuterOptions() + + clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) + + finalTemplatesList = append(finalTemplatesList, &templates.Template{ + ID: clusterID, + RequestsHTTP: cluster[0].RequestsHTTP, + Executer: clusterer.NewExecuter(cluster, &executerOpts), + TotalRequests: len(cluster[0].RequestsHTTP), + }) + clusterCount += len(cluster) + } else { + finalTemplatesList = append(finalTemplatesList, cluster...) + } + } + return finalTemplatesList, clusterCount +} diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go new file mode 100644 index 000000000..1eb4f5e42 --- /dev/null +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -0,0 +1,133 @@ +// Package hybrid implements a hybrid hmap/filekv backed input provider +// for nuclei that can either stream or store results using different kv stores. +package hybrid + +import ( + "bufio" + "io" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/filekv" + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/hmap/store/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Input is a hmap/filekv backed nuclei Input provider +type Input struct { + inputCount int64 + dupeCount int64 + hostMap *hybrid.HybridMap + hostMapStream *filekv.FileDB +} + +// New creates a new hmap backed nuclei Input Provider +// and initializes it based on the passed options Model. +func New(options *types.Options) (*Input, error) { + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } + + input := &Input{hostMap: hm} + if options.Stream { + fkvOptions := filekv.DefaultOptions + if tmpFileName, err := fileutil.GetTempFileName(); err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } else { + fkvOptions.Path = tmpFileName + } + fkv, err := filekv.Open(fkvOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary unsorted input file") + } + input.hostMapStream = fkv + } + if initErr := input.initializeInputSources(options); initErr != nil { + return nil, initErr + } + if input.dupeCount > 0 { + gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", input.dupeCount) + } + return input, nil +} + +// Close closes the input provider +func (i *Input) Close() { + i.hostMap.Close() + if i.hostMapStream != nil { + i.hostMapStream.Close() + } +} + +// initializeInputSources initializes the input sources for hmap input +func (i *Input) initializeInputSources(options *types.Options) error { + // Handle targets flags + for _, target := range options.Targets { + i.normalizeStoreInputValue(target) + } + + // Handle stdin + if options.Stdin { + i.scanInputFromReader(os.Stdin) + } + + // Handle target file + if options.TargetsFilePath != "" { + input, inputErr := os.Open(options.TargetsFilePath) + if inputErr != nil { + return errors.Wrap(inputErr, "could not open targets file") + } + i.scanInputFromReader(input) + input.Close() + } + return nil +} + +// scanInputFromReader scans a line of input from reader and passes it for storage +func (i *Input) scanInputFromReader(reader io.Reader) { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + i.normalizeStoreInputValue(scanner.Text()) + } +} + +// normalizeStoreInputValue normalizes and stores passed input values +func (i *Input) normalizeStoreInputValue(value string) { + url := strings.TrimSpace(value) + if url == "" { + return + } + + if _, ok := i.hostMap.Get(url); ok { + i.dupeCount++ + return + } + + i.inputCount++ + _ = i.hostMap.Set(url, nil) + if i.hostMapStream != nil { + _ = i.hostMapStream.Set([]byte(url), nil) + } +} + +// Count returns the input count +func (i *Input) Count() int64 { + return i.inputCount +} + +// Scan calls an input provider till the callback is exhausted +func (i *Input) Scan(callback func(value string)) { + callbackFunc := func(k, _ []byte) error { + callback(string(k)) + return nil + } + if i.hostMapStream != nil { + _ = i.hostMapStream.Scan(callbackFunc) + } else { + i.hostMap.Scan(callbackFunc) + } +} diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go new file mode 100644 index 000000000..6237dfb99 --- /dev/null +++ b/v2/pkg/core/inputs/inputs.go @@ -0,0 +1,17 @@ +package inputs + +type SimpleInputProvider struct { + Inputs []string +} + +// Count returns the number of items for input provider +func (s *SimpleInputProvider) Count() int64 { + return int64(len(s.Inputs)) +} + +// Scan calls a callback function till the input provider is exhausted +func (s *SimpleInputProvider) Scan(callback func(value string)) { + for _, v := range s.Inputs { + callback(v) + } +} diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/core/workflow_execute.go similarity index 79% rename from v2/pkg/workflows/execute.go rename to v2/pkg/core/workflow_execute.go index c0710c6ad..8c255d7ad 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,21 +1,22 @@ -package workflows +package core import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/remeh/sizedwaitgroup" "go.uber.org/atomic" ) -// RunWorkflow runs a workflow on an input and returns true or false -func (w *Workflow) RunWorkflow(input string) bool { +// executeWorkflow runs a workflow on an input and returns true or false +func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { results := &atomic.Bool{} swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) for _, template := range w.Workflows { swg.Add() - func(template *WorkflowTemplate) { - if err := w.runWorkflowStep(template, input, results, &swg); err != nil { + func(template *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() @@ -27,7 +28,7 @@ func (w *Workflow) RunWorkflow(input string) bool { // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error { +func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { var firstMatched bool var err error var mainErr error @@ -90,8 +91,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range matcher.Subtemplates { swg.Add() - go func(subtemplate *WorkflowTemplate) { - if err := w.runWorkflowStep(subtemplate, input, results, swg); err != nil { + go func(subtemplate *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", subtemplate.Template, err) } swg.Done() @@ -114,8 +115,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range template.Subtemplates { swg.Add() - go func(template *WorkflowTemplate) { - if err := w.runWorkflowStep(template, input, results, swg); err != nil { + go func(template *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/core/workflow_execute_test.go similarity index 70% rename from v2/pkg/workflows/execute_test.go rename to v2/pkg/core/workflow_execute_test.go index 6d9ab6a09..a00ce6043 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -1,4 +1,4 @@ -package workflows +package core import ( "testing" @@ -10,18 +10,20 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) func TestWorkflowsSimple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") } @@ -29,20 +31,21 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, - {Executers: []*ProtocolExecuterPair{{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -53,21 +56,22 @@ func TestWorkflowsSubtemplates(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -78,19 +82,20 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: false, executeHook: func(input string) { firstInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -101,8 +106,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ @@ -111,14 +116,15 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Matchers: []*workflows.Matcher{{Name: "tomcat", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -129,8 +135,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ @@ -139,14 +145,15 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Matchers: []*workflows.Matcher{{Name: "apache", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go new file mode 100644 index 000000000..23b50d682 --- /dev/null +++ b/v2/pkg/core/workpool.go @@ -0,0 +1,63 @@ +package core + +import ( + "github.com/remeh/sizedwaitgroup" +) + +// WorkPool implements an execution pool for executing different +// types of task with different concurreny requirements. +// +// It also allows Configuration of such requirements. This is used +// for per-module like separate headless concurrency etc. +type WorkPool struct { + Headless *sizedwaitgroup.SizedWaitGroup + Default *sizedwaitgroup.SizedWaitGroup + config WorkPoolConfig +} + +// WorkPoolConfig is the configuration for workpool +type WorkPoolConfig struct { + // InputConcurrency is the concurrency for inputs values. + InputConcurrency int + // TypeConcurrency is the concurrency for the request type templates. + TypeConcurrency int + // HeadlessInputConcurrency is the concurrency for headless inputs values. + HeadlessInputConcurrency int + // TypeConcurrency is the concurrency for the headless request type templates. + HeadlessTypeConcurrency int +} + +// NewWorkPool returns a new WorkPool instance +func NewWorkPool(config WorkPoolConfig) *WorkPool { + headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency) + defaultWg := sizedwaitgroup.New(config.TypeConcurrency) + + return &WorkPool{ + config: config, + Headless: &headlessWg, + Default: &defaultWg, + } +} + +// Wait waits for all the workpool waitgroups to finish +func (w *WorkPool) Wait() { + w.Default.Wait() + w.Headless.Wait() +} + +// InputWorkPool is a workpool per-input +type InputWorkPool struct { + Waitgroup *sizedwaitgroup.SizedWaitGroup +} + +// InputPool returns a workpool for an input type +func (w *WorkPool) InputPool(templateType string) *InputWorkPool { + var count int + if templateType == "headless" { + count = w.config.HeadlessInputConcurrency + } else { + count = w.config.InputConcurrency + } + swg := sizedwaitgroup.New(count) + return &InputWorkPool{Waitgroup: &swg} +} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index b90be29c3..26555719d 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -209,3 +209,38 @@ func (w *StandardWriter) Close() { w.traceFile.Close() } } + +// MockOutputWriter is a mocked output writer. +type MockOutputWriter struct { + aurora aurora.Aurora + RequestCallback func(templateID, url, requestType string, err error) + WriteCallback func(o *ResultEvent) +} + +// NewMockOutputWriter creates a new mock output writer +func NewMockOutputWriter() *MockOutputWriter { + return &MockOutputWriter{aurora: aurora.NewAurora(false)} +} + +// Close closes the output writer interface +func (m *MockOutputWriter) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *MockOutputWriter) Colorizer() aurora.Aurora { + return m.aurora +} + +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) Write(result *ResultEvent) error { + if m.WriteCallback != nil { + m.WriteCallback(result) + } + return nil +} + +// Request writes a log the requests trace log +func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { + if m.RequestCallback != nil { + m.RequestCallback(templateID, url, requestType, err) + } +} diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 220ff49a6..fcd34340d 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -247,3 +247,27 @@ func (p *StatsTicker) Stop() { _ = p.server.Shutdown(context.Background()) } } + +type MockProgressClient struct{} + +// Stop stops the progress recorder. +func (m *MockProgressClient) Stop() {} + +// Init inits the progress bar with initial details for scan +func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} + +// AddToTotal adds a value to the total request count +func (m *MockProgressClient) AddToTotal(delta int64) {} + +// IncrementRequests increments the requests counter by 1. +func (m *MockProgressClient) IncrementRequests() {} + +// IncrementMatched increments the matched counter by 1. +func (m *MockProgressClient) IncrementMatched() {} + +// IncrementErrorsBy increments the error counter by count. +func (m *MockProgressClient) IncrementErrorsBy(count int64) {} + +// IncrementFailedRequestsBy increments the number of requests counter by count +// along with errors. +func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5c2914ff5..0ea42d460 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -103,6 +103,20 @@ func New(options *Options) (*Client, error) { return interactClient, nil } +// NewDefaultOptions returns the default options for interactsh client +func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options { + return &Options{ + ServerURL: "https://interactsh.com", + CacheSize: 5000, + Eviction: 60 * time.Second, + ColldownPeriod: 5 * time.Second, + PollDuration: 5 * time.Second, + Output: output, + IssuesClient: reporting, + Progress: progress, + } +} + func (c *Client) firstTimeInitializeClient() error { interactsh, err := client.New(&client.Options{ ServerURL: c.options.ServerURL, diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index ded0274dd..ec72fcc91 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -12,10 +12,12 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -94,15 +96,8 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) - event := &output.InternalWrappedEvent{InternalEvent: data} - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc) - if ok && event.OperatorsResult != nil { - event.Results = utils.MakeResultEvent(event, r.makeResultEventItem) - } - callback(event) - } + event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {}) + callback(event) return nil } @@ -125,7 +120,29 @@ func getAddress(toTest string) (string, error) { return toTest, nil } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(r, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (r *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{r.CompiledOperators} +} + +func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(r.options.TemplateID), TemplatePath: types.ToString(r.options.TemplatePath), diff --git a/v2/pkg/protocols/others/utils/utils.go b/v2/pkg/protocols/others/utils/utils.go deleted file mode 100644 index 89771ca90..000000000 --- a/v2/pkg/protocols/others/utils/utils.go +++ /dev/null @@ -1,83 +0,0 @@ -package utils - -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -// MakeResultEventItemFunc returns a result event for an internal wrapped event item -type MakeResultEventItemFunc func(wrapped *output.InternalWrappedEvent) *output.ResultEvent - -// MakeResultEvent creates a result event from internal wrapped event -func MakeResultEvent(wrapped *output.InternalWrappedEvent, makeEventItemFunc MakeResultEventItemFunc) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := makeEventItemFunc(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := makeEventItemFunc(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := makeEventItemFunc(wrapped) - results = append(results, data) - } - return results -} - -// ExtractFunc performs extracting operation for an extractor on model and returns true or false. -func ExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := data[extractor.Part] - if !ok { - return nil - } - itemStr := types.ToString(item) - - switch extractor.GetType() { - case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) - case extractors.KValExtractor: - return extractor.ExtractKval(data) - case extractors.JSONExtractor: - return extractor.ExtractJSON(itemStr) - case extractors.XPathExtractor: - return extractor.ExtractHTML(itemStr) - } - return nil -} - -// MatchFunc performs matching operation for a matcher on model and returns true or false. -func MatchFunc(data map[string]interface{}, matcher *matchers.Matcher) bool { - partItem, ok := data[matcher.Part] - if !ok && len(matcher.DSL) == 0 { - return false - } - item := types.ToString(partItem) - - switch matcher.GetType() { - case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) - case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) - case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) - case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) - case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) - } - return false -} diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index c48677f8d..a2487aa0a 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -15,12 +15,14 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -258,7 +260,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu // Run any internal extractors for the request here and add found values to map. if r.CompiledOperators != nil { - values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, utils.ExtractFunc) + values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) for k, v := range values { dynamicValues[k] = v } @@ -298,15 +300,11 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu data["host"] = input data["ip"] = r.dialer.GetDialedIP(hostname) - event := &output.InternalWrappedEvent{InternalEvent: data} - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc) - if ok && event.OperatorsResult != nil { - event.Results = utils.MakeResultEvent(event, r.makeResultEventItem) - } - callback(event) - } + event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues + }) + + callback(event) return nil } @@ -322,7 +320,29 @@ func getAddress(toTest string) (string, error) { return "", nil } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(r, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (r *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{r.CompiledOperators} +} + +func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(r.options.TemplateID), TemplatePath: types.ToString(r.options.TemplatePath), diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 97ca4b881..2892adb87 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -3,9 +3,12 @@ package protocols import ( "go.uber.org/ratelimit" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" @@ -59,9 +62,16 @@ type ExecuterOptions struct { Operators []*operators.Operators // only used by offlinehttp module + Colorizer aurora.Aurora WorkflowLoader model.WorkflowLoader } +// Copy returns a copy of the executeroptions structure +func (e ExecuterOptions) Copy() ExecuterOptions { + copy := e + return copy +} + // Request is an interface implemented any protocol based request generator. type Request interface { // Compile compiles the request generators preparing any requests possible. @@ -118,3 +128,51 @@ func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEven } return results } + +// MakeDefaultExtractFunc performs extracting operation for an extractor on model and returns true or false. +func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + item, ok := data[extractor.Part] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + case extractors.JSONExtractor: + return extractor.ExtractJSON(itemStr) + case extractors.XPathExtractor: + return extractor.ExtractHTML(itemStr) + } + return nil +} + +// MakeDefaultMatchFunc performs matching operation for a matcher on model and returns true or false. +func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + partItem, ok := data[matcher.Part] + if !ok && len(matcher.DSL) == 0 { + return false, nil + } + item := types.ToString(partItem) + + switch matcher.GetType() { + case matchers.SizeMatcher: + result := matcher.Result(matcher.MatchSize(len(item))) + return result, nil + case matchers.WordsMatcher: + result, value := matcher.MatchWords(item, nil) + return matcher.Result(result), value + case matchers.RegexMatcher: + result, value := matcher.MatchRegex(item) + return matcher.Result(result), value + case matchers.BinaryMatcher: + result, value := matcher.MatchBinary(item) + return matcher.Result(result), value + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)), nil + } + return false, nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 7b22d7086..2899ffc6b 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "strings" "github.com/pkg/errors" @@ -83,108 +84,21 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.CompiledWorkflow.Options = &options } - // Compile the requests found - requests := []protocols.Request{} - - if len(template.RequestsHTTP) > 0 { - if options.Options.OfflineHTTP { - operatorsList := []*operators.Operators{} - - mainLoop: - for _, req := range template.RequestsHTTP { - for _, path := range req.Path { - if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { - break mainLoop - } - } - operatorsList = append(operatorsList, &req.Operators) - } - if len(operatorsList) > 0 { - options.Operators = operatorsList - template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) - } - } else { - for _, req := range template.RequestsHTTP { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - } - if !options.Options.OfflineHTTP { - makeRequestsForTemplate(template, options) + if err := template.compileProtocolRequests(options); err != nil { + return nil, err } if template.Executer != nil { if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") } - template.TotalRequests += template.Executer.Requests() + template.TotalRequests = template.Executer.Requests() } if template.Executer == nil && template.CompiledWorkflow == nil { return nil, ErrCreateTemplateExecutor } template.Path = filePath - parsedTemplatesCache.Store(filePath, template, err) - return template, nil -} - -// Requests returns the total number of requests for the template. -func (t *Template) Requests() int { - sum := len(t.RequestsDNS) + - len(t.RequestsHTTP) + - len(t.RequestsFile) + - len(t.RequestsNetwork) + - len(t.RequestsHeadless) + - len(t.Workflows) + - len(t.RequestsSSL) + - len(t.RequestsWebsocket) - return sum -} - -// makeRequestsForTemplate compiles all the requests for the template. -func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptions) { - requests := []protocols.Request{} - - if len(template.RequestsDNS) > 0 { - for _, req := range template.RequestsDNS { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsFile) > 0 { - for _, req := range template.RequestsFile { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsNetwork) > 0 { - for _, req := range template.RequestsNetwork { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsHeadless) > 0 && options.Options.Headless { - for _, req := range template.RequestsHeadless { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsSSL) > 0 { - for _, req := range template.RequestsSSL { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsWebsocket) > 0 { - for _, req := range template.RequestsWebsocket { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - - template.Path = filePath - template.parseSelfContainedRequests() parsedTemplatesCache.Store(filePath, template, err) @@ -192,14 +106,106 @@ func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptio } // parseSelfContainedRequests parses the self contained template requests. -func (t *Template) parseSelfContainedRequests() { - if !t.SelfContained { +func (template *Template) parseSelfContainedRequests() { + if !template.SelfContained { return } - for _, request := range t.RequestsHTTP { + for _, request := range template.RequestsHTTP { request.SelfContained = true } - for _, request := range t.RequestsNetwork { + for _, request := range template.RequestsNetwork { request.SelfContained = true } } + +// Requests returns the total request count for the template +func (template *Template) Requests() int { + return len(template.RequestsDNS) + + len(template.RequestsHTTP) + + len(template.RequestsFile) + + len(template.RequestsNetwork) + + len(template.RequestsHeadless) + + len(template.Workflows) + + len(template.RequestsSSL) + + len(template.RequestsWebsocket) +} + +// compileProtocolRequests compiles all the protocol requests for the template +func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error { + templateRequests := template.Requests() + + if templateRequests == 0 { + return fmt.Errorf("no requests defined for %s", template.ID) + } + + if options.Options.OfflineHTTP { + template.compileOfflineHTTPRequest(options) + return nil + } + + var requests []protocols.Request + switch { + case len(template.RequestsDNS) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsDNS) + + case len(template.RequestsFile) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsFile) + + case len(template.RequestsNetwork) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsNetwork) + + case len(template.RequestsHTTP) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsHTTP) + + case len(template.RequestsHeadless) > 0 && options.Options.Headless: + requests = template.convertRequestToProtocolsRequest(template.RequestsHeadless) + + case len(template.RequestsSSL) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsSSL) + + case len(template.RequestsWebsocket) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket) + } + template.Executer = executer.NewExecuter(requests, &options) + return nil +} + +// convertRequestToProtocolsRequest is a convenience wrapper to convert +// arbitrary interfaces which are slices of requests from the template to a +// slice of protocols.Request interface items. +func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request { + switch reflect.TypeOf(requests).Kind() { + case reflect.Slice: + s := reflect.ValueOf(requests) + + requestSlice := make([]protocols.Request, s.Len()) + for i := 0; i < s.Len(); i++ { + value := s.Index(i) + valueInterface := value.Interface() + requestSlice[i] = valueInterface.(protocols.Request) + } + return requestSlice + } + return nil +} + +// compileOfflineHTTPRequest iterates all requests if offline http mode is +// specified and collects all matchers for all the base request templates +// (those with URL {{BaseURL}} and it's slash variation.) +func (template *Template) compileOfflineHTTPRequest(options protocols.ExecuterOptions) { + operatorsList := []*operators.Operators{} + +mainLoop: + for _, req := range template.RequestsHTTP { + for _, path := range req.Path { + if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { + break mainLoop + } + } + operatorsList = append(operatorsList, &req.Operators) + } + if len(operatorsList) > 0 { + options.Operators = operatorsList + template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + } +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 85213aa01..f71fa1dfa 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -81,3 +81,33 @@ type Template struct { Path string `yaml:"-" json:"-"` } + +// TemplateTypes is a list of accepted template types +var TemplateTypes = []string{ + "dns", + "file", + "http", + "headless", + "network", + "workflow", +} + +// Type returns the type of the template +func (t *Template) Type() string { + switch { + case len(t.RequestsDNS) > 0: + return "dns" + case len(t.RequestsFile) > 0: + return "file" + case len(t.RequestsHTTP) > 0: + return "http" + case len(t.RequestsHeadless) > 0: + return "headless" + case len(t.RequestsNetwork) > 0: + return "network" + case len(t.Workflows) > 0: + return "workflow" + default: + return "" + } +} diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 9d510f2e2..246c4bf8e 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -62,19 +62,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr return nil } for _, path := range paths { - opts := protocols.ExecuterOptions{ - Output: options.Output, - Options: options.Options, - Progress: options.Progress, - Catalog: options.Catalog, - Browser: options.Browser, - RateLimiter: options.RateLimiter, - IssuesClient: options.IssuesClient, - Interactsh: options.Interactsh, - ProjectFile: options.ProjectFile, - HostErrorsCache: options.HostErrorsCache, - } - template, err := Parse(path, preprocessor, opts) + template, err := Parse(path, preprocessor, options.Copy()) if err != nil { gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err) continue diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245..7f5a1851b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -77,6 +77,10 @@ type Options struct { BulkSize int // TemplateThreads is the number of templates executed in parallel TemplateThreads int + // HeadlessBulkSize is the of targets analyzed in parallel for each headless template + HeadlessBulkSize int + // HeadlessTemplateThreads is the number of headless templates executed in parallel + HeadlessTemplateThreads int // Timeout is the seconds to wait for a response from the server. Timeout int // Retries is the number of times to retry the request @@ -177,3 +181,17 @@ func (options *Options) AddVarPayload(key string, value interface{}) { func (options *Options) VarsPayload() map[string]interface{} { return options.varsPayload } + +// DefaultOptions returns default options for nuclei +func DefaultOptions() *Options { + return &Options{ + RateLimit: 150, + BulkSize: 25, + TemplateThreads: 25, + HeadlessBulkSize: 10, + HeadlessTemplateThreads: 10, + Timeout: 5, + Retries: 1, + MaxHostError: 30, + } +}