diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 3fad543dc..66333e821 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -11,7 +11,7 @@ func main() { nucleiRunner, err := runner.New(options) if err != nil { - gologger.Fatalf("Could not create runner: %s\n", err) + gologger.Fatal().Msgf("Could not create runner: %s\n", err) } nucleiRunner.RunEnumeration() diff --git a/v2/go.mod b/v2/go.mod index c3304b641..4b25d93ff 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,27 +6,32 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/corpix/uarand v0.1.1 - github.com/d5/tengo/v2 v2.6.2 github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.16.1 + github.com/kr/pretty v0.1.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/miekg/dns v1.1.35 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.7 github.com/projectdiscovery/collaborator v0.0.2 github.com/projectdiscovery/fastdialer v0.0.2 - github.com/projectdiscovery/gologger v1.0.1 + github.com/projectdiscovery/gologger v1.1.3 github.com/projectdiscovery/hmap v0.0.1 - github.com/projectdiscovery/nuclei/v2 v2.2.0 github.com/projectdiscovery/rawhttp v0.0.4 github.com/projectdiscovery/retryabledns v1.0.5 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/segmentio/ksuid v1.0.3 github.com/spaolacci/murmur3 v1.1.0 + github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.6.1 + go.uber.org/atomic v1.7.0 + go.uber.org/multierr v1.6.0 go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20201216054612-986b41b23924 + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/v2/go.sum b/v2/go.sum index 83a7183d2..20b07aa3c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -8,8 +8,6 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= -github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= -github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -34,6 +32,11 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -43,8 +46,12 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= @@ -55,21 +62,16 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/clistats v0.0.7 h1:Q/erjrk2p3BIQq1RaHVtBpgboghNz0u1/lyQ2fr8Cn0= github.com/projectdiscovery/clistats v0.0.7/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= -github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s= github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= -github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc= github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc= github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA= -github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= -github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= +github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU= +github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= -github.com/projectdiscovery/nuclei/v2 v2.2.0 h1:nUrTXM/AIJ8PfEPxEl/pkAHj7iu0TgAkE3e075a1JN0= -github.com/projectdiscovery/nuclei/v2 v2.2.0/go.mod h1:JIgYr5seElQh161hT/BUw3g1C4UuWR+VAcT16aZdyJ8= github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E= github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI= @@ -78,11 +80,12 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= -github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= @@ -91,11 +94,12 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -105,18 +109,18 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -131,13 +135,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 5186db62e..4134cb6b1 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -5,6 +5,7 @@ import ( "flag" "net/url" "os" + "path" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -16,6 +17,9 @@ import ( func ParseOptions() *types.Options { options := &types.Options{} + home, _ := os.UserHomeDir() + templatesDirectory := path.Join(home, "nuclei-templates") + flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") @@ -39,7 +43,7 @@ func ParseOptions() *types.Options { flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") - flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates") + flag.StringVar(&options.TemplatesDirectory, "update-directory", templatesDirectory, "Directory to use for storing nuclei-templates") flag.BoolVar(&options.JSON, "json", false, "Write json output to files") flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output") flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan") diff --git a/v2/internal/runner/paths.go b/v2/internal/runner/paths.go deleted file mode 100644 index be4b42a33..000000000 --- a/v2/internal/runner/paths.go +++ /dev/null @@ -1,44 +0,0 @@ -package runner - -import ( - "fmt" - "os" - "path" - "strings" - - "github.com/projectdiscovery/gologger" -) - -// isRelative checks if a given path is a relative path -func isRelative(filePath string) bool { - if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") { - return false - } - return true -} - -// resolvePath gets the absolute path to the template by either -// looking in the current directory or checking the nuclei templates directory. -// -// Current directory is given preference over the nuclei-templates directory. -func (r *Runner) resolvePath(templateName string) (string, error) { - curDirectory, err := os.Getwd() - if err != nil { - return "", err - } - - templatePath := path.Join(curDirectory, templateName) - if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debug().Msgf("Found template in current directory: %s\n", templatePath) - return templatePath, nil - } - - if r.templatesConfig != nil { - templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName) - if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debug().Msgf("Found template in nuclei-templates directory: %s\n", templatePath) - return templatePath, nil - } - } - return "", fmt.Errorf("no such path found: %s", templateName) -} diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 82fbb46f7..c4bfe2bfa 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -6,319 +6,57 @@ import ( "path" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" ) -/* -// workflowTemplates contains the initialized workflow templates per template group -type workflowTemplates struct { - Name string - Templates []*workflows.Template -} - -// processTemplateWithList processes a template and runs the enumeration on all the targets -func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { - var httpExecuter *executer.HTTPExecuter - var dnsExecuter *executer.DNSExecuter - var err error - - // Create an executer based on the request type. - switch value := request.(type) { - case *requests.DNSRequest: - dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Template: template, - DNSRequest: value, - Writer: r.output, - VHost: r.options.Vhost, - JSON: r.options.JSON, - JSONRequests: r.options.JSONRequests, - NoMeta: r.options.NoMeta, - ColoredOutput: !r.options.NoColor, - Colorizer: r.colorizer, - Decolorizer: r.decolorizer, - RateLimiter: r.ratelimiter, - }) - case *requests.BulkHTTPRequest: - httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Template: template, - BulkHTTPRequest: value, - Writer: r.output, - Timeout: r.options.Timeout, - Retries: r.options.Retries, - ProxyURL: r.options.ProxyURL, - ProxySocksURL: r.options.ProxySocksURL, - RandomAgent: r.options.RandomAgent, - CustomHeaders: r.options.CustomHeaders, - JSON: r.options.JSON, - Vhost: r.options.Vhost, - JSONRequests: r.options.JSONRequests, - NoMeta: r.options.NoMeta, - CookieReuse: value.CookieReuse, - ColoredOutput: !r.options.NoColor, - Colorizer: &r.colorizer, - Decolorizer: r.decolorizer, - StopAtFirstMatch: r.options.StopAtFirstMatch, - PF: r.pf, - Dialer: r.dialer, - RateLimiter: r.ratelimiter, - }) - } - - if err != nil { - p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount()) - gologger.Warningf("Could not create http client: %s\n", err) - - return false - } - - var globalresult atomicboolean.AtomBool - +// processTemplateWithList process a template on the URL list +func (r *Runner) processTemplateWithList(template *templates.Template) bool { + results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hm.Scan(func(k, _ []byte) error { + r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) wg.Add() go func(URL string) { defer wg.Done() - - var result *executer.Result - - if httpExecuter != nil { - result = httpExecuter.ExecuteHTTP(p, URL) - globalresult.Or(result.GotResults) - } - - if dnsExecuter != nil { - result = dnsExecuter.ExecuteDNS(p, URL) - globalresult.Or(result.GotResults) - } - - if result.Error != nil { - gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error) + match, err := template.Executer.Execute(URL) + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } else { + results.CAS(false, match) } }(URL) return nil }) - wg.Wait() - - // See if we got any results from the executers - return globalresult.Get() + return results.Load() } -// ProcessWorkflowWithList coming from stdin or list of targets -func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) bool { - result := false - - workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow) - if err != nil { - gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err) - return false - } - logicBytes := []byte(workflow.Logic) - +// processTemplateWithList process a template on the URL list +func (r *Runner) processWorkflowWithList(template *templates.Template) bool { + results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hm.Scan(func(k, _ []byte) error { - targetURL := string(k) + + r.hostMap.Scan(func(k, _ []byte) error { + URL := string(k) wg.Add() - - go func(targetURL string) { + go func(URL string) { defer wg.Done() - - script := tengo.NewScript(logicBytes) - if !r.options.Sandbox { - script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) - } else { - script.SetImports(stdlib.GetModuleMap(sandboxedModules...)) - } - - variables := make(map[string]*workflows.NucleiVar) - for _, workflowTemplate := range *workflowTemplatesList { - name := workflowTemplate.Name - variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL} - err := script.Add(name, variable) - if err != nil { - gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err) - continue - } - variables[name] = variable - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.options.MaxWorkflowDuration)*time.Minute) - defer cancel() - - _, err := script.RunContext(ctx) + match, err := template.RunWorkflow(URL) if err != nil { - gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err) + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } else { + results.CAS(false, match) } - - for _, variable := range variables { - result = !variable.IsFalsy() - if result { - break - } - } - }(targetURL) + }(URL) return nil }) - wg.Wait() - - return result + return results.Load() } -func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workflows.Workflow) (*[]workflowTemplates, error) { - var jar *cookiejar.Jar - - if workflow.CookieReuse { - var err error - jar, err = cookiejar.New(nil) - if err != nil { - return nil, err - } - } - - // Single yaml provided - var wflTemplatesList []workflowTemplates - - for name, value := range workflow.Variables { - // Check if the template is an absolute path or relative path. - // If the path is absolute, use it. Otherwise, - if isRelative(value) { - newPath, err := r.resolvePath(value) - if err != nil { - newPath, err = resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value) - if err != nil { - return nil, err - } - } - - value = newPath - } - - var wtlst []*workflows.Template - - if strings.HasSuffix(value, ".yaml") { - t, err := templates.Parse(value) - if err != nil { - return nil, err - } - - template := &workflows.Template{Progress: p} - if len(t.BulkRequestsHTTP) > 0 { - template.HTTPOptions = &executer.HTTPOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Writer: r.output, - Template: t, - Timeout: r.options.Timeout, - Retries: r.options.Retries, - ProxyURL: r.options.ProxyURL, - ProxySocksURL: r.options.ProxySocksURL, - RandomAgent: r.options.RandomAgent, - CustomHeaders: r.options.CustomHeaders, - Vhost: r.options.Vhost, - JSON: r.options.JSON, - JSONRequests: r.options.JSONRequests, - CookieJar: jar, - ColoredOutput: !r.options.NoColor, - Colorizer: &r.colorizer, - Decolorizer: r.decolorizer, - PF: r.pf, - RateLimiter: r.ratelimiter, - NoMeta: r.options.NoMeta, - StopAtFirstMatch: r.options.StopAtFirstMatch, - Dialer: r.dialer, - } - } else if len(t.RequestsDNS) > 0 { - template.DNSOptions = &executer.DNSOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Template: t, - Writer: r.output, - VHost: r.options.Vhost, - JSON: r.options.JSON, - JSONRequests: r.options.JSONRequests, - ColoredOutput: !r.options.NoColor, - Colorizer: r.colorizer, - Decolorizer: r.decolorizer, - NoMeta: r.options.NoMeta, - RateLimiter: r.ratelimiter, - } - } - - if template.DNSOptions != nil || template.HTTPOptions != nil { - wtlst = append(wtlst, template) - } - } else { - matches := []string{} - - err := godirwalk.Walk(value, &godirwalk.Options{ - Callback: func(path string, d *godirwalk.Dirent) error { - if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - matches = append(matches, path) - } - - return nil - }, - ErrorCallback: func(path string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Unsorted: true, - }) - - if err != nil { - return nil, err - } - - // 0 matches means no templates were found in directory - if len(matches) == 0 { - return nil, fmt.Errorf("no match found in the directory %s", value) - } - - for _, match := range matches { - t, err := templates.Parse(match) - if err != nil { - return nil, err - } - template := &workflows.Template{Progress: p} - if len(t.BulkRequestsHTTP) > 0 { - template.HTTPOptions = &executer.HTTPOptions{ - Debug: r.options.Debug, - Writer: r.output, - Template: t, - Timeout: r.options.Timeout, - Retries: r.options.Retries, - ProxyURL: r.options.ProxyURL, - ProxySocksURL: r.options.ProxySocksURL, - RandomAgent: r.options.RandomAgent, - CustomHeaders: r.options.CustomHeaders, - Vhost: r.options.Vhost, - CookieJar: jar, - TraceLog: r.traceLog, - } - } else if len(t.RequestsDNS) > 0 { - template.DNSOptions = &executer.DNSOptions{ - Debug: r.options.Debug, - Template: t, - Writer: r.output, - VHost: r.options.Vhost, - TraceLog: r.traceLog, - } - } - if template.DNSOptions != nil || template.HTTPOptions != nil { - wtlst = append(wtlst, template) - } - } - } - wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst}) - } - return &wflTemplatesList, nil -}*/ - // resolvePathWithBaseFolder resolves a path with the base folder func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) { templatePath := path.Join(baseFolder, templateName) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 368616ac5..3b6b5889d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -11,13 +11,13 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/collaborator" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" + "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" "go.uber.org/ratelimit" ) @@ -29,6 +29,7 @@ type Runner struct { templatesConfig *nucleiConfig options *types.Options projectFile *projectfile.ProjectFile + catalogue *catalogue.Catalogue progress *progress.Progress colorizer aurora.Aurora severityColors *colorizer.Colorizer @@ -61,6 +62,7 @@ func New(options *types.Options) (*Runner, error) { if runner.templatesConfig != nil { runner.readNucleiIgnoreFile() } + runner.catalogue = catalogue.New(runner.options.TemplatesDirectory) if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil { gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err) @@ -176,8 +178,8 @@ func (r *Runner) Close() { // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { // resolves input templates definitions and any optional exclusion - includedTemplates := r.getTemplatesFor(r.options.Templates) - excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates) + includedTemplates := r.catalogue.GetTemplatesPath(r.options.Templates) + excludedTemplates := r.catalogue.GetTemplatesPath(r.options.ExcludedTemplates) // defaults to all templates allTemplates := includedTemplates @@ -222,10 +224,10 @@ func (r *Runner) RunEnumeration() { if t.Workflow != nil { continue } - totalRequests += int64(t.Requests()) * r.inputCount + totalRequests += int64(t.TotalRequests) * r.inputCount } - results := atomicboolean.New() + results := &atomic.Bool{} wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) // Starts polling or ignore collaborator.DefaultCollaborator.Poll() @@ -241,14 +243,9 @@ func (r *Runner) RunEnumeration() { wgtemplates.Add() go func(template *templates.Template) { if template.Workflow != nil { - results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow))) - - } - for _, request := range template.RequestsDNS { - results.Or(r.processTemplateWithList(p, tt, request)) - } - for _, request := range template.RequestsHTTP { - results.Or(r.processTemplateWithList(p, tt, request)) + results.CAS(false, r.processWorkflowWithList(template)) + } else { + results.CAS(false, r.processTemplateWithList(template)) } }(t) } @@ -256,7 +253,7 @@ func (r *Runner) RunEnumeration() { p.Stop() } - if !results.Get() { + if !results.Load() { if r.output != nil { r.output.Close() os.Remove(r.options.Output) diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index b8a11d8f1..1497c1287 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -3,7 +3,6 @@ package runner import ( "fmt" "os" - "path/filepath" "strings" "github.com/karrick/godirwalk" @@ -12,105 +11,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) -// getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths. -func (r *Runner) getTemplatesFor(definitions []string) []string { - // keeps track of processed dirs and files - processed := make(map[string]bool) - allTemplates := []string{} - - // parses user input, handle file/directory cases and produce a list of unique templates - for _, t := range definitions { - var absPath string - var err error - - if strings.Contains(t, "*") { - dirs := strings.Split(t, "/") - priorDir := strings.Join(dirs[:len(dirs)-1], "/") - absPath, err = r.resolvePathIfRelative(priorDir) - absPath += "/" + dirs[len(dirs)-1] - } else { - // resolve and convert relative to absolute path - absPath, err = r.resolvePathIfRelative(t) - } - if err != nil { - gologger.Error().Msgf("Could not find template file '%s': %s\n", t, err) - continue - } - - // Template input includes a wildcard - if strings.Contains(absPath, "*") { - var matches []string - matches, err = filepath.Glob(absPath) - if err != nil { - gologger.Error().Msgf("Wildcard found, but unable to glob '%s': %s\n", absPath, err) - continue - } - - // couldn't find templates in directory - if len(matches) == 0 { - gologger.Error().Msgf("Error, no templates were found with '%s'.\n", absPath) - continue - } else { - gologger.Verbose().Msgf("Identified %d templates\n", len(matches)) - } - - for _, match := range matches { - if !r.checkIfInNucleiIgnore(match) { - processed[match] = true - allTemplates = append(allTemplates, match) - } - } - } else { - // determine file/directory - isFile, err := isFilePath(absPath) - if err != nil { - gologger.Error().Msgf("Could not stat '%s': %s\n", absPath, err) - continue - } - // test for uniqueness - if !isNewPath(absPath, processed) { - continue - } - // mark this absolute path as processed - // - if it's a file, we'll never process it again - // - if it's a dir, we'll never walk it again - processed[absPath] = true - - if isFile { - allTemplates = append(allTemplates, absPath) - } else { - matches := []string{} - - // Recursively walk down the Templates directory and run all the template file checks - err := directoryWalker(absPath, - func(path string, d *godirwalk.Dirent) error { - if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) { - matches = append(matches, path) - processed[path] = true - } - } - return nil - }, - ) - // directory couldn't be walked - if err != nil { - gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", absPath, err) - continue - } - - // couldn't find templates in directory - if len(matches) == 0 { - gologger.Error().Msgf("Error, no templates were found in '%s'.\n", absPath) - continue - } - allTemplates = append(allTemplates, matches...) - } - } - } - return allTemplates -} - // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered // by severity, along with a flag indicating if workflows are present. func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []*templates.Template, workflowCount int) { @@ -148,6 +48,7 @@ func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { Output: r.output, Options: r.options, Progress: r.progress, + Catalogue: r.catalogue, RateLimiter: r.ratelimiter, ProjectFile: r.projectFile, } @@ -211,19 +112,6 @@ func (r *Runner) listAvailableTemplates() { } } -func (r *Runner) resolvePathIfRelative(filePath string) (string, error) { - if isRelative(filePath) { - newPath, err := r.resolvePath(filePath) - - if err != nil { - return "", err - } - return newPath, nil - } - - return filePath, nil -} - func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool { for _, s := range allowedSeverities { if s != "" && strings.HasPrefix(templateSeverity, s) { diff --git a/v2/pkg/catalogue/catalogue.go b/v2/pkg/catalogue/catalogue.go new file mode 100644 index 000000000..eaba68593 --- /dev/null +++ b/v2/pkg/catalogue/catalogue.go @@ -0,0 +1,14 @@ +package catalogue + +// Catalogue is a template catalouge helper implementation +type Catalogue struct { + ignoreFiles []string + templatesDirectory string +} + +// New creates a new catalogue structure using provided input items +func New(directory string) *Catalogue { + catalogue := &Catalogue{templatesDirectory: directory} + catalogue.readNucleiIgnoreFile() + return catalogue +} diff --git a/v2/pkg/catalogue/find.go b/v2/pkg/catalogue/find.go new file mode 100644 index 000000000..0191f7873 --- /dev/null +++ b/v2/pkg/catalogue/find.go @@ -0,0 +1,153 @@ +package catalogue + +import ( + "os" + "path" + "path/filepath" + "strings" + + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" +) + +// GetTemplatesPath returns a list of absolute paths for the provided template list. +func (c *Catalogue) GetTemplatesPath(definitions []string) []string { + // keeps track of processed dirs and files + processed := make(map[string]bool) + allTemplates := []string{} + + for _, t := range definitions { + paths, err := c.GetTemplatePath(t) + if err != nil { + gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) + } + for _, path := range paths { + if _, ok := processed[path]; !ok { + processed[path] = true + allTemplates = append(allTemplates, path) + } + } + } + gologger.Verbose().Msgf("Identified %d templates", len(allTemplates)) + return allTemplates +} + +// GetTemplatePath parses the specified input template path and returns a compiled +// list of finished absolute paths to the templates evaluating any glob patterns +// or folders provided as in. +func (c *Catalogue) GetTemplatePath(target string) ([]string, error) { + processed := make(map[string]struct{}) + + absPath, err := c.convertPathToAbsolute(target) + if err != nil { + return nil, errors.Wrapf(err, "could not find template file") + } + + // Template input includes a wildcard + if strings.Contains(absPath, "*") { + matches, err := c.findGlobPathMatches(absPath, processed) + if err != nil { + return nil, errors.Wrap(err, "could not find glob matches") + } + if len(matches) == 0 { + return nil, errors.Errorf("no templates found for path") + } + return matches, nil + } + + // Template input is either a file or a directory + match, file, err := c.findFileMatches(absPath, processed) + if err != nil { + return nil, errors.Wrap(err, "could not find file") + } + if file { + if match != "" { + return []string{match}, nil + } + return nil, nil + } + + // Recursively walk down the Templates directory and run all + // the template file checks + matches, err := c.findDirectoryMatches(absPath, processed) + if err != nil { + return nil, errors.Wrap(err, "could not find directory matches") + } + if len(matches) == 0 { + return nil, errors.Errorf("no templates found in path") + } + return matches, nil +} + +// convertPathToAbsolute resolves the paths provided to absolute paths +// before doing any operations on them regardless of them being blob, folders, files, etc. +func (c *Catalogue) convertPathToAbsolute(t string) (string, error) { + if strings.Contains(t, "*") { + file := path.Base(t) + absPath, err := c.ResolvePath(path.Dir(t), "") + if err != nil { + return "", err + } + return path.Join(absPath, file), nil + } + return c.ResolvePath(t, "") +} + +// findGlobPathMatches returns the matched files from a glob path +func (c *Catalogue) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { + matches, err := filepath.Glob(absPath) + if err != nil { + return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) + } + results := make([]string, 0, len(matches)) + for _, match := range matches { + if _, ok := processed[match]; !ok { + processed[match] = struct{}{} + results = append(results, match) + } + } + return results, nil +} + +// findFileMatches finds if a path is an absolute file. If the path +// is a file, it returns true otherwise false with no errors. +func (c *Catalogue) findFileMatches(absPath string, processed map[string]struct{}) (string, bool, error) { + info, err := os.Stat(absPath) + if err != nil { + return "", false, err + } + if !info.Mode().IsRegular() { + return "", false, nil + } + if _, ok := processed[absPath]; !ok { + processed[absPath] = struct{}{} + return absPath, true, nil + } + return "", true, nil +} + +// findDirectoryMatches finds matches for templates from a directory +func (c *Catalogue) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { + var results []string + err := godirwalk.Walk(absPath, &godirwalk.Options{ + Unsorted: true, + ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { + return godirwalk.SkipNode + }, + Callback: func(path string, d *godirwalk.Dirent) error { + if !d.IsDir() && strings.HasSuffix(path, ".yaml") { + if c.checkIfInNucleiIgnore(path) { + return nil + } + + if _, ok := processed[path]; !ok { + results = append(results, path) + processed[path] = struct{}{} + } + } + return nil + }, + }) + return results, err +} diff --git a/v2/pkg/catalogue/ignore.go b/v2/pkg/catalogue/ignore.go new file mode 100644 index 000000000..591d03b44 --- /dev/null +++ b/v2/pkg/catalogue/ignore.go @@ -0,0 +1,70 @@ +package catalogue + +import ( + "bufio" + "os" + "path" + "strings" +) + +const nucleiIgnoreFile = ".nuclei-ignore" + +// readNucleiIgnoreFile reads the nuclei ignore file marking it in map +func (c *Catalogue) readNucleiIgnoreFile() { + file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile)) + if err != nil { + return + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + if strings.HasPrefix(text, "#") { + continue + } + c.ignoreFiles = append(c.ignoreFiles, text) + } +} + +// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules. +func (c *Catalogue) checkIfInNucleiIgnore(item string) bool { + if c.templatesDirectory == "" { + return false + } + + for _, paths := range c.ignoreFiles { + // If we have a path to ignore, check if it's in the item. + if paths[len(paths)-1] == '/' { + if strings.Contains(item, paths) { + return true + } + + continue + } + // Check for file based extension in ignores + if strings.HasSuffix(item, paths) { + return true + } + } + return false +} + +// ignoreFilesWithExcludes ignores results with exclude paths +func (c *Catalogue) ignoreFilesWithExcludes(results, excluded []string) []string { + excludeMap := make(map[string]struct{}, len(excluded)) + for _, excl := range excluded { + excludeMap[excl] = struct{}{} + } + + templates := make([]string, 0, len(results)) + for _, incl := range results { + if _, found := excludeMap[incl]; !found { + templates = append(templates, incl) + } + } + return templates +} diff --git a/v2/pkg/catalogue/path.go b/v2/pkg/catalogue/path.go new file mode 100644 index 000000000..2ff00d0e7 --- /dev/null +++ b/v2/pkg/catalogue/path.go @@ -0,0 +1,45 @@ +package catalogue + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +// ResolvePath resolves the path to an absolute one in various ways. +// +// It checks if the filename is an absolute path, looks in the current directory +// or checking the nuclei templates directory. If a second path is given, +// it also tries to find paths relative to that second path. +func (c *Catalogue) ResolvePath(templateName, second string) (string, error) { + if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") { + return templateName, nil + } + + if second != "" { + secondBasePath := path.Join(filepath.Dir(second), templateName) + if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) { + return secondBasePath, nil + } + } + + curDirectory, err := os.Getwd() + if err != nil { + return "", err + } + + templatePath := path.Join(curDirectory, templateName) + if _, err := os.Stat(templatePath); !os.IsNotExist(err) { + return templatePath, nil + } + + if c.templatesDirectory != "" { + templatePath := path.Join(c.templatesDirectory, templateName) + if _, err := os.Stat(templatePath); !os.IsNotExist(err) { + return templatePath, nil + } + } + return "", fmt.Errorf("no such path found: %s", templateName) +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 49794c52b..23e79d2eb 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -130,13 +130,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values) } -// Remaining returns the remaining number of requests for the generator -func (r *requestGenerator) Remaining() int { +// Total returns the total number of requests for the generator +func (r *requestGenerator) Total() int { if r.payloadIterator != nil { - payloadRemaining := r.payloadIterator.Remaining() - return (len(r.request.Raw) - r.currentIndex + 1) * payloadRemaining + return len(r.request.Raw) * r.payloadIterator.Remaining() } - return len(r.request.Path) - r.currentIndex + 1 + return len(r.request.Path) } // baseURLWithTemplatePrefs returns the url for BaseURL keeping diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/http/executer.go index fdcdb7586..df92c1c34 100644 --- a/v2/pkg/protocols/http/executer.go +++ b/v2/pkg/protocols/http/executer.go @@ -33,7 +33,7 @@ func (e *Executer) Compile() error { func (e *Executer) Requests() int { var count int for _, request := range e.requests { - count += int64(request.Requests()) + count += int(request.Requests()) } return count } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 89dcb74da..2edec90e0 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -75,7 +75,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } swg.Add() @@ -136,7 +136,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } request.pipelinedClient = pipeclient @@ -187,7 +187,7 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } @@ -201,7 +201,7 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int e.options.Progress.IncrementRequests() if request.original.options.Options.StopAtFirstMatch && len(output) > 0 { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) break } } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 67ea9e21d..3293fc8e1 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -2,6 +2,7 @@ package protocols import ( "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -38,6 +39,8 @@ type ExecuterOptions struct { Progress *progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. RateLimiter ratelimit.Limiter + // Catalogue is a template catalogue implementation for nuclei + Catalogue *catalogue.Catalogue // ProjectFile is the project file for nuclei ProjectFile *projectfile.ProjectFile } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index be7e24df3..576608921 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) @@ -27,7 +28,6 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { defer f.Close() // Setting up variables regarding template metadata - template.path = file options.TemplateID = template.ID options.TemplateInfo = template.Info options.TemplatePath = file @@ -37,27 +37,90 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { return nil, fmt.Errorf("both http and dns requests for %s", template.ID) } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflow.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } + // Compile the workflow request + if template.Workflow != nil { + if err := template.compileWorkflow(options); err != nil { + return nil, errors.Wrap(err, "could not compile workflow") + } + } + // Compile the requests found for _, request := range template.RequestsDNS { - template.totalRequests += request.Requests() + template.TotalRequests += request.Requests() } for _, request := range template.RequestsHTTP { - template.totalRequests += request.Requests() + template.TotalRequests += request.Requests() } if len(template.RequestsDNS) > 0 { - template.executer = dns.NewExecuter(template.RequestsDNS, options) - err = template.executer.Compile() + template.Executer = dns.NewExecuter(template.RequestsDNS, options) + err = template.Executer.Compile() } if len(template.RequestsHTTP) > 0 { - template.executer = http.NewExecuter(template.RequestsHTTP, options) - err = template.executer.Compile() + template.Executer = http.NewExecuter(template.RequestsHTTP, options) + err = template.Executer.Compile() } if err != nil { return nil, errors.Wrap(err, "could not compile request") } return template, nil } + +// compileWorkflow compiles the workflow for execution +func (t *Template) compileWorkflow(options *protocols.ExecuterOptions) error { + for _, workflow := range t.Workflows { + if err := t.parseWorkflow(workflow, options); err != nil { + return err + } + } + return nil +} + +// parseWorkflow parses and compiles all templates in a workflow recursively +func (t *Template) parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error { + if err := t.parseWorkflowTemplate(workflow, options); err != nil { + return err + } + for _, subtemplates := range workflow.Subtemplates { + if err := t.parseWorkflow(subtemplates, options); err != nil { + return err + } + } + for _, matcher := range workflow.Matchers { + for _, subtemplates := range matcher.Subtemplates { + if err := t.parseWorkflow(subtemplates, options); err != nil { + return err + } + } + } + return nil +} + +// parseWorkflowTemplate parses a workflow template creating an executer +func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error { + opts := protocols.ExecuterOptions{ + Output: options.Output, + Options: options.Options, + Progress: options.Progress, + Catalogue: options.Catalogue, + RateLimiter: options.RateLimiter, + ProjectFile: options.ProjectFile, + } + paths, err := options.Catalogue.GetTemplatePath(workflow.Template) + if err != nil { + return errors.Wrap(err, "could not get workflow template") + } + if len(paths) != 1 { + return errors.Wrap(err, "invalid number of templates matched") + } + + template, err := Parse(paths[0], &opts) + if err != nil { + return errors.Wrap(err, "could not parse workflow template") + } + workflow.Executer = template.Executer + return nil +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index bbd679db4..d822c4ddd 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -17,21 +17,10 @@ type Template struct { RequestsHTTP []*http.Request `yaml:"requests,omitempty"` // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` - // Workflows is a yaml based workflow declaration code. *workflows.Workflow - - path string - totalRequests int - executer protocols.Executer -} - -// GetPath returns the path of the template. -func (t *Template) GetPath() string { - return t.path -} - -// Requests returns the number of requests for the template -func (t *Template) Requests() int { - return t.totalRequests + // TotalRequests is the total number of requests for the template. + TotalRequests int + // Executer is the actual template executor for running template requests + Executer protocols.Executer } diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 5dc43e0ff..30f2feece 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -3,7 +3,7 @@ package workflows import "go.uber.org/atomic" // RunWorkflow runs a workflow on an input and returns true or false -func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) { +func (w *Workflow) RunWorkflow(input string) (bool, error) { results := &atomic.Bool{} for _, template := range w.Workflows { @@ -20,7 +20,9 @@ func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) { func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool) error { var firstMatched bool if len(template.Matchers) == 0 { - matched, err := template.executer.Execute(input) + w.options.Progress.AddToTotal(int64(template.Executer.Requests())) + + matched, err := template.Executer.Execute(input) if err != nil { return err } @@ -29,7 +31,9 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } if len(template.Matchers) > 0 { - output, err := template.executer.ExecuteWithResults(input) + w.options.Progress.AddToTotal(int64(template.Executer.Requests())) + + output, err := template.Executer.ExecuteWithResults(input) if err != nil { return err } diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index f69cdcaf1..73e752972 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -6,6 +6,8 @@ import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" type Workflow struct { // Workflows is a yaml based workflow declaration code. Workflows []*WorkflowTemplate `yaml:"workflows"` + + options *protocols.ExecuterOptions } // WorkflowTemplate is a template to be ran as part of a workflow @@ -16,8 +18,8 @@ type WorkflowTemplate struct { Matchers []*Matcher `yaml:"matchers"` // Subtemplates are ran if the template matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates"` - - executer protocols.Executer + // Executer performs the actual execution for the workflow template + Executer protocols.Executer } // Matcher performs conditional matching on the workflow template results.