Added catalogue + template-workflow running + misc

v2/cmd/nuclei/main.go

@@ -11,7 +11,7 @@ func main() {
 
 	nucleiRunner, err := runner.New(options)
 	if err != nil {
-		gologger.Fatalf("Could not create runner: %s\n", err)
+		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
 
 	nucleiRunner.RunEnumeration()

v2/go.mod

@@ -6,27 +6,32 @@ require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/corpix/uarand v0.1.1
-	github.com/d5/tengo/v2 v2.6.2
 	github.com/google/go-github/v32 v32.1.0
 	github.com/json-iterator/go v1.1.10
 	github.com/karrick/godirwalk v1.16.1
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/miekg/dns v1.1.35
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.7
 	github.com/projectdiscovery/collaborator v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.2
-	github.com/projectdiscovery/gologger v1.0.1
+	github.com/projectdiscovery/gologger v1.1.3
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/nuclei/v2 v2.2.0
 	github.com/projectdiscovery/rawhttp v0.0.4
 	github.com/projectdiscovery/retryabledns v1.0.5
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
 	github.com/remeh/sizedwaitgroup v1.0.0
-	github.com/segmentio/ksuid v1.0.3
 	github.com/spaolacci/murmur3 v1.1.0
+	github.com/spf13/cast v1.3.1
 	github.com/stretchr/testify v1.6.1
+	go.uber.org/atomic v1.7.0
+	go.uber.org/multierr v1.6.0
 	go.uber.org/ratelimit v0.1.0
 	golang.org/x/net v0.0.0-20201216054612-986b41b23924
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -8,8 +8,6 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
 github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
 github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
-github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
-github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -34,6 +32,11 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
@@ -43,8 +46,12 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
@@ -55,21 +62,16 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
 github.com/projectdiscovery/clistats v0.0.7 h1:Q/erjrk2p3BIQq1RaHVtBpgboghNz0u1/lyQ2fr8Cn0=
 github.com/projectdiscovery/clistats v0.0.7/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
 github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
 github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
 github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc=
 github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
-github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
+github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU=
-github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
+github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/nuclei/v2 v2.2.0 h1:nUrTXM/AIJ8PfEPxEl/pkAHj7iu0TgAkE3e075a1JN0=
-github.com/projectdiscovery/nuclei/v2 v2.2.0/go.mod h1:JIgYr5seElQh161hT/BUw3g1C4UuWR+VAcT16aZdyJ8=
 github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E=
 github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
@@ -78,11 +80,12 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
-github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
-github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
@@ -91,11 +94,12 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
 go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -105,18 +109,18 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY=
 golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -131,13 +135,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

v2/internal/runner/options.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"net/url"
 	"os"
+	"path"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
@@ -16,6 +17,9 @@ import (
 func ParseOptions() *types.Options {
 	options := &types.Options{}
 
+	home, _ := os.UserHomeDir()
+	templatesDirectory := path.Join(home, "nuclei-templates")
+
 	flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode")
 	flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
 	flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
@@ -39,7 +43,7 @@ func ParseOptions() *types.Options {
 	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
 	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
 	flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
-	flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
+	flag.StringVar(&options.TemplatesDirectory, "update-directory", templatesDirectory, "Directory to use for storing nuclei-templates")
 	flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
 	flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output")
 	flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")

v2/internal/runner/paths.go

@@ -1,44 +0,0 @@
-package runner
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-// isRelative checks if a given path is a relative path
-func isRelative(filePath string) bool {
-	if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") {
-		return false
-	}
-	return true
-}
-
-// resolvePath gets the absolute path to the template by either
-// looking in the current directory or checking the nuclei templates directory.
-//
-// Current directory is given preference over the nuclei-templates directory.
-func (r *Runner) resolvePath(templateName string) (string, error) {
-	curDirectory, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	templatePath := path.Join(curDirectory, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Debug().Msgf("Found template in current directory: %s\n", templatePath)
-		return templatePath, nil
-	}
-
-	if r.templatesConfig != nil {
-		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
-		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-			gologger.Debug().Msgf("Found template in nuclei-templates directory: %s\n", templatePath)
-			return templatePath, nil
-		}
-	}
-	return "", fmt.Errorf("no such path found: %s", templateName)
-}

v2/internal/runner/processor.go

@@ -6,319 +6,57 @@ import (
 	"path"
 
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
 )
 
-/*
+// processTemplateWithList process a template on the URL list
-// workflowTemplates contains the initialized workflow templates per template group
+func (r *Runner) processTemplateWithList(template *templates.Template) bool {
-type workflowTemplates struct {
+	results := &atomic.Bool{}
-	Name      string
-	Templates []*workflows.Template
-}
-
-// processTemplateWithList processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
-	var httpExecuter *executer.HTTPExecuter
-	var dnsExecuter *executer.DNSExecuter
-	var err error
-
-	// Create an executer based on the request type.
-	switch value := request.(type) {
-	case *requests.DNSRequest:
-		dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
-			TraceLog:      r.traceLog,
-			Debug:         r.options.Debug,
-			Template:      template,
-			DNSRequest:    value,
-			Writer:        r.output,
-			VHost:         r.options.Vhost,
-			JSON:          r.options.JSON,
-			JSONRequests:  r.options.JSONRequests,
-			NoMeta:        r.options.NoMeta,
-			ColoredOutput: !r.options.NoColor,
-			Colorizer:     r.colorizer,
-			Decolorizer:   r.decolorizer,
-			RateLimiter:   r.ratelimiter,
-		})
-	case *requests.BulkHTTPRequest:
-		httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
-			TraceLog:         r.traceLog,
-			Debug:            r.options.Debug,
-			Template:         template,
-			BulkHTTPRequest:  value,
-			Writer:           r.output,
-			Timeout:          r.options.Timeout,
-			Retries:          r.options.Retries,
-			ProxyURL:         r.options.ProxyURL,
-			ProxySocksURL:    r.options.ProxySocksURL,
-			RandomAgent:      r.options.RandomAgent,
-			CustomHeaders:    r.options.CustomHeaders,
-			JSON:             r.options.JSON,
-			Vhost:            r.options.Vhost,
-			JSONRequests:     r.options.JSONRequests,
-			NoMeta:           r.options.NoMeta,
-			CookieReuse:      value.CookieReuse,
-			ColoredOutput:    !r.options.NoColor,
-			Colorizer:        &r.colorizer,
-			Decolorizer:      r.decolorizer,
-			StopAtFirstMatch: r.options.StopAtFirstMatch,
-			PF:               r.pf,
-			Dialer:           r.dialer,
-			RateLimiter:      r.ratelimiter,
-		})
-	}
-
-	if err != nil {
-		p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount())
-		gologger.Warningf("Could not create http client: %s\n", err)
-
-		return false
-	}
-
-	var globalresult atomicboolean.AtomBool
-
 	wg := sizedwaitgroup.New(r.options.BulkSize)
 
-	r.hm.Scan(func(k, _ []byte) error {
+	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()
-
+			match, err := template.Executer.Execute(URL)
-			var result *executer.Result
+			if err != nil {
-
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
-			if httpExecuter != nil {
+			} else {
-				result = httpExecuter.ExecuteHTTP(p, URL)
+				results.CAS(false, match)
-				globalresult.Or(result.GotResults)
-			}
-
-			if dnsExecuter != nil {
-				result = dnsExecuter.ExecuteDNS(p, URL)
-				globalresult.Or(result.GotResults)
-			}
-
-			if result.Error != nil {
-				gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error)
 			}
 		}(URL)
 		return nil
 	})
-
 	wg.Wait()
-
+	return results.Load()
-	// See if we got any results from the executers
-	return globalresult.Get()
 }
 
-// ProcessWorkflowWithList coming from stdin or list of targets
+// processTemplateWithList process a template on the URL list
-func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) bool {
+func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
-	result := false
+	results := &atomic.Bool{}
-
-	workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow)
-	if err != nil {
-		gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err)
-		return false
-	}
-	logicBytes := []byte(workflow.Logic)
-
 	wg := sizedwaitgroup.New(r.options.BulkSize)
-	r.hm.Scan(func(k, _ []byte) error {
+
-		targetURL := string(k)
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
 		wg.Add()
-
+		go func(URL string) {
-		go func(targetURL string) {
 			defer wg.Done()
-
+			match, err := template.RunWorkflow(URL)
-			script := tengo.NewScript(logicBytes)
+			if err != nil {
-			if !r.options.Sandbox {
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
-				script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 			} else {
-				script.SetImports(stdlib.GetModuleMap(sandboxedModules...))
+				results.CAS(false, match)
 			}
-
+		}(URL)
-			variables := make(map[string]*workflows.NucleiVar)
-			for _, workflowTemplate := range *workflowTemplatesList {
-				name := workflowTemplate.Name
-				variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL}
-				err := script.Add(name, variable)
-				if err != nil {
-					gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err)
-					continue
-				}
-				variables[name] = variable
-			}
-
-			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.options.MaxWorkflowDuration)*time.Minute)
-			defer cancel()
-
-			_, err := script.RunContext(ctx)
-			if err != nil {
-				gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
-			}
-
-			for _, variable := range variables {
-				result = !variable.IsFalsy()
-				if result {
-					break
-				}
-			}
-		}(targetURL)
 		return nil
 	})
-
 	wg.Wait()
-
+	return results.Load()
-	return result
 }
 
-func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workflows.Workflow) (*[]workflowTemplates, error) {
-	var jar *cookiejar.Jar
-
-	if workflow.CookieReuse {
-		var err error
-		jar, err = cookiejar.New(nil)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Single yaml provided
-	var wflTemplatesList []workflowTemplates
-
-	for name, value := range workflow.Variables {
-		// Check if the template is an absolute path or relative path.
-		// If the path is absolute, use it. Otherwise,
-		if isRelative(value) {
-			newPath, err := r.resolvePath(value)
-			if err != nil {
-				newPath, err = resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value)
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			value = newPath
-		}
-
-		var wtlst []*workflows.Template
-
-		if strings.HasSuffix(value, ".yaml") {
-			t, err := templates.Parse(value)
-			if err != nil {
-				return nil, err
-			}
-
-			template := &workflows.Template{Progress: p}
-			if len(t.BulkRequestsHTTP) > 0 {
-				template.HTTPOptions = &executer.HTTPOptions{
-					TraceLog:         r.traceLog,
-					Debug:            r.options.Debug,
-					Writer:           r.output,
-					Template:         t,
-					Timeout:          r.options.Timeout,
-					Retries:          r.options.Retries,
-					ProxyURL:         r.options.ProxyURL,
-					ProxySocksURL:    r.options.ProxySocksURL,
-					RandomAgent:      r.options.RandomAgent,
-					CustomHeaders:    r.options.CustomHeaders,
-					Vhost:            r.options.Vhost,
-					JSON:             r.options.JSON,
-					JSONRequests:     r.options.JSONRequests,
-					CookieJar:        jar,
-					ColoredOutput:    !r.options.NoColor,
-					Colorizer:        &r.colorizer,
-					Decolorizer:      r.decolorizer,
-					PF:               r.pf,
-					RateLimiter:      r.ratelimiter,
-					NoMeta:           r.options.NoMeta,
-					StopAtFirstMatch: r.options.StopAtFirstMatch,
-					Dialer:           r.dialer,
-				}
-			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executer.DNSOptions{
-					TraceLog:      r.traceLog,
-					Debug:         r.options.Debug,
-					Template:      t,
-					Writer:        r.output,
-					VHost:         r.options.Vhost,
-					JSON:          r.options.JSON,
-					JSONRequests:  r.options.JSONRequests,
-					ColoredOutput: !r.options.NoColor,
-					Colorizer:     r.colorizer,
-					Decolorizer:   r.decolorizer,
-					NoMeta:        r.options.NoMeta,
-					RateLimiter:   r.ratelimiter,
-				}
-			}
-
-			if template.DNSOptions != nil || template.HTTPOptions != nil {
-				wtlst = append(wtlst, template)
-			}
-		} else {
-			matches := []string{}
-
-			err := godirwalk.Walk(value, &godirwalk.Options{
-				Callback: func(path string, d *godirwalk.Dirent) error {
-					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-						matches = append(matches, path)
-					}
-
-					return nil
-				},
-				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-					return godirwalk.SkipNode
-				},
-				Unsorted: true,
-			})
-
-			if err != nil {
-				return nil, err
-			}
-
-			// 0 matches means no templates were found in directory
-			if len(matches) == 0 {
-				return nil, fmt.Errorf("no match found in the directory %s", value)
-			}
-
-			for _, match := range matches {
-				t, err := templates.Parse(match)
-				if err != nil {
-					return nil, err
-				}
-				template := &workflows.Template{Progress: p}
-				if len(t.BulkRequestsHTTP) > 0 {
-					template.HTTPOptions = &executer.HTTPOptions{
-						Debug:         r.options.Debug,
-						Writer:        r.output,
-						Template:      t,
-						Timeout:       r.options.Timeout,
-						Retries:       r.options.Retries,
-						ProxyURL:      r.options.ProxyURL,
-						ProxySocksURL: r.options.ProxySocksURL,
-						RandomAgent:   r.options.RandomAgent,
-						CustomHeaders: r.options.CustomHeaders,
-						Vhost:         r.options.Vhost,
-						CookieJar:     jar,
-						TraceLog:      r.traceLog,
-					}
-				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executer.DNSOptions{
-						Debug:    r.options.Debug,
-						Template: t,
-						Writer:   r.output,
-						VHost:    r.options.Vhost,
-						TraceLog: r.traceLog,
-					}
-				}
-				if template.DNSOptions != nil || template.HTTPOptions != nil {
-					wtlst = append(wtlst, template)
-				}
-			}
-		}
-		wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst})
-	}
-	return &wflTemplatesList, nil
-}*/
-
 // resolvePathWithBaseFolder resolves a path with the base folder
 func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) {
 	templatePath := path.Join(baseFolder, templateName)

v2/internal/runner/runner.go

@@ -11,13 +11,13 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalogue"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
 )
 
@@ -29,6 +29,7 @@ type Runner struct {
 	templatesConfig *nucleiConfig
 	options         *types.Options
 	projectFile     *projectfile.ProjectFile
+	catalogue       *catalogue.Catalogue
 	progress        *progress.Progress
 	colorizer       aurora.Aurora
 	severityColors  *colorizer.Colorizer
@@ -61,6 +62,7 @@ func New(options *types.Options) (*Runner, error) {
 	if runner.templatesConfig != nil {
 		runner.readNucleiIgnoreFile()
 	}
+	runner.catalogue = catalogue.New(runner.options.TemplatesDirectory)
 
 	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
 		gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err)
@@ -176,8 +178,8 @@ func (r *Runner) Close() {
 // binary and runs the actual enumeration
 func (r *Runner) RunEnumeration() {
 	// resolves input templates definitions and any optional exclusion
-	includedTemplates := r.getTemplatesFor(r.options.Templates)
+	includedTemplates := r.catalogue.GetTemplatesPath(r.options.Templates)
-	excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates)
+	excludedTemplates := r.catalogue.GetTemplatesPath(r.options.ExcludedTemplates)
 	// defaults to all templates
 	allTemplates := includedTemplates
 
@@ -222,10 +224,10 @@ func (r *Runner) RunEnumeration() {
 		if t.Workflow != nil {
 			continue
 		}
-		totalRequests += int64(t.Requests()) * r.inputCount
+		totalRequests += int64(t.TotalRequests) * r.inputCount
 	}
 
-	results := atomicboolean.New()
+	results := &atomic.Bool{}
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
 	// Starts polling or ignore
 	collaborator.DefaultCollaborator.Poll()
@@ -241,14 +243,9 @@ func (r *Runner) RunEnumeration() {
 			wgtemplates.Add()
 			go func(template *templates.Template) {
 				if template.Workflow != nil {
-					results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow)))
+					results.CAS(false, r.processWorkflowWithList(template))
-
+				} else {
-				}
+					results.CAS(false, r.processTemplateWithList(template))
-				for _, request := range template.RequestsDNS {
-					results.Or(r.processTemplateWithList(p, tt, request))
-				}
-				for _, request := range template.RequestsHTTP {
-					results.Or(r.processTemplateWithList(p, tt, request))
 				}
 			}(t)
 		}
@@ -256,7 +253,7 @@ func (r *Runner) RunEnumeration() {
 		p.Stop()
 	}
 
-	if !results.Get() {
+	if !results.Load() {
 		if r.output != nil {
 			r.output.Close()
 			os.Remove(r.options.Output)

v2/internal/runner/templates.go

@@ -3,7 +3,6 @@ package runner
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/karrick/godirwalk"
@@ -12,105 +11,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 )
 
-// getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths.
-func (r *Runner) getTemplatesFor(definitions []string) []string {
-	// keeps track of processed dirs and files
-	processed := make(map[string]bool)
-	allTemplates := []string{}
-
-	// parses user input, handle file/directory cases and produce a list of unique templates
-	for _, t := range definitions {
-		var absPath string
-		var err error
-
-		if strings.Contains(t, "*") {
-			dirs := strings.Split(t, "/")
-			priorDir := strings.Join(dirs[:len(dirs)-1], "/")
-			absPath, err = r.resolvePathIfRelative(priorDir)
-			absPath += "/" + dirs[len(dirs)-1]
-		} else {
-			// resolve and convert relative to absolute path
-			absPath, err = r.resolvePathIfRelative(t)
-		}
-		if err != nil {
-			gologger.Error().Msgf("Could not find template file '%s': %s\n", t, err)
-			continue
-		}
-
-		// Template input includes a wildcard
-		if strings.Contains(absPath, "*") {
-			var matches []string
-			matches, err = filepath.Glob(absPath)
-			if err != nil {
-				gologger.Error().Msgf("Wildcard found, but unable to glob '%s': %s\n", absPath, err)
-				continue
-			}
-
-			// couldn't find templates in directory
-			if len(matches) == 0 {
-				gologger.Error().Msgf("Error, no templates were found with '%s'.\n", absPath)
-				continue
-			} else {
-				gologger.Verbose().Msgf("Identified %d templates\n", len(matches))
-			}
-
-			for _, match := range matches {
-				if !r.checkIfInNucleiIgnore(match) {
-					processed[match] = true
-					allTemplates = append(allTemplates, match)
-				}
-			}
-		} else {
-			// determine file/directory
-			isFile, err := isFilePath(absPath)
-			if err != nil {
-				gologger.Error().Msgf("Could not stat '%s': %s\n", absPath, err)
-				continue
-			}
-			// test for uniqueness
-			if !isNewPath(absPath, processed) {
-				continue
-			}
-			// mark this absolute path as processed
-			// - if it's a file, we'll never process it again
-			// - if it's a dir, we'll never walk it again
-			processed[absPath] = true
-
-			if isFile {
-				allTemplates = append(allTemplates, absPath)
-			} else {
-				matches := []string{}
-
-				// Recursively walk down the Templates directory and run all the template file checks
-				err := directoryWalker(absPath,
-					func(path string, d *godirwalk.Dirent) error {
-						if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-							if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) {
-								matches = append(matches, path)
-								processed[path] = true
-							}
-						}
-						return nil
-					},
-				)
-				// directory couldn't be walked
-				if err != nil {
-					gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", absPath, err)
-					continue
-				}
-
-				// couldn't find templates in directory
-				if len(matches) == 0 {
-					gologger.Error().Msgf("Error, no templates were found in '%s'.\n", absPath)
-					continue
-				}
-				allTemplates = append(allTemplates, matches...)
-			}
-		}
-	}
-	return allTemplates
-}
-
 // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
 // by severity, along with a flag indicating if workflows are present.
 func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []*templates.Template, workflowCount int) {
@@ -148,6 +48,7 @@ func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
 		Output:      r.output,
 		Options:     r.options,
 		Progress:    r.progress,
+		Catalogue:   r.catalogue,
 		RateLimiter: r.ratelimiter,
 		ProjectFile: r.projectFile,
 	}
@@ -211,19 +112,6 @@ func (r *Runner) listAvailableTemplates() {
 	}
 }
 
-func (r *Runner) resolvePathIfRelative(filePath string) (string, error) {
-	if isRelative(filePath) {
-		newPath, err := r.resolvePath(filePath)
-
-		if err != nil {
-			return "", err
-		}
-		return newPath, nil
-	}
-
-	return filePath, nil
-}
-
 func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
 	for _, s := range allowedSeverities {
 		if s != "" && strings.HasPrefix(templateSeverity, s) {

v2/pkg/catalogue/catalogue.go

@@ -0,0 +1,14 @@
+package catalogue
+
+// Catalogue is a template catalouge helper implementation
+type Catalogue struct {
+	ignoreFiles        []string
+	templatesDirectory string
+}
+
+// New creates a new catalogue structure using provided input items
+func New(directory string) *Catalogue {
+	catalogue := &Catalogue{templatesDirectory: directory}
+	catalogue.readNucleiIgnoreFile()
+	return catalogue
+}

v2/pkg/catalogue/find.go

@@ -0,0 +1,153 @@
+package catalogue
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+)
+
+// GetTemplatesPath returns a list of absolute paths for the provided template list.
+func (c *Catalogue) GetTemplatesPath(definitions []string) []string {
+	// keeps track of processed dirs and files
+	processed := make(map[string]bool)
+	allTemplates := []string{}
+
+	for _, t := range definitions {
+		paths, err := c.GetTemplatePath(t)
+		if err != nil {
+			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+		}
+		for _, path := range paths {
+			if _, ok := processed[path]; !ok {
+				processed[path] = true
+				allTemplates = append(allTemplates, path)
+			}
+		}
+	}
+	gologger.Verbose().Msgf("Identified %d templates", len(allTemplates))
+	return allTemplates
+}
+
+// GetTemplatePath parses the specified input template path and returns a compiled
+// list of finished absolute paths to the templates evaluating any glob patterns
+// or folders provided as in.
+func (c *Catalogue) GetTemplatePath(target string) ([]string, error) {
+	processed := make(map[string]struct{})
+
+	absPath, err := c.convertPathToAbsolute(target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not find template file")
+	}
+
+	// Template input includes a wildcard
+	if strings.Contains(absPath, "*") {
+		matches, err := c.findGlobPathMatches(absPath, processed)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not find glob matches")
+		}
+		if len(matches) == 0 {
+			return nil, errors.Errorf("no templates found for path")
+		}
+		return matches, nil
+	}
+
+	// Template input is either a file or a directory
+	match, file, err := c.findFileMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find file")
+	}
+	if file {
+		if match != "" {
+			return []string{match}, nil
+		}
+		return nil, nil
+	}
+
+	// Recursively walk down the Templates directory and run all
+	// the template file checks
+	matches, err := c.findDirectoryMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find directory matches")
+	}
+	if len(matches) == 0 {
+		return nil, errors.Errorf("no templates found in path")
+	}
+	return matches, nil
+}
+
+// convertPathToAbsolute resolves the paths provided to absolute paths
+// before doing any operations on them regardless of them being blob, folders, files, etc.
+func (c *Catalogue) convertPathToAbsolute(t string) (string, error) {
+	if strings.Contains(t, "*") {
+		file := path.Base(t)
+		absPath, err := c.ResolvePath(path.Dir(t), "")
+		if err != nil {
+			return "", err
+		}
+		return path.Join(absPath, file), nil
+	}
+	return c.ResolvePath(t, "")
+}
+
+// findGlobPathMatches returns the matched files from a glob path
+func (c *Catalogue) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	matches, err := filepath.Glob(absPath)
+	if err != nil {
+		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+	}
+	results := make([]string, 0, len(matches))
+	for _, match := range matches {
+		if _, ok := processed[match]; !ok {
+			processed[match] = struct{}{}
+			results = append(results, match)
+		}
+	}
+	return results, nil
+}
+
+// findFileMatches finds if a path is an absolute file. If the path
+// is a file, it returns true otherwise false with no errors.
+func (c *Catalogue) findFileMatches(absPath string, processed map[string]struct{}) (string, bool, error) {
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return "", false, err
+	}
+	if !info.Mode().IsRegular() {
+		return "", false, nil
+	}
+	if _, ok := processed[absPath]; !ok {
+		processed[absPath] = struct{}{}
+		return absPath, true, nil
+	}
+	return "", true, nil
+}
+
+// findDirectoryMatches finds matches for templates from a directory
+func (c *Catalogue) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	var results []string
+	err := godirwalk.Walk(absPath, &godirwalk.Options{
+		Unsorted: true,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Callback: func(path string, d *godirwalk.Dirent) error {
+			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+				if c.checkIfInNucleiIgnore(path) {
+					return nil
+				}
+
+				if _, ok := processed[path]; !ok {
+					results = append(results, path)
+					processed[path] = struct{}{}
+				}
+			}
+			return nil
+		},
+	})
+	return results, err
+}

v2/pkg/catalogue/ignore.go

@@ -0,0 +1,70 @@
+package catalogue
+
+import (
+	"bufio"
+	"os"
+	"path"
+	"strings"
+)
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
+func (c *Catalogue) readNucleiIgnoreFile() {
+	file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile))
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		if strings.HasPrefix(text, "#") {
+			continue
+		}
+		c.ignoreFiles = append(c.ignoreFiles, text)
+	}
+}
+
+// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
+func (c *Catalogue) checkIfInNucleiIgnore(item string) bool {
+	if c.templatesDirectory == "" {
+		return false
+	}
+
+	for _, paths := range c.ignoreFiles {
+		// If we have a path to ignore, check if it's in the item.
+		if paths[len(paths)-1] == '/' {
+			if strings.Contains(item, paths) {
+				return true
+			}
+
+			continue
+		}
+		// Check for file based extension in ignores
+		if strings.HasSuffix(item, paths) {
+			return true
+		}
+	}
+	return false
+}
+
+// ignoreFilesWithExcludes ignores results with exclude paths
+func (c *Catalogue) ignoreFilesWithExcludes(results, excluded []string) []string {
+	excludeMap := make(map[string]struct{}, len(excluded))
+	for _, excl := range excluded {
+		excludeMap[excl] = struct{}{}
+	}
+
+	templates := make([]string, 0, len(results))
+	for _, incl := range results {
+		if _, found := excludeMap[incl]; !found {
+			templates = append(templates, incl)
+		}
+	}
+	return templates
+}

v2/pkg/catalogue/path.go

@@ -0,0 +1,45 @@
+package catalogue
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// ResolvePath resolves the path to an absolute one in various ways.
+//
+// It checks if the filename is an absolute path, looks in the current directory
+// or checking the nuclei templates directory. If a second path is given,
+// it also tries to find paths relative to that second path.
+func (c *Catalogue) ResolvePath(templateName, second string) (string, error) {
+	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+		return templateName, nil
+	}
+
+	if second != "" {
+		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
+			return secondBasePath, nil
+		}
+	}
+
+	curDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	templatePath := path.Join(curDirectory, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		return templatePath, nil
+	}
+
+	if c.templatesDirectory != "" {
+		templatePath := path.Join(c.templatesDirectory, templateName)
+		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+			return templatePath, nil
+		}
+	}
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/pkg/protocols/http/build_request.go

@@ -130,13 +130,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	return r.makeHTTPRequestFromModel(ctx, data, values)
 }
 
-// Remaining returns the remaining number of requests for the generator
+// Total returns the total number of requests for the generator
-func (r *requestGenerator) Remaining() int {
+func (r *requestGenerator) Total() int {
 	if r.payloadIterator != nil {
-		payloadRemaining := r.payloadIterator.Remaining()
+		return len(r.request.Raw) * r.payloadIterator.Remaining()
-		return (len(r.request.Raw) - r.currentIndex + 1) * payloadRemaining
 	}
-	return len(r.request.Path) - r.currentIndex + 1
+	return len(r.request.Path)
 }
 
 // baseURLWithTemplatePrefs returns the url for BaseURL keeping

v2/pkg/protocols/http/executer.go

@@ -33,7 +33,7 @@ func (e *Executer) Compile() error {
 func (e *Executer) Requests() int {
 	var count int
 	for _, request := range e.requests {
-		count += int64(request.Requests())
+		count += int(request.Requests())
 	}
 	return count
 }

v2/pkg/protocols/http/request.go

@@ -75,7 +75,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in
 			break
 		}
 		if err != nil {
-			e.options.Progress.DecrementRequests(int64(generator.Remaining()))
+			e.options.Progress.DecrementRequests(int64(generator.Total()))
 			return nil, err
 		}
 		swg.Add()
@@ -136,7 +136,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter
 			break
 		}
 		if err != nil {
-			e.options.Progress.DecrementRequests(int64(generator.Remaining()))
+			e.options.Progress.DecrementRequests(int64(generator.Total()))
 			return nil, err
 		}
 		request.pipelinedClient = pipeclient
@@ -187,7 +187,7 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int
 			break
 		}
 		if err != nil {
-			e.options.Progress.DecrementRequests(int64(generator.Remaining()))
+			e.options.Progress.DecrementRequests(int64(generator.Total()))
 			return nil, err
 		}
 
@@ -201,7 +201,7 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int
 		e.options.Progress.IncrementRequests()
 
 		if request.original.options.Options.StopAtFirstMatch && len(output) > 0 {
-			e.options.Progress.DecrementRequests(int64(generator.Remaining()))
+			e.options.Progress.DecrementRequests(int64(generator.Total()))
 			break
 		}
 	}

v2/pkg/protocols/protocols.go

@@ -2,6 +2,7 @@ package protocols
 
 import (
 	"github.com/projectdiscovery/nuclei/v2/internal/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalogue"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -38,6 +39,8 @@ type ExecuterOptions struct {
 	Progress *progress.Progress
 	// RateLimiter is a rate-limiter for limiting sent number of requests.
 	RateLimiter ratelimit.Limiter
+	// Catalogue is a template catalogue implementation for nuclei
+	Catalogue *catalogue.Catalogue
 	// ProjectFile is the project file for nuclei
 	ProjectFile *projectfile.ProjectFile
 }

v2/pkg/templates/compile.go

@@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 	"gopkg.in/yaml.v2"
 )
 
@@ -27,7 +28,6 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
 	defer f.Close()
 
 	// Setting up variables regarding template metadata
-	template.path = file
 	options.TemplateID = template.ID
 	options.TemplateInfo = template.Info
 	options.TemplatePath = file
@@ -37,27 +37,90 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
 		return nil, fmt.Errorf("both http and dns requests for %s", template.ID)
 	}
 	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflows) == 0 {
+	if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflow.Workflows) == 0 {
 		return nil, fmt.Errorf("no requests defined for %s", template.ID)
 	}
 
+	// Compile the workflow request
+	if template.Workflow != nil {
+		if err := template.compileWorkflow(options); err != nil {
+			return nil, errors.Wrap(err, "could not compile workflow")
+		}
+	}
+
 	// Compile the requests found
 	for _, request := range template.RequestsDNS {
-		template.totalRequests += request.Requests()
+		template.TotalRequests += request.Requests()
 	}
 	for _, request := range template.RequestsHTTP {
-		template.totalRequests += request.Requests()
+		template.TotalRequests += request.Requests()
 	}
 	if len(template.RequestsDNS) > 0 {
-		template.executer = dns.NewExecuter(template.RequestsDNS, options)
+		template.Executer = dns.NewExecuter(template.RequestsDNS, options)
-		err = template.executer.Compile()
+		err = template.Executer.Compile()
 	}
 	if len(template.RequestsHTTP) > 0 {
-		template.executer = http.NewExecuter(template.RequestsHTTP, options)
+		template.Executer = http.NewExecuter(template.RequestsHTTP, options)
-		err = template.executer.Compile()
+		err = template.Executer.Compile()
 	}
 	if err != nil {
 		return nil, errors.Wrap(err, "could not compile request")
 	}
 	return template, nil
 }
+
+// compileWorkflow compiles the workflow for execution
+func (t *Template) compileWorkflow(options *protocols.ExecuterOptions) error {
+	for _, workflow := range t.Workflows {
+		if err := t.parseWorkflow(workflow, options); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// parseWorkflow parses and compiles all templates in a workflow recursively
+func (t *Template) parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error {
+	if err := t.parseWorkflowTemplate(workflow, options); err != nil {
+		return err
+	}
+	for _, subtemplates := range workflow.Subtemplates {
+		if err := t.parseWorkflow(subtemplates, options); err != nil {
+			return err
+		}
+	}
+	for _, matcher := range workflow.Matchers {
+		for _, subtemplates := range matcher.Subtemplates {
+			if err := t.parseWorkflow(subtemplates, options); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// parseWorkflowTemplate parses a workflow template creating an executer
+func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error {
+	opts := protocols.ExecuterOptions{
+		Output:      options.Output,
+		Options:     options.Options,
+		Progress:    options.Progress,
+		Catalogue:   options.Catalogue,
+		RateLimiter: options.RateLimiter,
+		ProjectFile: options.ProjectFile,
+	}
+	paths, err := options.Catalogue.GetTemplatePath(workflow.Template)
+	if err != nil {
+		return errors.Wrap(err, "could not get workflow template")
+	}
+	if len(paths) != 1 {
+		return errors.Wrap(err, "invalid number of templates matched")
+	}
+
+	template, err := Parse(paths[0], &opts)
+	if err != nil {
+		return errors.Wrap(err, "could not parse workflow template")
+	}
+	workflow.Executer = template.Executer
+	return nil
+}

v2/pkg/templates/templates.go

@@ -17,21 +17,10 @@ type Template struct {
 	RequestsHTTP []*http.Request `yaml:"requests,omitempty"`
 	// RequestsDNS contains the dns request to make in the template
 	RequestsDNS []*dns.Request `yaml:"dns,omitempty"`
-
 	// Workflows is a yaml based workflow declaration code.
 	*workflows.Workflow
-
+	// TotalRequests is the total number of requests for the template.
-	path          string
+	TotalRequests int
-	totalRequests int
+	// Executer is the actual template executor for running template requests
-	executer      protocols.Executer
+	Executer protocols.Executer
-}
-
-// GetPath returns the path of the template.
-func (t *Template) GetPath() string {
-	return t.path
-}
-
-// Requests returns the number of requests for the template
-func (t *Template) Requests() int {
-	return t.totalRequests
 }

v2/pkg/workflows/execute.go

@@ -3,7 +3,7 @@ package workflows
 import "go.uber.org/atomic"
 
 // RunWorkflow runs a workflow on an input and returns true or false
-func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) {
+func (w *Workflow) RunWorkflow(input string) (bool, error) {
 	results := &atomic.Bool{}
 
 	for _, template := range w.Workflows {
@@ -20,7 +20,9 @@ func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) {
 func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool) error {
 	var firstMatched bool
 	if len(template.Matchers) == 0 {
-		matched, err := template.executer.Execute(input)
+		w.options.Progress.AddToTotal(int64(template.Executer.Requests()))
+
+		matched, err := template.Executer.Execute(input)
 		if err != nil {
 			return err
 		}
@@ -29,7 +31,9 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 	}
 
 	if len(template.Matchers) > 0 {
-		output, err := template.executer.ExecuteWithResults(input)
+		w.options.Progress.AddToTotal(int64(template.Executer.Requests()))
+
+		output, err := template.Executer.ExecuteWithResults(input)
 		if err != nil {
 			return err
 		}

v2/pkg/workflows/workflows.go

@@ -6,6 +6,8 @@ import "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 type Workflow struct {
 	// Workflows is a yaml based workflow declaration code.
 	Workflows []*WorkflowTemplate `yaml:"workflows"`
+
+	options *protocols.ExecuterOptions
 }
 
 // WorkflowTemplate is a template to be ran as part of a workflow
@@ -16,8 +18,8 @@ type WorkflowTemplate struct {
 	Matchers []*Matcher `yaml:"matchers"`
 	// Subtemplates are ran if the template matches.
 	Subtemplates []*WorkflowTemplate `yaml:"subtemplates"`
-
+	// Executer performs the actual execution for the workflow template
-	executer protocols.Executer
+	Executer protocols.Executer
 }
 
 // Matcher performs conditional matching on the workflow template results.