From 1fba3b063774c3800b799ffe511226094e9b0cee Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 15 Oct 2020 23:39:00 +0200 Subject: [PATCH] wip --- v2/go.mod | 18 +++++-- v2/go.sum | 61 ++++++++++++++++++++---- v2/internal/runner/options.go | 2 +- v2/internal/runner/processor.go | 2 + v2/internal/runner/runner.go | 15 ++++++ v2/pkg/executer/executer_dns.go | 4 ++ v2/pkg/executer/executer_http.go | 65 ++++++++++++++++++++----- v2/pkg/executer/http_utils.go | 81 ++++++++++++++++++++++++++++++++ 8 files changed, 221 insertions(+), 27 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e83ec4160..083b88d7a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -2,25 +2,35 @@ module github.com/projectdiscovery/nuclei/v2 go 1.14 +replace github.com/projectdiscovery/hmap v0.0.0 => /home/marco/go/src/github.com/projectdiscovery/hmap + require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible + github.com/coocood/freecache v1.1.1 // indirect github.com/d5/tengo/v2 v2.6.2 + github.com/golang/snappy v0.0.2 // indirect github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.31 + github.com/miekg/dns v1.1.33 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/gologger v1.0.1 + github.com/projectdiscovery/hmap v0.0.0 github.com/projectdiscovery/httpx v1.0.2 - github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1 + github.com/projectdiscovery/rawhttp v0.0.3 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 + github.com/stretchr/testify v1.5.1 github.com/vbauerster/mpb/v5 v5.3.0 + go.uber.org/atomic v1.7.0 // indirect go.uber.org/ratelimit v0.1.0 - golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 - golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e + golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee // indirect + golang.org/x/net v0.0.0-20201010224723-4f7140c49acb + golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 // indirect gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/go.sum b/v2/go.sum index 54bc5e3cc..4fa384923 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,31 +1,41 @@ -github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= +github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= +github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= @@ -39,22 +49,34 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.33 h1:8KUVEKrUw2dmu1Ys0aWnkEJgoRaLAzNysfCh2KSMWiI= +github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/httpx v1.0.2 h1:g7EeRAPckZgWcHkcAH2Qzv9MkRACVRLF+T2LJcM7SCk= github.com/projectdiscovery/httpx v1.0.2/go.mod h1:OwvMc5ogx69xukKXY6kIrDP6dgOYr4VtEWyr6o573Xs= -github.com/projectdiscovery/rawhttp v0.0.1 h1:g9lzZPNBZsIAscTS2VLgv8RGQJcFXsOOTlzU49mDeBk= -github.com/projectdiscovery/rawhttp v0.0.1/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4= github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1 h1:I3aE8ta92M2XbrYKNYOTlXhodxyH+zQOt1jIatorhQA= github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto= +github.com/projectdiscovery/rawhttp v0.0.3/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= @@ -62,21 +84,28 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/vbauerster/mpb v1.1.3 h1:IRgic8VFaURXkW0VxDLkNOiNaAgtw0okB2YIaVvJDI4= -github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/vbauerster/mpb/v5 v5.3.0 h1:vgrEJjUzHaSZKDRRxul5Oh4C72Yy/5VEMb0em+9M0mQ= github.com/vbauerster/mpb/v5 v5.3.0/go.mod h1:4yTkvAb8Cm4eylAp6t0JRq6pXDkFJ4krUlDqWYkakAs= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -85,8 +114,13 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -94,15 +128,22 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88= +golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 87c202cbc..811134bc1 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -83,7 +83,7 @@ func ParseOptions() *Options { flag.IntVar(&options.RateLimit, "rate-limit", -1, "Per Target Rate-Limit") flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)") flag.IntVar(&options.BulkSize, "bulk-size", 150, "Number of hosts analyzed in parallel per template") - flag.BoolVar(&options.ProjectFile, "project-file", true, "Use a project file to avoid sending same request multiple times") + flag.BoolVar(&options.ProjectFile, "project-file", false, "Use a project file to avoid sending same request multiple times") flag.Parse() diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 02b804063..3dbd52dd4 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -48,6 +48,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat ColoredOutput: !r.options.NoColor, Colorizer: r.colorizer, Decolorizer: r.decolorizer, + HM: r.hm, }) case *requests.BulkHTTPRequest: httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{ @@ -67,6 +68,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat Colorizer: &r.colorizer, Decolorizer: r.decolorizer, StopAtFirstMatch: r.options.StopAtFirstMatch, + HM: r.hm, }) } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f45beddd1..c9418128d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -12,6 +12,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" @@ -34,6 +35,8 @@ type Runner struct { // options contains configuration options for runner options *Options + hm *hybrid.HybridMap + // progress tracking progress progress.IProgress @@ -164,6 +167,15 @@ func New(options *Options) (*Runner, error) { // Creates the progress tracking object runner.progress = progress.NewProgress(runner.colorizer.Colorizer, options.EnableProgressBar) + // create project file if requested + if options.ProjectFile { + var err error + runner.hm, err = hybrid.New(hybrid.DefaultDiskOptions) + if err != nil { + return runner, err + } + } + return runner, nil } @@ -173,6 +185,9 @@ func (r *Runner) Close() { r.output.Close() } os.Remove(r.tempFile) + if r.hm != nil { + r.hm.Close() + } } // RunEnumeration sets up the input layer for giving input nuclei. diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 0044bfba2..05ced7de1 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" @@ -19,6 +20,7 @@ import ( // DNSExecuter is a client for performing a DNS request // for a template. type DNSExecuter struct { + hm *hybrid.HybridMap coloredOutput bool debug bool jsonOutput bool @@ -50,6 +52,7 @@ type DNSOptions struct { Template *templates.Template DNSRequest *requests.DNSRequest Writer *bufwriter.Writer + HM *hybrid.HybridMap Colorizer colorizer.NucleiColorizer Decolorizer *regexp.Regexp @@ -71,6 +74,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter { coloredOutput: options.ColoredOutput, colorizer: options.Colorizer, decolorizer: options.Decolorizer, + hm: options.HM, } return executer diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index abff342da..8370854d0 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "net" "net/http" "net/http/cookiejar" @@ -19,6 +20,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/httpx/common/cache" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" @@ -41,6 +43,7 @@ const ( // HTTPExecuter is client for performing HTTP requests // for a template. type HTTPExecuter struct { + hm *hybrid.HybridMap customHeaders requests.CustomHeaders colorizer colorizer.NucleiColorizer httpClient *retryablehttp.Client @@ -77,6 +80,7 @@ type HTTPOptions struct { CookieReuse bool ColoredOutput bool StopAtFirstMatch bool + HM *hybrid.HybridMap } // NewHTTPExecuter creates a new HTTP executer from a template @@ -130,6 +134,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { colorizer: *options.Colorizer, decolorizer: options.Decolorizer, stopAtFirstMatch: options.StopAtFirstMatch, + hm: options.HM, } return executer, nil @@ -316,21 +321,26 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, e.setCustomHeaders(request) var ( - resp *http.Response - err error + resp *http.Response + err error + dumpedRequest []byte + fromcache bool ) - if e.debug { - dumpedRequest, dumpErr := requests.Dump(request, reqURL) - if dumpErr != nil { - return dumpErr + if e.debug || e.hm != nil { + dumpedRequest, err = requests.Dump(request, reqURL) + if err != nil { + return err } + } + if e.debug { gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) } timeStart := time.Now() + if request.Pipeline { resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data))) if err != nil { @@ -354,13 +364,30 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, return err } } else { - // retryablehttp - resp, err = e.httpClient.Do(request.Request) - if err != nil { - if resp != nil { - resp.Body.Close() + // if nuclei-project is available check if the request was already sent previously + if e.hm != nil { + reqHash, err := hash(dumpedRequest) + log.Println(reqURL, reqHash) + // if the computation was successful check within the cache + if err == nil { + data, ok := e.hm.Get(reqHash) + // if found reuse the item + if ok { + unmarshal(data, resp) + fromcache = true + } + } + } + + // retryablehttp + if resp == nil { + resp, err = e.httpClient.Do(request.Request) + if err != nil { + if resp != nil { + resp.Body.Close() + } + return err } - return err } } @@ -398,6 +425,20 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, return errors.Wrap(err, "could not decompress http body") } + // if nuclei-project is enabled store the response if not previously done + if e.hm != nil && !fromcache { + reqHash, err := hash(dumpedRequest) + // if the computation was successful store within the cache + if err == nil { + intResp := toInternalResponse(resp, data) + data, err := marshal(intResp) + // once marshaled without errors store in the cache + if err == nil { + e.hm.Set(reqHash, data) + } + } + } + // Convert response body from []byte to string with zero copy body := unsafeToString(data) diff --git a/v2/pkg/executer/http_utils.go b/v2/pkg/executer/http_utils.go index 8937367ae..8247a4b59 100644 --- a/v2/pkg/executer/http_utils.go +++ b/v2/pkg/executer/http_utils.go @@ -1,6 +1,12 @@ package executer import ( + "bytes" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "io" + "io/ioutil" "net/http" "strings" "unsafe" @@ -49,3 +55,78 @@ func headersToString(headers http.Header) string { return builder.String() } + +func hash(v interface{}) (string, error) { + data, err := marshal(v) + if err != nil { + return "", err + } + + sh := sha256.New() + + io.WriteString(sh, string(data)) + return hex.EncodeToString(sh.Sum(nil)), nil +} + +func marshal(data interface{}) ([]byte, error) { + var b bytes.Buffer + enc := gob.NewEncoder(&b) + err := enc.Encode(data) + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +func unmarshal(data []byte, obj interface{}) error { + var b bytes.Buffer + dec := gob.NewDecoder(&b) + err := dec.Decode(obj) + if err != nil { + return err + } + + return nil +} + +type InternalResponse struct { + HTTPMajor int + HTTPMinor int + StatusCode int + StatusReason string + Headers map[string][]string + Body []byte +} + +func newInternalResponse() *InternalResponse { + return &InternalResponse{ + Headers: make(map[string][]string), + } +} + +func toInternalResponse(resp *http.Response, body []byte) *InternalResponse { + intResp := newInternalResponse() + + intResp.HTTPMajor = resp.ProtoMajor + intResp.HTTPMinor = resp.ProtoMinor + intResp.StatusCode = resp.StatusCode + intResp.StatusReason = resp.Status + for k, v := range resp.Header { + intResp.Headers[k] = v + } + intResp.Body = body + return intResp +} + +func fromInternalResponse(intResp *InternalResponse) *http.Response { + return &http.Response{ + ProtoMinor: intResp.HTTPMinor, + ProtoMajor: intResp.HTTPMajor, + Status: intResp.StatusReason, + StatusCode: intResp.StatusCode, + Header: intResp.Headers, + ContentLength: int64(len(intResp.Body)), + Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)), + } +}