diff --git a/.gitignore b/.gitignore index 9386137b5..b9e561e3a 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,5 @@ integration_tests/fuzzplayground /nuclei-stats /nuclei-stats-* /scan-charts - +/**/debug-* diff --git a/go.mod b/go.mod index e775ff43d..537cc9ce6 100644 --- a/go.mod +++ b/go.mod @@ -16,16 +16,16 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.57 + github.com/miekg/dns v1.1.59 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.72 - github.com/projectdiscovery/hmap v0.0.42 - github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 - github.com/projectdiscovery/rawhttp v0.1.50 - github.com/projectdiscovery/retryabledns v1.0.59 - github.com/projectdiscovery/retryablehttp-go v1.0.60 + github.com/projectdiscovery/fastdialer v0.1.0 + github.com/projectdiscovery/hmap v0.0.43 + github.com/projectdiscovery/interactsh v1.1.9 + github.com/projectdiscovery/rawhttp v0.1.49 + github.com/projectdiscovery/retryabledns v1.0.60 + github.com/projectdiscovery/retryablehttp-go v1.0.62 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -35,7 +35,7 @@ require ( github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 github.com/valyala/fasttemplate v1.2.2 - github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 + github.com/weppos/publicsuffix-go v0.30.2 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.25.0 @@ -73,7 +73,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/labstack/echo/v4 v4.10.2 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa - github.com/lib/pq v1.10.1 + github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.22 github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 @@ -94,12 +94,12 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.52 - github.com/projectdiscovery/utils v0.0.94 + github.com/projectdiscovery/utils v0.1.0 github.com/projectdiscovery/wappalyzergo v0.1.0 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 - github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc + github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 golang.org/x/term v0.20.0 gopkg.in/yaml.v3 v3.0.1 @@ -134,7 +134,7 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.3.8 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect @@ -175,7 +175,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kataras/jwt v0.1.10 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/logrusorgru/aurora/v4 v4.0.0 // indirect @@ -203,16 +203,15 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect - github.com/refraction-networking/utls v1.6.1 // indirect + github.com/refraction-networking/utls v1.6.6 // indirect github.com/sashabaranov/go-openai v1.15.3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/tidwall/btree v1.7.0 // indirect - github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/buntdb v1.3.1 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -233,7 +232,7 @@ require ( github.com/zeebo/blake3 v0.2.3 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/sync v0.6.0 // indirect + golang.org/x/sync v0.7.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect mellium.im/sasl v0.3.1 // indirect ) @@ -298,16 +297,16 @@ require ( github.com/ysmood/leakless v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect - go.etcd.io/bbolt v1.3.8 // indirect + github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db // indirect + go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a - golang.org/x/mod v0.14.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/mod v0.17.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.17.0 + golang.org/x/tools v0.21.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/go.sum b/go.sum index 7da521271..c176b4995 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= @@ -367,8 +367,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= @@ -394,8 +392,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= @@ -630,8 +626,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -662,8 +658,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk= github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa/go.mod h1:HbwNE4XGwjgtUELkvQaAOjWrpianHYZdQVNqSdYW3UM= -github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo= -github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -714,8 +710,8 @@ github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3r github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= -github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= @@ -773,8 +769,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= @@ -837,8 +831,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY= github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og= -github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E= -github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0= +github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= +github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -853,12 +847,12 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= -github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= +github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg= +github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc= github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc= github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss= -github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 h1:0ih5JJyTht2sv4J4lbfSPaUkKdFDci+Bwxg6vsyd2F4= -github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663/go.mod h1:gTrQCQfYlhoIM4X11+zwL7uRuraOUCoiE4Z6z1QD86g= +github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc= +github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= @@ -871,14 +865,14 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4= github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y= -github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM= -github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg= +github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM= +github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= -github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= -github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= +github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ= +github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU= +github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY= +github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -889,8 +883,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= -github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo= -github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= +github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= +github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0= github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -927,13 +921,11 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= -github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA= -github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo= +github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4llzEXOkL43Ig= +github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1015,17 +1007,17 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg= -github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50= +github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 h1:GXIyLuIJ5Qk46lI8WJ83qHBZKUI3zhmMmuoY9HICUIQ= +github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= -github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o= +github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -1068,8 +1060,8 @@ github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY= -github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 h1:h2JizvZl9aIj6za9S5AyrkU+OzIS4CetQthH/ejO+lg= -github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= +github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU= +github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= @@ -1132,15 +1124,15 @@ github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+T github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w= -github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s= -github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk= +github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I= +github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g= github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo= github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 h1:LaMyYFWQA7kh3ovPfAaFDTKlJu3JGng8khruOtsBVnE= github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706/go.mod h1:re2kMcs84XHb8Xl6RInt0emoKCuphfmfjHYuteviLHQ= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1188,8 +1180,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1202,8 +1194,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1226,8 +1218,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1283,7 +1275,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1310,8 +1303,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1383,9 +1376,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1398,8 +1390,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1415,7 +1407,7 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -1476,8 +1468,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/runner/options.go b/internal/runner/options.go index cad5202f8..3d36f5112 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" @@ -114,7 +115,12 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) - err := protocolinit.Init(options) + err := protocolstate.Init(options) + if err != nil { + gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) + } + + err = protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index a0c46f6d0..24010d753 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -359,6 +359,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions, // Close releases all the resources and cleans up func (r *Runner) Close() { + // dump hosterrors cache + if r.hostErrors != nil { + r.hostErrors.Close() + } if r.output != nil { r.output.Close() } @@ -464,7 +468,7 @@ func (r *Runner) RunEnumeration() error { Parser: r.parser, } - if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" { + if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { // Go StdLib style experimental/debug feature switch executorOpts.ExportReqURLPattern = true } diff --git a/lib/multi.go b/lib/multi.go index 5b2c7d677..7c713e8aa 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -123,7 +123,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t return err } } - defer tmpEngine.Close() // create ephemeral nuclei objects/instances/types using base nuclei engine unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts) if err != nil { diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 13e874651..cfe9c88a2 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -25,7 +25,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" @@ -119,7 +118,6 @@ func (e *NucleiEngine) init() error { e.parser = templates.NewParser() sharedInit.Do(func() { - _ = protocolstate.Init(e.opts) _ = protocolinit.Init(e.opts) }) diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 736671127..3918a0124 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.2.7` + Version = `v3.2.8-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" @@ -68,3 +68,13 @@ func trimDevIfExists(version string) string { } return version } + +// similar to go pattern of enabling debug related features +// we add custom/extra switches for debugging purposes +const ( + // DebugArgHostErrorStats is used to print host error stats + // when it is closed + DebugArgHostErrorStats = "host-error-stats" + // DebugExportReqURLPattern is used to export request URL pattern + DebugExportURLPattern = "req-url-pattern" +) diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index 619d7fd47..062393e54 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -46,9 +46,10 @@ type Config struct { LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` // internal / unexported fields - disableUpdates bool `json:"-"` // disable updates both version check and template updates - homeDir string `json:"-"` // User Home Directory - configDir string `json:"-"` // Nuclei Global Config Directory + disableUpdates bool `json:"-"` // disable updates both version check and template updates + homeDir string `json:"-"` // User Home Directory + configDir string `json:"-"` // Nuclei Global Config Directory + debugArgs []string `json:"-"` // debug args } // IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates. @@ -329,6 +330,46 @@ func (c *Config) copyIgnoreFile() { } } +// IsDebugArgEnabled checks if debug arg is enabled +// this could be a feature specific to debugging like PPROF or printing stats +// of max host error etc +func (c *Config) IsDebugArgEnabled(arg string) bool { + for _, v := range c.debugArgs { + if v == arg { + return true + } + } + return false +} + +// parseDebugArgs from string +func (c *Config) parseDebugArgs(data string) { + // use space as seperator instead of commas + tmp := strings.Fields(data) + for _, v := range tmp { + key := v + value := "" + // if it is key value pair then split it + if strings.Contains(v, "=") { + parts := strings.SplitN(v, "=", 2) + if len(parts) != 2 { + continue + } + key, value = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + } + if value == "false" || value == "0" { + // if false or disabled then skip + continue + } + switch key { + case DebugArgHostErrorStats: + c.debugArgs = append(c.debugArgs, DebugArgHostErrorStats) + case DebugExportURLPattern: + c.debugArgs = append(c.debugArgs, DebugExportURLPattern) + } + } +} + func init() { // first attempt to migrate all files from old config directory to new config directory goflags.AttemptConfigMigration() // regardless how many times this is called it will only migrate once based on condition @@ -377,6 +418,7 @@ func init() { // and even if it is changed we don't follow it since it is not expected behavior // If custom templates are in default locations only then they are loaded while running nuclei DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory) + DefaultConfig.parseDebugArgs(env.GetEnvOrDefault("NUCLEI_ARGS", "")) } // Add Default Config adds default when .templates-config.json file is not present diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 86302bb71..0e91c3883 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -107,7 +107,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ currentInfo.Unlock() // Skip if the host has had errors - if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) { + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(contextargs.NewWithMetaInput(ctx, scannedValue)) { return true } diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go index d0d3ede00..028df4d01 100644 --- a/pkg/core/workflow_execute.go +++ b/pkg/core/workflow_execute.go @@ -98,7 +98,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan } if err != nil { if w.Options.HostErrorsCache != nil { - w.Options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) + w.Options.HostErrorsCache.MarkFailed(ctx.Input, err) } if len(template.Executers) == 1 { mainErr = err diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 05eacda6c..367bd013b 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -10,9 +10,15 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" contextutil "github.com/projectdiscovery/utils/context" + "github.com/projectdiscovery/utils/errkit" stringsutil "github.com/projectdiscovery/utils/strings" ) +var ( + // ErrJSExecDeadline is the error returned when alloted time for script execution exceeds + ErrJSExecDeadline = errkit.New("js engine execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() +) + // Compiler provides a runtime to execute goja runtime // based javascript scripts efficiently while also // providing them access to custom modules defined in libs/. @@ -107,7 +113,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, } // execute with context and timeout - ctx, cancel := context.WithTimeout(opts.Context, time.Duration(opts.Timeout)*time.Second) + ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline) defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { @@ -119,6 +125,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, return ExecuteProgram(program, args, opts) }) if err != nil { + if val, ok := err.(*goja.Exception); ok { + err = val.Unwrap() + } return nil, err } var res ExecuteResult diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go index 8a6d4a04b..8a171dc3d 100644 --- a/pkg/js/compiler/init.go +++ b/pkg/js/compiler/init.go @@ -1,6 +1,8 @@ package compiler -import "github.com/projectdiscovery/nuclei/v3/pkg/types" +import ( + "github.com/projectdiscovery/nuclei/v3/pkg/types" +) // jsprotocolInit @@ -9,6 +11,7 @@ var ( JsProtocolTimeout = 10 PoolingJsVmConcurrency = 100 NonPoolingVMConcurrency = 20 + JsTimeoutMultiplier = 1.5 ) // Init initializes the javascript protocol @@ -21,7 +24,9 @@ func Init(opts *types.Options) error { // 100 is reasonable default opts.JsConcurrency = 100 } - JsProtocolTimeout = opts.Timeout + // we have dialer timeout set to 10s so js needs to be at least + // 15s to return the actual error if not it will be a dialer timeout + JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier) PoolingJsVmConcurrency = opts.JsConcurrency PoolingJsVmConcurrency -= NonPoolingVMConcurrency return nil diff --git a/pkg/js/libs/postgres/memo.postgres.go b/pkg/js/libs/postgres/memo.postgres.go index 786d87133..9c61356b0 100755 --- a/pkg/js/libs/postgres/memo.postgres.go +++ b/pkg/js/libs/postgres/memo.postgres.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - _ "github.com/lib/pq" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap" utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils" diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go index 5aea41501..f20ea85a7 100644 --- a/pkg/js/libs/postgres/postgres.go +++ b/pkg/js/libs/postgres/postgres.go @@ -9,10 +9,11 @@ import ( "time" "github.com/go-pg/pg" - _ "github.com/lib/pq" "github.com/praetorian-inc/fingerprintx/pkg/plugins" postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql" utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils" + "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" ) @@ -100,7 +101,7 @@ func executeQuery(host string, port int, username string, password string, dbNam target := net.JoinHostPort(host, fmt.Sprintf("%d", port)) connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", username, password, target, dbName) - db, err := sql.Open("postgres", connStr) + db, err := sql.Open(pgwrap.PGWrapDriver, connStr) if err != nil { return nil, err } @@ -152,6 +153,9 @@ func connect(host string, port int, username string, password string, dbName str User: username, Password: password, Database: dbName, + Dialer: func(network, addr string) (net.Conn, error) { + return protocolstate.Dialer.Dial(context.Background(), network, addr) + }, IdleCheckFrequency: -1, }).WithContext(ctx).WithTimeout(10 * time.Second) defer db.Close() diff --git a/pkg/js/utils/pgwrap/pgwrap.go b/pkg/js/utils/pgwrap/pgwrap.go new file mode 100644 index 000000000..d1b82f7ab --- /dev/null +++ b/pkg/js/utils/pgwrap/pgwrap.go @@ -0,0 +1,53 @@ +package pgwrap + +import ( + "context" + "database/sql" + "database/sql/driver" + "net" + "time" + + "github.com/lib/pq" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" +) + +const ( + PGWrapDriver = "pgwrap" +) + +type pgDial struct { + fd *fastdialer.Dialer +} + +func (p *pgDial) Dial(network, address string) (net.Conn, error) { + return p.fd.Dial(context.TODO(), network, address) +} + +func (p *pgDial) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + ctx, cancel := context.WithTimeoutCause(context.Background(), timeout, fastdialer.ErrDialTimeout) + defer cancel() + return p.fd.Dial(ctx, network, address) +} + +func (p *pgDial) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return p.fd.Dial(ctx, network, address) +} + +// Unfortunately lib/pq does not provide easy to customize or +// replace dialer so we need to hijack it by wrapping it in our own +// driver and register it as postgres driver + +// PgDriver is the Postgres database driver. +type PgDriver struct{} + +// Open opens a new connection to the database. name is a connection string. +// Most users should only use it through database/sql package from the standard +// library. +func (d PgDriver) Open(name string) (driver.Conn, error) { + return pq.DialOpen(&pgDial{fd: protocolstate.Dialer}, name) +} + +func init() { + sql.Register(PGWrapDriver, &PgDriver{}) +} diff --git a/pkg/output/output.go b/pkg/output/output.go index 044d164f1..1ee710ff3 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "io" + "log/slog" "os" "path/filepath" "regexp" @@ -27,9 +28,12 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/nuclei/v3/pkg/utils" + "github.com/projectdiscovery/utils/errkit" fileutil "github.com/projectdiscovery/utils/file" osutils "github.com/projectdiscovery/utils/os" + urlutil "github.com/projectdiscovery/utils/url" ) // Writer is an interface which writes output to somewhere for nuclei events. @@ -299,10 +303,13 @@ func (w *StandardWriter) Write(event *ResultEvent) error { // JSONLogRequest is a trace/error log request written to file type JSONLogRequest struct { - Template string `json:"template"` - Input string `json:"input"` - Error string `json:"error"` - Type string `json:"type"` + Template string `json:"template"` + Type string `json:"type"` + Input string `json:"input"` + Address string `json:"address"` + Error string `json:"error"` + Kind string `json:"kind,omitempty"` + Attrs interface{} `json:"attrs,omitempty"` } // Request writes a log the requests trace log @@ -315,12 +322,43 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques Input: input, Type: requestType, } - if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil { - request.Error = unwrappedErr.Error() - } else { - request.Error = "none" + parsed, _ := urlutil.ParseAbsoluteURL(input, false) + if parsed != nil { + request.Address = parsed.Hostname() + port := parsed.Port() + if port == "" { + switch parsed.Scheme { + case urlutil.HTTP: + port = "80" + case urlutil.HTTPS: + port = "443" + } + } + request.Address += ":" + port + } + errX := errkit.FromError(requestErr) + if errX == nil { + request.Error = "none" + } else { + request.Kind = errkit.ErrKindUnknown.String() + var cause error + if len(errX.Errors()) > 1 { + cause = errX.Errors()[0] + } + if cause == nil { + cause = errX + } + cause = tryParseCause(cause) + request.Error = cause.Error() + request.Kind = errkit.GetErrorKind(requestErr, nucleierr.ErrTemplateLogic).String() + if len(errX.Attrs()) > 0 { + request.Attrs = slog.GroupValue(errX.Attrs()...) + } + } + // check if address slog attr is avaiable in error if set use it + if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil { + request.Address = val.String() } - data, err := jsoniter.Marshal(request) if err != nil { return @@ -452,3 +490,24 @@ func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string, } } + +// tryParseCause tries to parse the cause of given error +// this is legacy support due to use of errorutil in existing libraries +// but this should not be required once all libraries are updated +func tryParseCause(err error) error { + if err == nil { + return nil + } + msg := err.Error() + if strings.HasPrefix(msg, "ReadStatusLine:") { + // last index is actual error (from rawhttp) + parts := strings.Split(msg, ":") + return errkit.New(strings.TrimSpace(parts[len(parts)-1])) + } + if strings.Contains(msg, "read ") { + // same here + parts := strings.Split(msg, ":") + return errkit.New(strings.TrimSpace(parts[len(parts)-1])) + } + return err +} diff --git a/pkg/output/output_test.go b/pkg/output/output_test.go index 7c96c0529..a21f7e889 100644 --- a/pkg/output/output_test.go +++ b/pkg/output/output_test.go @@ -30,7 +30,7 @@ func TestStandardWriterRequest(t *testing.T) { require.NoError(t, err) w.Request("path", "input", "http", nil) - require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String()) + require.Equal(t, `{"template":"path","type":"http","input":"input","address":"input:","error":"none"}`, traceWriter.String()) require.Empty(t, errorWriter.String()) }) @@ -47,7 +47,7 @@ func TestStandardWriterRequest(t *testing.T) { fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), ) - require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String()) + require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","type":"http","input":"https://example.com/tcpconfig.html","address":"example.com:443","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","kind":"unknown-error"}`, errorWriter.String()) }) } diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 70dda7254..f103f50d0 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -32,6 +32,7 @@ import ( templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" contextutil "github.com/projectdiscovery/utils/context" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" ) @@ -41,7 +42,10 @@ const ( ) var ( + // pythonEnvRegexCompiled is the compiled regex for python environment variables pythonEnvRegexCompiled = regexp.MustCompile(pythonEnvRegex) + // ErrCodeExecutionDeadline is the error returned when alloted time for script execution exceeds + ErrCodeExecutionDeadline = errkit.New("code execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() ) // Request is a request for the SSL protocol @@ -214,7 +218,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) { diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go index 8e8f0361c..ed004ae37 100644 --- a/pkg/protocols/common/contextargs/contextargs.go +++ b/pkg/protocols/common/contextargs/contextargs.go @@ -37,6 +37,13 @@ func New(ctx context.Context) *Context { return NewWithInput(ctx, "") } +// NewWithMetaInput creates a new contextargs instance with meta input +func NewWithMetaInput(ctx context.Context, input *MetaInput) *Context { + n := New(ctx) + n.MetaInput = input + return n +} + // Create a new contextargs instance with input string func NewWithInput(ctx context.Context, input string) *Context { jar, err := cookiejar.New(nil) @@ -174,3 +181,20 @@ func (ctx *Context) Clone() *Context { } return newCtx } + +// GetCopyIfHostOutdated returns a new contextargs if the host is outdated +func GetCopyIfHostOutdated(ctx *Context, url string) *Context { + if ctx.MetaInput.Input == "" { + newctx := ctx.Clone() + newctx.MetaInput.Input = url + return newctx + } + orig, _ := urlutil.Parse(ctx.MetaInput.Input) + newURL, _ := urlutil.Parse(url) + if orig != nil && newURL != nil && orig.Host != newURL.Host { + newCtx := ctx.Clone() + newCtx.MetaInput.Input = newURL.Host + return newCtx + } + return ctx +} diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go index afda2fda2..72eac6247 100644 --- a/pkg/protocols/common/contextargs/metainput.go +++ b/pkg/protocols/common/contextargs/metainput.go @@ -4,11 +4,13 @@ import ( "bytes" "crypto/md5" "fmt" + "net" "strings" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" urlutil "github.com/projectdiscovery/utils/url" + "github.com/segmentio/ksuid" ) // MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput) @@ -47,6 +49,56 @@ func (metaInput *MetaInput) URL() (*urlutil.URL, error) { return instance, nil } +// Port returns the port of the target +// if port is not present then empty string is returned +func (metaInput *MetaInput) Port() string { + target, err := urlutil.ParseAbsoluteURL(metaInput.Input, false) + if err != nil { + return "" + } + return target.Port() +} + +// Address return the remote address of target +// Note: it does not resolve the domain to ip +// it is meant to be used by hosterrorsCache if invalid metainput +// is provided then it uses a random ksuid as hostname to avoid skipping valid targets +func (metaInput *MetaInput) Address() string { + var hostname, port string + target, err := urlutil.ParseAbsoluteURL(metaInput.Target(), false) + if err != nil { + if metaInput.CustomIP == "" { + // since this is used in hosterrorscache we add a random id + // which will never be used to avoid skipping valid targets + hostname = fmt.Sprintf("invalid-%s", ksuid.New().String()) + } + } else { + hostname = target.Hostname() + port = target.Port() + if port == "" { + switch target.Scheme { + case urlutil.HTTP: + port = "80" + case urlutil.HTTPS: + port = "443" + default: + port = "80" + } + } + } + if metaInput.CustomIP != "" { + hostname = metaInput.CustomIP + } + if port == "" { + if strings.HasPrefix(hostname, "http://") { + port = "80" + } else if strings.HasPrefix(hostname, "https://") { + port = "443" + } + } + return net.JoinHostPort(hostname, port) +} + // ID returns a unique id/hash for metainput func (metaInput *MetaInput) ID() string { if metaInput.CustomIP != "" { diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 3abd1ec55..e40f13dde 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -10,17 +10,25 @@ import ( "github.com/bluele/gcache" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" + "github.com/projectdiscovery/utils/errkit" ) // CacheInterface defines the signature of the hosterrorscache so that // users of Nuclei as embedded lib may implement their own cache type CacheInterface interface { - SetVerbose(verbose bool) // log verbosely - Close() // close the cache - Check(value string) bool // return true if the host should be skipped - MarkFailed(value string, err error) // record a failure (and cause) for the host + SetVerbose(verbose bool) // log verbosely + Close() // close the cache + Check(ctx *contextargs.Context) bool // return true if the host should be skipped + MarkFailed(ctx *contextargs.Context, err error) // record a failure (and cause) for the host } +var ( + _ CacheInterface = (*Cache)(nil) +) + // Cache is a cache for host based errors. It allows skipping // certain hosts based on an error threshold. // @@ -34,8 +42,10 @@ type Cache struct { } type cacheItem struct { - errors atomic.Int32 sync.Once + errors atomic.Int32 + isPermanentErr bool + cause error // optional cause } const DefaultMaxHostsCount = 10000 @@ -55,6 +65,16 @@ func (c *Cache) SetVerbose(verbose bool) { // Close closes the host errors cache func (c *Cache) Close() { + if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) { + items := c.failedTargets.GetALL(false) + for k, v := range items { + val, ok := v.(*cacheItem) + if !ok { + continue + } + gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load()) + } + } c.failedTargets.Purge() } @@ -88,14 +108,19 @@ func (c *Cache) normalizeCacheValue(value string) string { // - URL: https?:// type // - Host:port type // - host type -func (c *Cache) Check(value string) bool { - finalValue := c.normalizeCacheValue(value) +func (c *Cache) Check(ctx *contextargs.Context) bool { + finalValue := c.GetKeyFromContext(ctx, nil) existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue) if err != nil { return false } existingCacheItemValue := existingCacheItem.(*cacheItem) + if existingCacheItemValue.isPermanentErr { + // skipping permanent errors is expected so verbose instead of info + gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause) + return true + } if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) { existingCacheItemValue.Do(func() { @@ -107,15 +132,22 @@ func (c *Cache) Check(value string) bool { } // MarkFailed marks a host as failed previously -func (c *Cache) MarkFailed(value string, err error) { +func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) { if !c.checkError(err) { return } - finalValue := c.normalizeCacheValue(value) + finalValue := c.GetKeyFromContext(ctx, err) existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue) if err != nil || existingCacheItem == nil { newItem := &cacheItem{errors: atomic.Int32{}} newItem.errors.Store(1) + if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) { + // skip this address altogether + // permanent errors are always permanent hence this is created once + // and never updated so no need to synchronize + newItem.isPermanentErr = true + newItem.cause = err + } _ = c.failedTargets.Set(finalValue, newItem) return } @@ -124,19 +156,77 @@ func (c *Cache) MarkFailed(value string, err error) { _ = c.failedTargets.Set(finalValue, existingCacheItemValue) } -var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host|timeout awaiting response headers)`) +// GetKeyFromContext returns the key for the cache from the context +func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string { + // Note: + // ideally any changes made to remote addr in template like {{Hostname}}:81 etc + // should be reflected in contextargs but it is not yet reflected in some cases + // and needs refactor of ScanContext + ContextArgs to achieve that + // i.e why we use real address from error if present + address := ctx.MetaInput.Address() + // get address override from error + if err != nil { + tmp := errkit.GetAttrValue(err, "address") + if tmp.Any() != nil { + address = tmp.String() + } + } + finalValue := c.normalizeCacheValue(address) + return finalValue +} + +var reCheckError = regexp.MustCompile(`(no address found for host|could not resolve host|connection refused|connection reset by peer|could not connect to any address found for host|timeout awaiting response headers)`) // checkError checks if an error represents a type that should be // added to the host skipping table. +// it first parses error and extracts the cause and checks for blacklisted +// or common errors that should be skipped func (c *Cache) checkError(err error) bool { if err == nil { return false } - errString := err.Error() - for _, msg := range c.TrackError { - if strings.Contains(errString, msg) { - return true + kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic) + switch kind { + case nucleierr.ErrTemplateLogic: + // these are errors that are not related to the target + // and are due to template logic + return false + case errkit.ErrKindNetworkTemporary: + // these should not be counted as host errors + return false + case errkit.ErrKindNetworkPermanent: + // these should be counted as host errors + return true + case errkit.ErrKindDeadline: + // these should not be counted as host errors + return false + default: + // parse error for furthur processing + errX := errkit.FromError(err) + tmp := errX.Cause() + cause := tmp.Error() + if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") { + // this is a FP and should not be counted as a host error + // because server closes connection when it reads corrupted bytes which we send via rawhttp + return false } + if strings.HasPrefix(cause, "ReadStatusLine:") { + // error is present in last part when using rawhttp + // this will be fixed once errkit is used everywhere + lastIndex := strings.LastIndex(cause, ":") + if lastIndex == -1 { + lastIndex = 0 + } + if lastIndex >= len(cause)-1 { + lastIndex = 0 + } + cause = cause[lastIndex+1:] + } + for _, msg := range c.TrackError { + if strings.Contains(cause, msg) { + return true + } + } + return reCheckError.MatchString(cause) } - return reCheckError.MatchString(errString) } diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index a4730366e..6fd04b475 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -1,11 +1,13 @@ package hosterrorscache import ( + "context" "fmt" "sync" "sync/atomic" "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/stretchr/testify/require" ) @@ -13,8 +15,8 @@ func TestCacheCheck(t *testing.T) { cache := New(3, DefaultMaxHostsCount, nil) for i := 0; i < 100; i++ { - cache.MarkFailed("test", fmt.Errorf("could not resolve host")) - got := cache.Check("test") + cache.MarkFailed(newCtxArgs("test"), fmt.Errorf("could not resolve host")) + got := cache.Check(newCtxArgs("test")) if i < 2 { // till 3 the host is not flagged to skip require.False(t, got) @@ -24,7 +26,7 @@ func TestCacheCheck(t *testing.T) { } } - value := cache.Check("test") + value := cache.Check(newCtxArgs("test")) require.Equal(t, true, value, "could not get checked value") } @@ -32,8 +34,8 @@ func TestTrackErrors(t *testing.T) { cache := New(3, DefaultMaxHostsCount, []string{"custom error"}) for i := 0; i < 100; i++ { - cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error")) - got := cache.Check("custom") + cache.MarkFailed(newCtxArgs("custom"), fmt.Errorf("got: nested: custom error")) + got := cache.Check(newCtxArgs("custom")) if i < 2 { // till 3 the host is not flagged to skip require.False(t, got) @@ -42,7 +44,7 @@ func TestTrackErrors(t *testing.T) { require.True(t, got) } } - value := cache.Check("custom") + value := cache.Check(newCtxArgs("custom")) require.Equal(t, true, value, "could not get checked value") } @@ -73,16 +75,18 @@ func TestCacheMarkFailed(t *testing.T) { tests := []struct { host string - expected int + expected int32 }{ {"http://example.com:80", 1}, {"example.com:80", 2}, - {"example.com", 1}, + // earlier if port is not provided then port was omitted + // but from now it will default to appropriate http scheme based port with 80 as default + {"example.com:443", 1}, } for _, test := range tests { - normalizedCacheValue := cache.normalizeCacheValue(test.host) - cache.MarkFailed(test.host, fmt.Errorf("no address found for host")) + normalizedCacheValue := cache.GetKeyFromContext(newCtxArgs(test.host), nil) + cache.MarkFailed(newCtxArgs(test.host), fmt.Errorf("no address found for host")) failedTarget, err := cache.failedTargets.Get(normalizedCacheValue) require.Nil(t, err) require.NotNil(t, failedTarget) @@ -102,7 +106,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { }{ {"http://example.com:80", 200}, {"example.com:80", 200}, - {"example.com", 100}, + {"example.com:443", 100}, } // the cache is not atomic during items creation, so we pre-create them with counter to zero @@ -120,14 +124,14 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - cache.MarkFailed(currentTest.host, fmt.Errorf("could not resolve host")) + cache.MarkFailed(newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host")) }() } } wg.Wait() for _, test := range tests { - require.True(t, cache.Check(test.host)) + require.True(t, cache.Check(newCtxArgs(test.host))) normalizedCacheValue := cache.normalizeCacheValue(test.host) failedTarget, err := cache.failedTargets.Get(normalizedCacheValue) @@ -139,3 +143,8 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { require.EqualValues(t, test.expected, value.errors.Load()) } } + +func newCtxArgs(value string) *contextargs.Context { + ctx := contextargs.NewWithInput(context.TODO(), value) + return ctx +} diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go index 2ab5d7ca9..59f7dd40b 100644 --- a/pkg/protocols/common/protocolinit/init.go +++ b/pkg/protocols/common/protocolinit/init.go @@ -13,7 +13,6 @@ import ( // Init initializes the client pools for the protocols func Init(options *types.Options) error { - if err := protocolstate.Init(options); err != nil { return err } @@ -40,4 +39,5 @@ func Init(options *types.Options) error { func Close() { protocolstate.Dialer.Close() + protocolstate.Dialer = nil } diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index ae5ac6cb9..89675e769 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -210,6 +210,7 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) { func Close() { if Dialer != nil { Dialer.Close() + Dialer = nil } StopActiveMemGuardian() } diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 348cab0ae..57cc37109 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -24,6 +24,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http" contextutil "github.com/projectdiscovery/utils/context" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" @@ -35,6 +36,8 @@ import ( var ( errinvalidArguments = errorutil.New("invalid arguments provided") ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access") + // ErrActionExecDealine is the error returned when alloted time for action execution exceeds + ErrActionExecDealine = errkit.New("headless action execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() ) const ( @@ -619,7 +622,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro // Just wait the event to happen waitFunc := func() (err error) { // execute actual wait event - ctx, cancel := context.WithTimeout(context.Background(), maxDuration) + ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, ErrActionExecDealine) defer cancel() err = contextutil.ExecFunc(ctx, p.page.WaitEvent(waitEvent)) return diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 3e2baf55a..4200e25b7 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -35,9 +35,10 @@ var ( forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] - // ResponseHeaderTimeout is the timeout for response headers + // MaxResponseHeaderTimeout is the timeout for response headers // to be read from the server (this prevents infinite hang started by server if any) - ResponseHeaderTimeout = time.Duration(5) * time.Second + // Note: this will be overridden temporarily when using @timeout request annotation + MaxResponseHeaderTimeout = time.Duration(10) * time.Second // HttpTimeoutMultiplier is the multiplier for the http timeout HttpTimeoutMultiplier = 3 ) @@ -53,6 +54,9 @@ func Init(options *types.Options) error { if normalClient != nil { return nil } + if options.Timeout > 10 { + MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second + } if options.ShouldFollowHTTPRedirects() { forceMaxRedirects = options.MaxRedirects } @@ -111,6 +115,8 @@ type Configuration struct { RedirectFlow RedirectFlow // Connection defines custom connection configuration Connection *ConnectionConfiguration + // ResponseHeaderTimeout is the timeout for response body to be read from the server + ResponseHeaderTimeout time.Duration } // Hash returns the hash of the configuration to allow client pooling @@ -129,13 +135,15 @@ func (c *Configuration) Hash() string { builder.WriteString(strconv.FormatBool(c.DisableCookie)) builder.WriteString("c") builder.WriteString(strconv.FormatBool(c.Connection != nil)) + builder.WriteString("r") + builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10)) hash := builder.String() return hash } // HasStandardOptions checks whether the configuration requires custom settings func (c *Configuration) HasStandardOptions() bool { - return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout + return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0 } // GetRawHTTP returns the rawhttp request client @@ -182,6 +190,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl maxIdleConns := 0 maxConnsPerHost := 0 maxIdleConnsPerHost := -1 + // do not split given timeout into chunks for retry + // because this won't work on slow hosts + retryableHttpOptions.NoAdjustTimeout = true if configuration.Threads > 0 || options.ScanStrategy == scanstrategy.HostSpray.String() { // Single host @@ -235,6 +246,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return nil, errors.Wrap(err, "could not create client certificate") } + // responseHeaderTimeout is max timeout for response headers to be read + responseHeaderTimeout := MaxResponseHeaderTimeout + if configuration.ResponseHeaderTimeout != 0 { + responseHeaderTimeout = configuration.ResponseHeaderTimeout + } + transport := &http.Transport{ ForceAttemptHTTP2: options.ForceAttemptHTTP2, DialContext: Dialer.Dial, @@ -252,7 +269,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl MaxConnsPerHost: maxConnsPerHost, TLSClientConfig: tlsConfig, DisableKeepAlives: disableKeepAlives, - ResponseHeaderTimeout: ResponseHeaderTimeout, + ResponseHeaderTimeout: responseHeaderTimeout, } if types.ProxyURL != "" { @@ -351,7 +368,7 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed } } -func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error { +func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) error { if maxRedirects == 0 { if len(via) > defaultMaxRedirects { return http.ErrUseLastResponse diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go new file mode 100644 index 000000000..b6e60b636 --- /dev/null +++ b/pkg/protocols/http/httpclientpool/options.go @@ -0,0 +1,54 @@ +package httpclientpool + +import ( + "io" + "net/http" + "strings" + "time" + + "github.com/projectdiscovery/rawhttp" +) + +// WithCustomTimeout is a configuration for custom timeout +type WithCustomTimeout struct { + Timeout time.Duration +} + +// RawHttpRequestOpts is a configuration for raw http request +type RawHttpRequestOpts struct { + // Method is the http method to use + Method string + // URL is the url to request + URL string + // Path is request path to use + Path string + // Headers is the headers to use + Headers map[string][]string + // Body is the body to use + Body io.Reader + // Options is more client related options + Options *rawhttp.Options +} + +// SendRawRequest sends a raw http request with the provided options and returns http response +func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Response, error) { + resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options) + if err != nil { + cause := err.Error() + if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") { + // this error is caused when rawhttp client sends a corrupted or malformed request packet to server + // some servers may attempt gracefully shutdown but most will just abruptly close the connection which results + // in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow + req, _ := http.NewRequest(opts.Method, opts.URL, opts.Body) + req.Header = opts.Headers + resp = &http.Response{ + Request: req, + StatusCode: http.StatusBadRequest, + Status: http.StatusText(http.StatusBadRequest), + Body: io.NopCloser(strings.NewReader("")), + } + return resp, nil + } + } + return resp, err +} diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index c372e3262..7b6c6cee2 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -35,8 +35,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/rawhttp" convUtil "github.com/projectdiscovery/utils/conversion" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" httpUtils "github.com/projectdiscovery/utils/http" "github.com/projectdiscovery/utils/reader" @@ -55,7 +57,9 @@ const ( var ( MaxBodyRead = int64(10 * 1024 * 1024) // 10MB // ErrMissingVars is error occured when variables are missing - ErrMissingVars = errors.New("stop execution due to unresolved variables") + ErrMissingVars = errkit.New("stop execution due to unresolved variables").SetKind(nucleierr.ErrTemplateLogic).Build() + // ErrHttpEngineRequestDeadline is error occured when request deadline set by http request engine is exceeded + ErrHttpEngineRequestDeadline = errkit.New("http request engine deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() ) // Type returns the type of the protocol request @@ -147,15 +151,15 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous return } // marks thsi host as unresponsive if applicable - request.markUnresponsiveHost(input, err) - if request.isUnresponsiveHost(input) { + request.markUnresponsiveAddress(input, err) + if request.isUnresponsiveAddress(input) { // stop all inflight requests spmHandler.Cancel() } }) for i := 0; i < request.RaceNumberRequests; i++ { - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) { // stop sending more requests condition is met break } @@ -163,7 +167,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous // execute http request go func(httpRequest *generatedRequest) { defer spmHandler.Release() - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) { // stop sending more requests condition is met return } @@ -232,8 +236,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV return } // marks thsi host as unresponsive if applicable - request.markUnresponsiveHost(input, err) - if request.isUnresponsiveHost(input) { + request.markUnresponsiveAddress(input, err) + if request.isUnresponsiveAddress(input) { // stop all inflight requests spmHandler.Cancel() } @@ -261,7 +265,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV } // break if stop at first match is found or host is unresponsive - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) { break } @@ -277,17 +281,23 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV if input.MetaInput.Input == "" { input.MetaInput.Input = generatedHttpRequest.URL() } + updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL()) + if request.isUnresponsiveAddress(updatedInput) { + // skip on unresponsive host no need to continue + spmHandler.Cancel() + return nil + } spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() { return } // putting ratelimiter here prevents any unnecessary waiting if any request.options.RateLimitTake() // after ratelimit take, check if we need to stop - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() { return } @@ -370,8 +380,8 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu return } // marks thsi host as unresponsive if applicable - request.markUnresponsiveHost(input, err) - if request.isUnresponsiveHost(input) { + request.markUnresponsiveAddress(input, err) + if request.isUnresponsiveAddress(input) { // stop all inflight requests spmHandler.Cancel() } @@ -389,7 +399,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu default: } - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) || spmHandler.Cancelled() { // skip if first match is found break } @@ -403,11 +413,17 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu if input.MetaInput.Input == "" { input.MetaInput.Input = generatedHttpRequest.URL() } + updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL()) + if request.isUnresponsiveAddress(updatedInput) { + // skip on unresponsive host no need to continue + spmHandler.Cancel() + return nil + } generatedHttpRequest.pipelinedClient = pipeClient spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) { // skip if first match is found return } @@ -466,7 +482,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.RateLimitTake() ctx := request.newContext(input) - ctxWithTimeout, cancel := context.WithTimeout(ctx, httpclientpool.GetHttpTimeout(request.options.Options)) + ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline) defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) @@ -477,6 +493,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return true, err } + // ideally if http template used a custom port or hostname + // we would want to update it in input but currently templateCtx logic + // is closely tied to contextargs.Context so we are temporarily creating + // a copy and using it to check for host errors etc + // but this should be replaced once templateCtx is refactored properly + updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL()) if generatedHttpRequest.customCancelFunction != nil { defer generatedHttpRequest.customCancelFunction() @@ -487,7 +509,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa input.MetaInput.Input = generatedHttpRequest.URL() } // Check if hosts keep erroring - if request.isUnresponsiveHost(input) { + if request.isUnresponsiveAddress(updatedInput) { return true, nil } var gotMatches bool @@ -527,7 +549,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } if execReqErr != nil { // if applicable mark the host as unresponsive - request.markUnresponsiveHost(input, execReqErr) + request.markUnresponsiveAddress(updatedInput, execReqErr) requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL()) request.options.Progress.IncrementFailedRequestsBy(1) } else { @@ -580,7 +602,7 @@ const drainReqSize = int64(8 * 1024) // executeRequest executes the actual generated request and returns error if occurred func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) { // Check if hosts keep erroring - if request.isUnresponsiveHost(input) { + if request.isUnresponsiveAddress(input) { return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input) } @@ -713,7 +735,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ inputUrl = url.String() } formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path) - resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) + + // send rawhttp request and get response + resp, err = httpclientpool.SendRawRequest(generatedRequest.original.rawhttpClient, &httpclientpool.RawHttpRequestOpts{ + Method: generatedRequest.rawRequest.Method, + URL: inputUrl, + Path: generatedRequest.rawRequest.Path, + Headers: generators.ExpandMapValues(generatedRequest.rawRequest.Headers), + Body: io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), + Options: &options, + }) } else { //** For Normal requests **// hostname = generatedRequest.request.URL.Host @@ -731,17 +762,34 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if errSignature := request.handleSignature(generatedRequest); errSignature != nil { return errSignature } - httpclient := request.httpClient + + // this will be assigned/updated if this specific request has a custom configuration + var modifiedConfig *httpclientpool.Configuration + + // check for cookie related configuration if input.CookieJar != nil { connConfiguration := request.connConfiguration connConfiguration.Connection.SetCookieJar(input.CookieJar) - client, err := httpclientpool.Get(request.options.Options, connConfiguration) + modifiedConfig = connConfiguration + } + // check for request updatedTimeout annotation + updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout) + if ok { + if modifiedConfig == nil { + modifiedConfig = request.connConfiguration + } + modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout + } + + if modifiedConfig != nil { + client, err := httpclientpool.Get(request.options.Options, modifiedConfig) if err != nil { return errors.Wrap(err, "could not get http client") } httpclient = client } + resp, err = httpclient.Do(generatedRequest.request) } } @@ -1087,20 +1135,20 @@ func (request *Request) newContext(input *contextargs.Context) context.Context { return input.Context() } -// markUnresponsiveHost checks if the error is a unreponsive host error and marks it -func (request *Request) markUnresponsiveHost(input *contextargs.Context, err error) { +// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it +func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) { if err == nil { return } if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) + request.options.HostErrorsCache.MarkFailed(input, err) } } -// isUnresponsiveHost checks if the error is a unreponsive based on its execution history -func (request *Request) isUnresponsiveHost(input *contextargs.Context) bool { +// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history +func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool { if request.options.HostErrorsCache != nil { - return request.options.HostErrorsCache.Check(input.MetaInput.ID()) + return request.options.HostErrorsCache.Check(input) } return false } diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go index 67bf7c167..c0f4b6977 100644 --- a/pkg/protocols/http/request_annotations.go +++ b/pkg/protocols/http/request_annotations.go @@ -10,7 +10,9 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/errkit" iputil "github.com/projectdiscovery/utils/ip" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -27,6 +29,15 @@ var ( reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`) // @once sets the request to be executed only once for a specific URL reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`) + + // ErrTimeoutAnnotationDeadline is the error returned when a specific amount of time was exceeded for a request + // which was alloted using @timeout annotation this usually means that vulnerability was not found + // in rare case it could also happen due to network congestion + // the assigned class is TemplateLogic since this in almost every case means that server is not vulnerable + ErrTimeoutAnnotationDeadline = errkit.New("timeout annotation deadline exceeded").SetKind(nucleierr.ErrTemplateLogic).Build() + // ErrRequestTimeoutDeadline is the error returned when a specific amount of time was exceeded for a request + // this happens when the request execution exceeds alloted time + ErrRequestTimeoutDeadline = errkit.New("request timeout deadline exceeded when notimeout is set").SetKind(errkit.ErrKindDeadline).Build() ) type flowMark int @@ -120,12 +131,16 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req value := strings.TrimSpace(duration[1]) if parsed, err := time.ParseDuration(value); err == nil { //nolint:govet // cancelled automatically by withTimeout - ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed) + // global timeout is overridden by annotation by replacing context + ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline) + // add timeout value to context + ctx = context.WithValue(ctx, httpclientpool.WithCustomTimeout{}, httpclientpool.WithCustomTimeout{Timeout: parsed}) request = request.Clone(ctx) } } else { //nolint:govet // cancelled automatically by withTimeout - ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), httpclientpool.GetHttpTimeout(r.options.Options)) + // global timeout is overridden by annotation by replacing context + ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline) request = request.Clone(ctx) } } diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index bc5f9caa9..8d980b4bf 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -6,7 +6,6 @@ package http // -> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result] import ( - "context" "fmt" "net/http" "strings" @@ -133,7 +132,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value return request.executeGeneratedFuzzingRequest(gr, input, callback) }, Values: values, - BaseRequest: baseRequest.Clone(context.TODO()), + BaseRequest: baseRequest.Clone(input.Context()), }) if err == nil { applicable = true @@ -158,7 +157,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool { hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) hasInteractMarkers := len(gr.InteractURLs) > 0 - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input) { return false } request.options.RateLimitTake() @@ -201,7 +200,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, } if requestErr != nil { if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr) + request.options.HostErrorsCache.MarkFailed(input, requestErr) } gologger.Verbose().Msgf("[%s] Error occurred in request: %s\n", request.options.TemplateID, requestErr) } diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 84457b5fc..71f9f5341 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -32,6 +32,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" syncutil "github.com/projectdiscovery/utils/sync" @@ -360,7 +361,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV } if request.generator != nil && request.Threads > 1 { - request.executeRequestParallel(context.Background(), hostPort, hostname, input, payloadValues, callback) + request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback) return nil } @@ -387,11 +388,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV } callback(result) }, requestOptions); err != nil { - _ = err - // Review: should we log error here? - // it is technically not error as it is expected to fail - // gologger.Warning().Msgf("Could not execute request: %s\n", err) - // do not return even if error occured + if errkit.IsNetworkPermanentErr(err) { + // gologger.Verbose().Msgf("Could not execute request: %s\n", err) + return err + } } // If this was a match, and we want to stop at first match, skip all further requests. shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch @@ -408,8 +408,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo if threads == 0 { threads = 1 } - ctx, cancel := context.WithCancel(ctxParent) - defer cancel() + ctx, cancel := context.WithCancelCause(ctxParent) + defer cancel(nil) requestOptions := request.options gotmatches := &atomic.Bool{} @@ -453,16 +453,15 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo } callback(result) }, requestOptions); err != nil { - _ = err - // Review: should we log error here? - // it is technically not error as it is expected to fail - // gologger.Warning().Msgf("Could not execute request: %s\n", err) - // do not return even if error occured + if errkit.IsNetworkPermanentErr(err) { + cancel(err) + return + } } // If this was a match, and we want to stop at first match, skip all further requests. if shouldStopAtFirstMatch && gotmatches.Load() { - cancel() + cancel(nil) return } }() @@ -507,7 +506,6 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte results = compiler.ExecuteResult{"success": false, "error": err.Error()} } request.options.Progress.IncrementRequests() - requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err) gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort) diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 1123f4056..62807abf9 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -1,7 +1,6 @@ package network import ( - "context" "encoding/hex" "fmt" "net" @@ -64,7 +63,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err errs = append(errs, err) continue } - conn, err := protocolstate.Dialer.Dial(context.TODO(), "tcp", addr) + conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr) if err != nil { errs = append(errs, err) continue @@ -119,6 +118,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata func (request *Request) executeOnTarget(input *contextargs.Context, visited mapsutil.Map[string, struct{}], metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var address string var err error + if request.isUnresponsiveAddress(input) { + // skip on unresponsive address no need to continue + return nil + } if request.SelfContained { address = "" @@ -173,6 +176,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA request.options.Progress.IncrementFailedRequestsBy(1) return err } + updatedTarget := input.Clone() + updatedTarget.MetaInput.Input = actualAddress // if request threads matches global payload concurrency we follow it shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency @@ -206,11 +211,19 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA m.Unlock() } } + if request.isUnresponsiveAddress(updatedTarget) { + // skip on unresponsive address no need to continue + return nil + } value = generators.MergeMaps(value, payloads) swg.Add() go func(vars map[string]interface{}) { defer swg.Done() + if request.isUnresponsiveAddress(updatedTarget) { + // skip on unresponsive address no need to continue + return + } if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, callback); err != nil { m.Lock() multiErr = multierr.Append(multiErr, err) @@ -240,13 +253,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac if host, _, err := net.SplitHostPort(actualAddress); err == nil { hostname = host } + updatedTarget := input.Clone() + updatedTarget.MetaInput.Input = actualAddress + + if request.isUnresponsiveAddress(updatedTarget) { + // skip on unresponsive address no need to continue + return nil + } if shouldUseTLS { - conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.DialTLS(input.Context(), "tcp", actualAddress) } else { - conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress) } if err != nil { + // adds it to unresponsive address list if applicable + request.markUnresponsiveAddress(updatedTarget, err) request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") @@ -475,3 +497,21 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte } return b[:count], nil } + +// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it +func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) { + if err == nil { + return + } + if request.options.HostErrorsCache != nil { + request.options.HostErrorsCache.MarkFailed(input, err) + } +} + +// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history +func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool { + if request.options.HostErrorsCache != nil { + return request.options.HostErrorsCache.Check(input) + } + return false +} diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index 0146b57c4..a956de0e3 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -1,7 +1,6 @@ package websocket import ( - "context" "crypto/tls" "fmt" "io" @@ -228,7 +227,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context, parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) addressToDial = parsedAddress.String() - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) + conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial) if err != nil { requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 97e7b67bd..603378e98 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -296,7 +296,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) + e.options.HostErrorsCache.MarkFailed(ctx.Input, err) } return results, err } @@ -330,7 +330,7 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R } if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) + e.options.HostErrorsCache.MarkFailed(ctx.Input, err) } return scanCtx.GenerateResult(), err } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 68410f7f1..4ce40ace9 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" @@ -27,6 +28,7 @@ import ( // Init initializes the protocols and their configurations func Init(options *types.Options) { + _ = protocolstate.Init(options) _ = protocolinit.Init(options) } diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index 29ace4477..3b337f6b2 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -85,7 +85,7 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { if err != nil { ctx.LogError(err) if g.options.HostErrorsCache != nil { - g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) + g.options.HostErrorsCache.MarkFailed(ctx.Input, err) } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err) } diff --git a/pkg/types/nucleierr/kinds.go b/pkg/types/nucleierr/kinds.go new file mode 100644 index 000000000..9042beaad --- /dev/null +++ b/pkg/types/nucleierr/kinds.go @@ -0,0 +1,28 @@ +package nucleierr + +import ( + "strings" + + "github.com/projectdiscovery/utils/errkit" +) + +var ( + // ErrTemplateLogic are errors that occured due to missing variable or something similar in template logic + // so this is more of a virtual error that is expected due to template logic + ErrTemplateLogic = errkit.NewPrimitiveErrKind("TemplateLogic", "Error expected due to template logic", isTemplateLogicKind) +) + +// isTemplateLogicKind checks if an error is of template logic kind +func isTemplateLogicKind(err *errkit.ErrorX) bool { + if err == nil || err.Cause() == nil { + return false + } + v := err.Cause().Error() + switch { + case strings.Contains(v, "timeout annotation deadline exceeded"): + return true + case strings.Contains(v, "stop execution due to unresolved variables"): + return true + } + return false +}