multiple bug fixes + performance improvements (#5148)

* prototype errkit

* complete errkit implementation

* add cause to all timeouts

* fix request timeout annotation @timeout

* increase responseHeaderTimeout to 8 for stability

* rawhttp error related improvements

* feat: add port status caching

* add port status caching to http

* migrate to new utils/errkit

* remote dialinterface + error cause

* debug dir support using .gitignore debug-*

* make nuclei easy to debug

* debug dir update .gitignore

* temp change (to revert)

* Revert "temp change (to revert)"

This reverts commit d3131f777713b9f80e2275142e80f36340a76d36.

* use available context instead of new one

* bump fastdialer

* fix hosterrorscache + misc improvements

* add 'address' field in error log

* fix js vague errors + pgwrap driver

* fix max host error + misc updates

* update tests as per changes

* fix request annotation context

* remove closed dialer reference

* fix sdk panic issue

* bump retryablehttp-go,utils,fastdialer

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>

.gitignore

@@ -39,5 +39,5 @@ integration_tests/fuzzplayground
 /nuclei-stats
 /nuclei-stats-*
 /scan-charts
-
+/**/debug-*
 

go.mod

@@ -16,16 +16,16 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/miekg/dns v1.1.57
+	github.com/miekg/dns v1.1.59
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.72
+	github.com/projectdiscovery/fastdialer v0.1.0
-	github.com/projectdiscovery/hmap v0.0.42
+	github.com/projectdiscovery/hmap v0.0.43
-	github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663
+	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.50
+	github.com/projectdiscovery/rawhttp v0.1.49
-	github.com/projectdiscovery/retryabledns v1.0.59
+	github.com/projectdiscovery/retryabledns v1.0.60
-	github.com/projectdiscovery/retryablehttp-go v1.0.60
+	github.com/projectdiscovery/retryablehttp-go v1.0.62
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -35,7 +35,7 @@ require (
 	github.com/spf13/cast v1.5.1
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/valyala/fasttemplate v1.2.2
-	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
+	github.com/weppos/publicsuffix-go v0.30.2
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/net v0.25.0
@@ -73,7 +73,7 @@ require (
 	github.com/h2non/filetype v1.1.3
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
-	github.com/lib/pq v1.10.1
+	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
@@ -94,12 +94,12 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.52
-	github.com/projectdiscovery/utils v0.0.94
+	github.com/projectdiscovery/utils v0.1.0
 	github.com/projectdiscovery/wappalyzergo v0.1.0
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
-	github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
+	github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
 	golang.org/x/term v0.20.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -134,7 +134,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.4 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cloudflare/cfssl v1.6.4 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudflare/circl v1.3.8 // indirect
 	github.com/containerd/continuity v0.4.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
@@ -175,7 +175,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kataras/jwt v0.1.10 // indirect
-	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
@@ -203,16 +203,15 @@ require (
 	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/quic-go/quic-go v0.42.0 // indirect
+	github.com/refraction-networking/utls v1.6.6 // indirect
-	github.com/refraction-networking/utls v1.6.1 // indirect
 	github.com/sashabaranov/go-openai v1.15.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.2.1 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
-	github.com/tidwall/buntdb v1.3.0 // indirect
+	github.com/tidwall/buntdb v1.3.1 // indirect
-	github.com/tidwall/gjson v1.17.0 // indirect
+	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
@@ -233,7 +232,7 @@ require (
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	mellium.im/sasl v0.3.1 // indirect
 )
@@ -298,16 +297,16 @@ require (
 	github.com/ysmood/leakless v0.8.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
-	github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
+	github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db // indirect
-	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/bbolt v1.3.10 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
 	golang.org/x/crypto v0.23.0 // indirect
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
-	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.17.0
+	golang.org/x/tools v0.21.0
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect

go.sum

@@ -235,8 +235,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8=
 github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
@@ -367,8 +367,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
@@ -394,8 +392,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
 github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
@@ -630,8 +626,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -662,8 +658,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk=
 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa/go.mod h1:HbwNE4XGwjgtUELkvQaAOjWrpianHYZdQVNqSdYW3UM=
-github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
-github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
 github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@@ -714,8 +710,8 @@ github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3r
 github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
-github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
 github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
@@ -773,8 +769,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
@@ -837,8 +831,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY=
 github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og=
-github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E=
+github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
-github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0=
+github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -853,12 +847,12 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
 github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM=
+github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg=
-github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c=
+github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc=
 github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc=
 github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss=
-github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 h1:0ih5JJyTht2sv4J4lbfSPaUkKdFDci+Bwxg6vsyd2F4=
+github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
-github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663/go.mod h1:gTrQCQfYlhoIM4X11+zwL7uRuraOUCoiE4Z6z1QD86g=
+github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I=
 github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
 github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
@@ -871,14 +865,14 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
 github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4=
 github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y=
-github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM=
+github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
-github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg=
+github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
+github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
-github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ=
+github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
-github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8=
+github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
-github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64=
+github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -889,8 +883,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
 github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
-github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo=
+github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
-github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
+github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
 github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
 github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -927,13 +921,11 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
-github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
 github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
-github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA=
+github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4llzEXOkL43Ig=
-github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo=
+github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -1015,17 +1007,17 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 h1:GXIyLuIJ5Qk46lI8WJ83qHBZKUI3zhmMmuoY9HICUIQ=
-github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
 github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA=
+github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o=
-github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
+github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
 github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
 github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
@@ -1068,8 +1060,8 @@ github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+
 github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
-github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 h1:h2JizvZl9aIj6za9S5AyrkU+OzIS4CetQthH/ejO+lg=
+github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU=
-github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ=
+github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM=
 github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
@@ -1132,15 +1124,15 @@ github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+T
 github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
 github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
 github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w=
-github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s=
+github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I=
-github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk=
+github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g=
 github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo=
 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 h1:LaMyYFWQA7kh3ovPfAaFDTKlJu3JGng8khruOtsBVnE=
 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706/go.mod h1:re2kMcs84XHb8Xl6RInt0emoKCuphfmfjHYuteviLHQ=
 github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -1188,8 +1180,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1202,8 +1194,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1226,8 +1218,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1283,7 +1275,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1310,8 +1303,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1383,9 +1376,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -1398,8 +1390,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1415,7 +1407,7 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
@@ -1476,8 +1468,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/runner/options.go

@@ -19,6 +19,7 @@ import (
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
@@ -114,7 +115,12 @@ func ParseOptions(options *types.Options) {
 	// Load the resolvers if user asked for them
 	loadResolvers(options)
 
-	err := protocolinit.Init(options)
+	err := protocolstate.Init(options)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
+	}
+
+	err = protocolinit.Init(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
 	}

internal/runner/runner.go

@@ -359,6 +359,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
+	// dump hosterrors cache
+	if r.hostErrors != nil {
+		r.hostErrors.Close()
+	}
 	if r.output != nil {
 		r.output.Close()
 	}
@@ -464,7 +468,7 @@ func (r *Runner) RunEnumeration() error {
 		Parser:             r.parser,
 	}
 
-	if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" {
+	if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
 		// Go StdLib style experimental/debug feature switch
 		executorOpts.ExportReqURLPattern = true
 	}

lib/multi.go

@@ -123,7 +123,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
 			return err
 		}
 	}
-	defer tmpEngine.Close()
 	// create ephemeral nuclei objects/instances/types using base nuclei engine
 	unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
 	if err != nil {

lib/sdk_private.go

@@ -25,7 +25,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -119,7 +118,6 @@ func (e *NucleiEngine) init() error {
 	e.parser = templates.NewParser()
 
 	sharedInit.Do(func() {
-		_ = protocolstate.Init(e.opts)
 		_ = protocolinit.Init(e.opts)
 	})
 

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.7`
+	Version = `v3.2.8-dev`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"
@@ -68,3 +68,13 @@ func trimDevIfExists(version string) string {
 	}
 	return version
 }
+
+// similar to go pattern of enabling debug related features
+// we add custom/extra switches for debugging purposes
+const (
+	// DebugArgHostErrorStats is used to print host error stats
+	// when it is closed
+	DebugArgHostErrorStats = "host-error-stats"
+	// DebugExportReqURLPattern is used to export request URL pattern
+	DebugExportURLPattern = "req-url-pattern"
+)

pkg/catalog/config/nucleiconfig.go

@@ -46,9 +46,10 @@ type Config struct {
 	LatestNucleiIgnoreHash       string `json:"nuclei-latest-ignore-hash,omitempty"`
 
 	// internal / unexported fields
-	disableUpdates bool   `json:"-"` // disable updates both version check and template updates
+	disableUpdates bool     `json:"-"` // disable updates both version check and template updates
-	homeDir        string `json:"-"` //  User Home Directory
+	homeDir        string   `json:"-"` //  User Home Directory
-	configDir      string `json:"-"` //  Nuclei Global Config Directory
+	configDir      string   `json:"-"` //  Nuclei Global Config Directory
+	debugArgs      []string `json:"-"` // debug args
 }
 
 // IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates.
@@ -329,6 +330,46 @@ func (c *Config) copyIgnoreFile() {
 	}
 }
 
+// IsDebugArgEnabled checks if debug arg is enabled
+// this could be a feature specific to debugging like PPROF or printing stats
+// of max host error etc
+func (c *Config) IsDebugArgEnabled(arg string) bool {
+	for _, v := range c.debugArgs {
+		if v == arg {
+			return true
+		}
+	}
+	return false
+}
+
+// parseDebugArgs from string
+func (c *Config) parseDebugArgs(data string) {
+	// use space as seperator instead of commas
+	tmp := strings.Fields(data)
+	for _, v := range tmp {
+		key := v
+		value := ""
+		// if it is key value pair then split it
+		if strings.Contains(v, "=") {
+			parts := strings.SplitN(v, "=", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			key, value = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])
+		}
+		if value == "false" || value == "0" {
+			// if false or disabled then skip
+			continue
+		}
+		switch key {
+		case DebugArgHostErrorStats:
+			c.debugArgs = append(c.debugArgs, DebugArgHostErrorStats)
+		case DebugExportURLPattern:
+			c.debugArgs = append(c.debugArgs, DebugExportURLPattern)
+		}
+	}
+}
+
 func init() {
 	// first attempt to migrate all files from old config directory to new config directory
 	goflags.AttemptConfigMigration() // regardless how many times this is called it will only migrate once based on condition
@@ -377,6 +418,7 @@ func init() {
 	// and even if it is changed we don't follow it since it is not expected behavior
 	// If custom templates are in default locations only then they are loaded while running nuclei
 	DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory)
+	DefaultConfig.parseDebugArgs(env.GetEnvOrDefault("NUCLEI_ARGS", ""))
 }
 
 // Add Default Config adds default when .templates-config.json file is not present

pkg/core/executors.go

@@ -107,7 +107,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
 		currentInfo.Unlock()
 
 		// Skip if the host has had errors
-		if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
+		if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(contextargs.NewWithMetaInput(ctx, scannedValue)) {
 			return true
 		}
 

pkg/core/workflow_execute.go

@@ -98,7 +98,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 			}
 			if err != nil {
 				if w.Options.HostErrorsCache != nil {
-					w.Options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
+					w.Options.HostErrorsCache.MarkFailed(ctx.Input, err)
 				}
 				if len(template.Executers) == 1 {
 					mainErr = err

pkg/js/compiler/compiler.go

@@ -10,9 +10,15 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	contextutil "github.com/projectdiscovery/utils/context"
+	"github.com/projectdiscovery/utils/errkit"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
+var (
+	// ErrJSExecDeadline is the error returned when alloted time for script execution exceeds
+	ErrJSExecDeadline = errkit.New("js engine execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
+)
+
 // Compiler provides a runtime to execute goja runtime
 // based javascript scripts efficiently while also
 // providing them access to custom modules defined in libs/.
@@ -107,7 +113,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 	}
 
 	// execute with context and timeout
-	ctx, cancel := context.WithTimeout(opts.Context, time.Duration(opts.Timeout)*time.Second)
+	ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline)
 	defer cancel()
 	// execute the script
 	results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
@@ -119,6 +125,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 		return ExecuteProgram(program, args, opts)
 	})
 	if err != nil {
+		if val, ok := err.(*goja.Exception); ok {
+			err = val.Unwrap()
+		}
 		return nil, err
 	}
 	var res ExecuteResult

pkg/js/compiler/init.go

@@ -1,6 +1,8 @@
 package compiler
 
-import "github.com/projectdiscovery/nuclei/v3/pkg/types"
+import (
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
 
 // jsprotocolInit
 
@@ -9,6 +11,7 @@ var (
 	JsProtocolTimeout       = 10
 	PoolingJsVmConcurrency  = 100
 	NonPoolingVMConcurrency = 20
+	JsTimeoutMultiplier     = 1.5
 )
 
 // Init initializes the javascript protocol
@@ -21,7 +24,9 @@ func Init(opts *types.Options) error {
 		// 100 is reasonable default
 		opts.JsConcurrency = 100
 	}
-	JsProtocolTimeout = opts.Timeout
+	// we have dialer timeout set to 10s so js needs to be at least
+	// 15s to return the actual error if not it will be a dialer timeout
+	JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier)
 	PoolingJsVmConcurrency = opts.JsConcurrency
 	PoolingJsVmConcurrency -= NonPoolingVMConcurrency
 	return nil

pkg/js/libs/postgres/memo.postgres.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	_ "github.com/lib/pq"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
 
 	utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
 

pkg/js/libs/postgres/postgres.go

@@ -9,10 +9,11 @@ import (
 	"time"
 
 	"github.com/go-pg/pg"
-	_ "github.com/lib/pq"
 	"github.com/praetorian-inc/fingerprintx/pkg/plugins"
 	postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql"
 	utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 )
 
@@ -100,7 +101,7 @@ func executeQuery(host string, port int, username string, password string, dbNam
 	target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
 
 	connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", username, password, target, dbName)
-	db, err := sql.Open("postgres", connStr)
+	db, err := sql.Open(pgwrap.PGWrapDriver, connStr)
 	if err != nil {
 		return nil, err
 	}
@@ -152,6 +153,9 @@ func connect(host string, port int, username string, password string, dbName str
 		User:               username,
 		Password:           password,
 		Database:           dbName,
+		Dialer: func(network, addr string) (net.Conn, error) {
+			return protocolstate.Dialer.Dial(context.Background(), network, addr)
+		},
 		IdleCheckFrequency: -1,
 	}).WithContext(ctx).WithTimeout(10 * time.Second)
 	defer db.Close()

pkg/js/utils/pgwrap/pgwrap.go

@@ -0,0 +1,53 @@
+package pgwrap
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"net"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+)
+
+const (
+	PGWrapDriver = "pgwrap"
+)
+
+type pgDial struct {
+	fd *fastdialer.Dialer
+}
+
+func (p *pgDial) Dial(network, address string) (net.Conn, error) {
+	return p.fd.Dial(context.TODO(), network, address)
+}
+
+func (p *pgDial) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {
+	ctx, cancel := context.WithTimeoutCause(context.Background(), timeout, fastdialer.ErrDialTimeout)
+	defer cancel()
+	return p.fd.Dial(ctx, network, address)
+}
+
+func (p *pgDial) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return p.fd.Dial(ctx, network, address)
+}
+
+// Unfortunately lib/pq does not provide easy to customize or
+// replace dialer so we need to hijack it by wrapping it in our own
+// driver and register it as postgres driver
+
+// PgDriver is the Postgres database driver.
+type PgDriver struct{}
+
+// Open opens a new connection to the database. name is a connection string.
+// Most users should only use it through database/sql package from the standard
+// library.
+func (d PgDriver) Open(name string) (driver.Conn, error) {
+	return pq.DialOpen(&pgDial{fd: protocolstate.Dialer}, name)
+}
+
+func init() {
+	sql.Register(PGWrapDriver, &PgDriver{})
+}

pkg/output/output.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -27,9 +28,12 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+	"github.com/projectdiscovery/utils/errkit"
 	fileutil "github.com/projectdiscovery/utils/file"
 	osutils "github.com/projectdiscovery/utils/os"
+	urlutil "github.com/projectdiscovery/utils/url"
 )
 
 // Writer is an interface which writes output to somewhere for nuclei events.
@@ -299,10 +303,13 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 
 // JSONLogRequest is a trace/error log request written to file
 type JSONLogRequest struct {
-	Template string `json:"template"`
+	Template string      `json:"template"`
-	Input    string `json:"input"`
+	Type     string      `json:"type"`
-	Error    string `json:"error"`
+	Input    string      `json:"input"`
-	Type     string `json:"type"`
+	Address  string      `json:"address"`
+	Error    string      `json:"error"`
+	Kind     string      `json:"kind,omitempty"`
+	Attrs    interface{} `json:"attrs,omitempty"`
 }
 
 // Request writes a log the requests trace log
@@ -315,12 +322,43 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 		Input:    input,
 		Type:     requestType,
 	}
-	if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil {
+	parsed, _ := urlutil.ParseAbsoluteURL(input, false)
-		request.Error = unwrappedErr.Error()
+	if parsed != nil {
-	} else {
+		request.Address = parsed.Hostname()
-		request.Error = "none"
+		port := parsed.Port()
+		if port == "" {
+			switch parsed.Scheme {
+			case urlutil.HTTP:
+				port = "80"
+			case urlutil.HTTPS:
+				port = "443"
+			}
+		}
+		request.Address += ":" + port
+	}
+	errX := errkit.FromError(requestErr)
+	if errX == nil {
+		request.Error = "none"
+	} else {
+		request.Kind = errkit.ErrKindUnknown.String()
+		var cause error
+		if len(errX.Errors()) > 1 {
+			cause = errX.Errors()[0]
+		}
+		if cause == nil {
+			cause = errX
+		}
+		cause = tryParseCause(cause)
+		request.Error = cause.Error()
+		request.Kind = errkit.GetErrorKind(requestErr, nucleierr.ErrTemplateLogic).String()
+		if len(errX.Attrs()) > 0 {
+			request.Attrs = slog.GroupValue(errX.Attrs()...)
+		}
+	}
+	// check if address slog attr is avaiable in error if set use it
+	if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil {
+		request.Address = val.String()
 	}
-
 	data, err := jsoniter.Marshal(request)
 	if err != nil {
 		return
@@ -452,3 +490,24 @@ func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string,
 	}
 
 }
+
+// tryParseCause tries to parse the cause of given error
+// this is legacy support due to use of errorutil in existing libraries
+// but this should not be required once all libraries are updated
+func tryParseCause(err error) error {
+	if err == nil {
+		return nil
+	}
+	msg := err.Error()
+	if strings.HasPrefix(msg, "ReadStatusLine:") {
+		// last index is actual error (from rawhttp)
+		parts := strings.Split(msg, ":")
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+	}
+	if strings.Contains(msg, "read ") {
+		// same here
+		parts := strings.Split(msg, ":")
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+	}
+	return err
+}

pkg/output/output_test.go

@@ -30,7 +30,7 @@ func TestStandardWriterRequest(t *testing.T) {
 		require.NoError(t, err)
 		w.Request("path", "input", "http", nil)
 
-		require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String())
+		require.Equal(t, `{"template":"path","type":"http","input":"input","address":"input:","error":"none"}`, traceWriter.String())
 		require.Empty(t, errorWriter.String())
 	})
 
@@ -47,7 +47,7 @@ func TestStandardWriterRequest(t *testing.T) {
 			fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")),
 		)
 
-		require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String())
+		require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","type":"http","input":"https://example.com/tcpconfig.html","address":"example.com:443","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","kind":"unknown-error"}`, errorWriter.String())
 	})
 }
 

pkg/protocols/code/code.go

@@ -32,6 +32,7 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	contextutil "github.com/projectdiscovery/utils/context"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 )
 
@@ -41,7 +42,10 @@ const (
 )
 
 var (
+	// pythonEnvRegexCompiled is the compiled regex for python environment variables
 	pythonEnvRegexCompiled = regexp.MustCompile(pythonEnvRegex)
+	// ErrCodeExecutionDeadline is the error returned when alloted time for script execution exceeds
+	ErrCodeExecutionDeadline = errkit.New("code execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 // Request is a request for the SSL protocol
@@ -214,7 +218,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		}
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline)
 	defer cancel()
 	// Note: we use contextutil despite the fact that gozero accepts context as argument
 	gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) {

pkg/protocols/common/contextargs/contextargs.go

@@ -37,6 +37,13 @@ func New(ctx context.Context) *Context {
 	return NewWithInput(ctx, "")
 }
 
+// NewWithMetaInput creates a new contextargs instance with meta input
+func NewWithMetaInput(ctx context.Context, input *MetaInput) *Context {
+	n := New(ctx)
+	n.MetaInput = input
+	return n
+}
+
 // Create a new contextargs instance with input string
 func NewWithInput(ctx context.Context, input string) *Context {
 	jar, err := cookiejar.New(nil)
@@ -174,3 +181,20 @@ func (ctx *Context) Clone() *Context {
 	}
 	return newCtx
 }
+
+// GetCopyIfHostOutdated returns a new contextargs if the host is outdated
+func GetCopyIfHostOutdated(ctx *Context, url string) *Context {
+	if ctx.MetaInput.Input == "" {
+		newctx := ctx.Clone()
+		newctx.MetaInput.Input = url
+		return newctx
+	}
+	orig, _ := urlutil.Parse(ctx.MetaInput.Input)
+	newURL, _ := urlutil.Parse(url)
+	if orig != nil && newURL != nil && orig.Host != newURL.Host {
+		newCtx := ctx.Clone()
+		newCtx.MetaInput.Input = newURL.Host
+		return newCtx
+	}
+	return ctx
+}

pkg/protocols/common/contextargs/metainput.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"crypto/md5"
 	"fmt"
+	"net"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
 	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/segmentio/ksuid"
 )
 
 // MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput)
@@ -47,6 +49,56 @@ func (metaInput *MetaInput) URL() (*urlutil.URL, error) {
 	return instance, nil
 }
 
+// Port returns the port of the target
+// if port is not present then empty string is returned
+func (metaInput *MetaInput) Port() string {
+	target, err := urlutil.ParseAbsoluteURL(metaInput.Input, false)
+	if err != nil {
+		return ""
+	}
+	return target.Port()
+}
+
+// Address return the remote address of target
+// Note: it does not resolve the domain to ip
+// it is meant to be used by hosterrorsCache if invalid metainput
+// is provided then it uses a random ksuid as hostname to avoid skipping valid targets
+func (metaInput *MetaInput) Address() string {
+	var hostname, port string
+	target, err := urlutil.ParseAbsoluteURL(metaInput.Target(), false)
+	if err != nil {
+		if metaInput.CustomIP == "" {
+			// since this is used in hosterrorscache we add a random id
+			// which will never be used to avoid skipping valid targets
+			hostname = fmt.Sprintf("invalid-%s", ksuid.New().String())
+		}
+	} else {
+		hostname = target.Hostname()
+		port = target.Port()
+		if port == "" {
+			switch target.Scheme {
+			case urlutil.HTTP:
+				port = "80"
+			case urlutil.HTTPS:
+				port = "443"
+			default:
+				port = "80"
+			}
+		}
+	}
+	if metaInput.CustomIP != "" {
+		hostname = metaInput.CustomIP
+	}
+	if port == "" {
+		if strings.HasPrefix(hostname, "http://") {
+			port = "80"
+		} else if strings.HasPrefix(hostname, "https://") {
+			port = "443"
+		}
+	}
+	return net.JoinHostPort(hostname, port)
+}
+
 // ID returns a unique id/hash for metainput
 func (metaInput *MetaInput) ID() string {
 	if metaInput.CustomIP != "" {

pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -10,17 +10,25 @@ import (
 
 	"github.com/bluele/gcache"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
+	"github.com/projectdiscovery/utils/errkit"
 )
 
 // CacheInterface defines the signature of the hosterrorscache so that
 // users of Nuclei as embedded lib may implement their own cache
 type CacheInterface interface {
-	SetVerbose(verbose bool)            // log verbosely
+	SetVerbose(verbose bool)                        // log verbosely
-	Close()                             // close the cache
+	Close()                                         // close the cache
-	Check(value string) bool            // return true if the host should be skipped
+	Check(ctx *contextargs.Context) bool            // return true if the host should be skipped
-	MarkFailed(value string, err error) // record a failure (and cause) for the host
+	MarkFailed(ctx *contextargs.Context, err error) // record a failure (and cause) for the host
 }
 
+var (
+	_ CacheInterface = (*Cache)(nil)
+)
+
 // Cache is a cache for host based errors. It allows skipping
 // certain hosts based on an error threshold.
 //
@@ -34,8 +42,10 @@ type Cache struct {
 }
 
 type cacheItem struct {
-	errors atomic.Int32
 	sync.Once
+	errors         atomic.Int32
+	isPermanentErr bool
+	cause          error // optional cause
 }
 
 const DefaultMaxHostsCount = 10000
@@ -55,6 +65,16 @@ func (c *Cache) SetVerbose(verbose bool) {
 
 // Close closes the host errors cache
 func (c *Cache) Close() {
+	if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) {
+		items := c.failedTargets.GetALL(false)
+		for k, v := range items {
+			val, ok := v.(*cacheItem)
+			if !ok {
+				continue
+			}
+			gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load())
+		}
+	}
 	c.failedTargets.Purge()
 }
 
@@ -88,14 +108,19 @@ func (c *Cache) normalizeCacheValue(value string) string {
 //   - URL: https?:// type
 //   - Host:port type
 //   - host type
-func (c *Cache) Check(value string) bool {
+func (c *Cache) Check(ctx *contextargs.Context) bool {
-	finalValue := c.normalizeCacheValue(value)
+	finalValue := c.GetKeyFromContext(ctx, nil)
 
 	existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
 	if err != nil {
 		return false
 	}
 	existingCacheItemValue := existingCacheItem.(*cacheItem)
+	if existingCacheItemValue.isPermanentErr {
+		// skipping permanent errors is expected so verbose instead of info
+		gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause)
+		return true
+	}
 
 	if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) {
 		existingCacheItemValue.Do(func() {
@@ -107,15 +132,22 @@ func (c *Cache) Check(value string) bool {
 }
 
 // MarkFailed marks a host as failed previously
-func (c *Cache) MarkFailed(value string, err error) {
+func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
 	if !c.checkError(err) {
 		return
 	}
-	finalValue := c.normalizeCacheValue(value)
+	finalValue := c.GetKeyFromContext(ctx, err)
 	existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
 	if err != nil || existingCacheItem == nil {
 		newItem := &cacheItem{errors: atomic.Int32{}}
 		newItem.errors.Store(1)
+		if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) {
+			// skip this address altogether
+			// permanent errors are always permanent hence this is created once
+			// and never updated so no need to synchronize
+			newItem.isPermanentErr = true
+			newItem.cause = err
+		}
 		_ = c.failedTargets.Set(finalValue, newItem)
 		return
 	}
@@ -124,19 +156,77 @@ func (c *Cache) MarkFailed(value string, err error) {
 	_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
 }
 
-var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host|timeout awaiting response headers)`)
+// GetKeyFromContext returns the key for the cache from the context
+func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string {
+	// Note:
+	// ideally any changes made to remote addr in template like {{Hostname}}:81 etc
+	// should be reflected in contextargs but it is not yet reflected in some cases
+	// and needs refactor of ScanContext + ContextArgs to achieve that
+	// i.e why we use real address from error if present
+	address := ctx.MetaInput.Address()
+	// get address override from error
+	if err != nil {
+		tmp := errkit.GetAttrValue(err, "address")
+		if tmp.Any() != nil {
+			address = tmp.String()
+		}
+	}
+	finalValue := c.normalizeCacheValue(address)
+	return finalValue
+}
+
+var reCheckError = regexp.MustCompile(`(no address found for host|could not resolve host|connection refused|connection reset by peer|could not connect to any address found for host|timeout awaiting response headers)`)
 
 // checkError checks if an error represents a type that should be
 // added to the host skipping table.
+// it first parses error and extracts the cause and checks for blacklisted
+// or common errors that should be skipped
 func (c *Cache) checkError(err error) bool {
 	if err == nil {
 		return false
 	}
-	errString := err.Error()
+	kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
-	for _, msg := range c.TrackError {
+	switch kind {
-		if strings.Contains(errString, msg) {
+	case nucleierr.ErrTemplateLogic:
-			return true
+		// these are errors that are not related to the target
+		// and are due to template logic
+		return false
+	case errkit.ErrKindNetworkTemporary:
+		// these should not be counted as host errors
+		return false
+	case errkit.ErrKindNetworkPermanent:
+		// these should be counted as host errors
+		return true
+	case errkit.ErrKindDeadline:
+		// these should not be counted as host errors
+		return false
+	default:
+		// parse error for furthur processing
+		errX := errkit.FromError(err)
+		tmp := errX.Cause()
+		cause := tmp.Error()
+		if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") {
+			// this is a FP and should not be counted as a host error
+			// because server closes connection when it reads corrupted bytes which we send via rawhttp
+			return false
 		}
+		if strings.HasPrefix(cause, "ReadStatusLine:") {
+			// error is present in last part when using rawhttp
+			// this will be fixed once errkit is used everywhere
+			lastIndex := strings.LastIndex(cause, ":")
+			if lastIndex == -1 {
+				lastIndex = 0
+			}
+			if lastIndex >= len(cause)-1 {
+				lastIndex = 0
+			}
+			cause = cause[lastIndex+1:]
+		}
+		for _, msg := range c.TrackError {
+			if strings.Contains(cause, msg) {
+				return true
+			}
+		}
+		return reCheckError.MatchString(cause)
 	}
-	return reCheckError.MatchString(errString)
 }

pkg/protocols/common/hosterrorscache/hosterrorscache_test.go

@@ -1,11 +1,13 @@
 package hosterrorscache
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"sync/atomic"
 	"testing"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/stretchr/testify/require"
 )
 
@@ -13,8 +15,8 @@ func TestCacheCheck(t *testing.T) {
 	cache := New(3, DefaultMaxHostsCount, nil)
 
 	for i := 0; i < 100; i++ {
-		cache.MarkFailed("test", fmt.Errorf("could not resolve host"))
+		cache.MarkFailed(newCtxArgs("test"), fmt.Errorf("could not resolve host"))
-		got := cache.Check("test")
+		got := cache.Check(newCtxArgs("test"))
 		if i < 2 {
 			// till 3 the host is not flagged to skip
 			require.False(t, got)
@@ -24,7 +26,7 @@ func TestCacheCheck(t *testing.T) {
 		}
 	}
 
-	value := cache.Check("test")
+	value := cache.Check(newCtxArgs("test"))
 	require.Equal(t, true, value, "could not get checked value")
 }
 
@@ -32,8 +34,8 @@ func TestTrackErrors(t *testing.T) {
 	cache := New(3, DefaultMaxHostsCount, []string{"custom error"})
 
 	for i := 0; i < 100; i++ {
-		cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error"))
+		cache.MarkFailed(newCtxArgs("custom"), fmt.Errorf("got: nested: custom error"))
-		got := cache.Check("custom")
+		got := cache.Check(newCtxArgs("custom"))
 		if i < 2 {
 			// till 3 the host is not flagged to skip
 			require.False(t, got)
@@ -42,7 +44,7 @@ func TestTrackErrors(t *testing.T) {
 			require.True(t, got)
 		}
 	}
-	value := cache.Check("custom")
+	value := cache.Check(newCtxArgs("custom"))
 	require.Equal(t, true, value, "could not get checked value")
 }
 
@@ -73,16 +75,18 @@ func TestCacheMarkFailed(t *testing.T) {
 
 	tests := []struct {
 		host     string
-		expected int
+		expected int32
 	}{
 		{"http://example.com:80", 1},
 		{"example.com:80", 2},
-		{"example.com", 1},
+		// earlier if port is not provided then port was omitted
+		// but from now it will default to appropriate http scheme based port with 80 as default
+		{"example.com:443", 1},
 	}
 
 	for _, test := range tests {
-		normalizedCacheValue := cache.normalizeCacheValue(test.host)
+		normalizedCacheValue := cache.GetKeyFromContext(newCtxArgs(test.host), nil)
-		cache.MarkFailed(test.host, fmt.Errorf("no address found for host"))
+		cache.MarkFailed(newCtxArgs(test.host), fmt.Errorf("no address found for host"))
 		failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
 		require.Nil(t, err)
 		require.NotNil(t, failedTarget)
@@ -102,7 +106,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 	}{
 		{"http://example.com:80", 200},
 		{"example.com:80", 200},
-		{"example.com", 100},
+		{"example.com:443", 100},
 	}
 
 	// the cache is not atomic during items creation, so we pre-create them with counter to zero
@@ -120,14 +124,14 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				cache.MarkFailed(currentTest.host, fmt.Errorf("could not resolve host"))
+				cache.MarkFailed(newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host"))
 			}()
 		}
 	}
 	wg.Wait()
 
 	for _, test := range tests {
-		require.True(t, cache.Check(test.host))
+		require.True(t, cache.Check(newCtxArgs(test.host)))
 
 		normalizedCacheValue := cache.normalizeCacheValue(test.host)
 		failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
@@ -139,3 +143,8 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 		require.EqualValues(t, test.expected, value.errors.Load())
 	}
 }
+
+func newCtxArgs(value string) *contextargs.Context {
+	ctx := contextargs.NewWithInput(context.TODO(), value)
+	return ctx
+}

pkg/protocols/common/protocolinit/init.go

@@ -13,7 +13,6 @@ import (
 
 // Init initializes the client pools for the protocols
 func Init(options *types.Options) error {
-
 	if err := protocolstate.Init(options); err != nil {
 		return err
 	}
@@ -40,4 +39,5 @@ func Init(options *types.Options) error {
 
 func Close() {
 	protocolstate.Dialer.Close()
+	protocolstate.Dialer = nil
 }

pkg/protocols/common/protocolstate/state.go

@@ -210,6 +210,7 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) {
 func Close() {
 	if Dialer != nil {
 		Dialer.Close()
+		Dialer = nil
 	}
 	StopActiveMemGuardian()
 }

pkg/protocols/headless/engine/page_actions.go

@@ -24,6 +24,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
 	contextutil "github.com/projectdiscovery/utils/context"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
@@ -35,6 +36,8 @@ import (
 var (
 	errinvalidArguments = errorutil.New("invalid arguments provided")
 	ErrLFAccessDenied   = errorutil.New("Use -allow-local-file-access flag to enable local file access")
+	// ErrActionExecDealine is the error returned when alloted time for action execution exceeds
+	ErrActionExecDealine = errkit.New("headless action execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 const (
@@ -619,7 +622,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro
 	// Just wait the event to happen
 	waitFunc := func() (err error) {
 		// execute actual wait event
-		ctx, cancel := context.WithTimeout(context.Background(), maxDuration)
+		ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, ErrActionExecDealine)
 		defer cancel()
 		err = contextutil.ExecFunc(ctx, p.page.WaitEvent(waitEvent))
 		return

pkg/protocols/http/httpclientpool/clientpool.go

@@ -35,9 +35,10 @@ var (
 	forceMaxRedirects int
 	normalClient      *retryablehttp.Client
 	clientPool        *mapsutil.SyncLockMap[string, *retryablehttp.Client]
-	// ResponseHeaderTimeout is the timeout for response headers
+	// MaxResponseHeaderTimeout is the timeout for response headers
 	// to be read from the server (this prevents infinite hang started by server if any)
-	ResponseHeaderTimeout = time.Duration(5) * time.Second
+	// Note: this will be overridden temporarily when using @timeout request annotation
+	MaxResponseHeaderTimeout = time.Duration(10) * time.Second
 	// HttpTimeoutMultiplier is the multiplier for the http timeout
 	HttpTimeoutMultiplier = 3
 )
@@ -53,6 +54,9 @@ func Init(options *types.Options) error {
 	if normalClient != nil {
 		return nil
 	}
+	if options.Timeout > 10 {
+		MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second
+	}
 	if options.ShouldFollowHTTPRedirects() {
 		forceMaxRedirects = options.MaxRedirects
 	}
@@ -111,6 +115,8 @@ type Configuration struct {
 	RedirectFlow RedirectFlow
 	// Connection defines custom connection configuration
 	Connection *ConnectionConfiguration
+	// ResponseHeaderTimeout is the timeout for response body to be read from the server
+	ResponseHeaderTimeout time.Duration
 }
 
 // Hash returns the hash of the configuration to allow client pooling
@@ -129,13 +135,15 @@ func (c *Configuration) Hash() string {
 	builder.WriteString(strconv.FormatBool(c.DisableCookie))
 	builder.WriteString("c")
 	builder.WriteString(strconv.FormatBool(c.Connection != nil))
+	builder.WriteString("r")
+	builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10))
 	hash := builder.String()
 	return hash
 }
 
 // HasStandardOptions checks whether the configuration requires custom settings
 func (c *Configuration) HasStandardOptions() bool {
-	return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout
+	return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0
 }
 
 // GetRawHTTP returns the rawhttp request client
@@ -182,6 +190,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	maxIdleConns := 0
 	maxConnsPerHost := 0
 	maxIdleConnsPerHost := -1
+	// do not split given timeout into chunks for retry
+	// because this won't work on slow hosts
+	retryableHttpOptions.NoAdjustTimeout = true
 
 	if configuration.Threads > 0 || options.ScanStrategy == scanstrategy.HostSpray.String() {
 		// Single host
@@ -235,6 +246,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		return nil, errors.Wrap(err, "could not create client certificate")
 	}
 
+	// responseHeaderTimeout is max timeout for response headers to be read
+	responseHeaderTimeout := MaxResponseHeaderTimeout
+	if configuration.ResponseHeaderTimeout != 0 {
+		responseHeaderTimeout = configuration.ResponseHeaderTimeout
+	}
+
 	transport := &http.Transport{
 		ForceAttemptHTTP2: options.ForceAttemptHTTP2,
 		DialContext:       Dialer.Dial,
@@ -252,7 +269,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		MaxConnsPerHost:       maxConnsPerHost,
 		TLSClientConfig:       tlsConfig,
 		DisableKeepAlives:     disableKeepAlives,
-		ResponseHeaderTimeout: ResponseHeaderTimeout,
+		ResponseHeaderTimeout: responseHeaderTimeout,
 	}
 
 	if types.ProxyURL != "" {
@@ -351,7 +368,7 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed
 	}
 }
 
-func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error {
+func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) error {
 	if maxRedirects == 0 {
 		if len(via) > defaultMaxRedirects {
 			return http.ErrUseLastResponse

pkg/protocols/http/httpclientpool/options.go

@@ -0,0 +1,54 @@
+package httpclientpool
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/projectdiscovery/rawhttp"
+)
+
+// WithCustomTimeout is a configuration for custom timeout
+type WithCustomTimeout struct {
+	Timeout time.Duration
+}
+
+// RawHttpRequestOpts is a configuration for raw http request
+type RawHttpRequestOpts struct {
+	// Method is the http method to use
+	Method string
+	// URL is the url to request
+	URL string
+	// Path is request path to use
+	Path string
+	// Headers is the headers to use
+	Headers map[string][]string
+	// Body is the body to use
+	Body io.Reader
+	// Options is more client related options
+	Options *rawhttp.Options
+}
+
+// SendRawRequest sends a raw http request with the provided options and returns http response
+func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Response, error) {
+	resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options)
+	if err != nil {
+		cause := err.Error()
+		if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") {
+			// this error is caused when rawhttp client sends a corrupted or malformed request packet to server
+			// some servers may attempt gracefully shutdown but most will just abruptly close the connection which results
+			// in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow
+			req, _ := http.NewRequest(opts.Method, opts.URL, opts.Body)
+			req.Header = opts.Headers
+			resp = &http.Response{
+				Request:    req,
+				StatusCode: http.StatusBadRequest,
+				Status:     http.StatusText(http.StatusBadRequest),
+				Body:       io.NopCloser(strings.NewReader("")),
+			}
+			return resp, nil
+		}
+	}
+	return resp, err
+}

pkg/protocols/http/request.go

@@ -35,8 +35,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/rawhttp"
 	convUtil "github.com/projectdiscovery/utils/conversion"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	httpUtils "github.com/projectdiscovery/utils/http"
 	"github.com/projectdiscovery/utils/reader"
@@ -55,7 +57,9 @@ const (
 var (
 	MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
 	// ErrMissingVars is error occured when variables are missing
-	ErrMissingVars = errors.New("stop execution due to unresolved variables")
+	ErrMissingVars = errkit.New("stop execution due to unresolved variables").SetKind(nucleierr.ErrTemplateLogic).Build()
+	// ErrHttpEngineRequestDeadline is error occured when request deadline set by http request engine is exceeded
+	ErrHttpEngineRequestDeadline = errkit.New("http request engine deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 // Type returns the type of the protocol request
@@ -147,15 +151,15 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 			return
 		}
 		// marks thsi host as unresponsive if applicable
-		request.markUnresponsiveHost(input, err)
+		request.markUnresponsiveAddress(input, err)
-		if request.isUnresponsiveHost(input) {
+		if request.isUnresponsiveAddress(input) {
 			// stop all inflight requests
 			spmHandler.Cancel()
 		}
 	})
 
 	for i := 0; i < request.RaceNumberRequests; i++ {
-		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
 			// stop sending more requests condition is met
 			break
 		}
@@ -163,7 +167,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 		// execute http request
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
 				// stop sending more requests condition is met
 				return
 			}
@@ -232,8 +236,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 			return
 		}
 		// marks thsi host as unresponsive if applicable
-		request.markUnresponsiveHost(input, err)
+		request.markUnresponsiveAddress(input, err)
-		if request.isUnresponsiveHost(input) {
+		if request.isUnresponsiveAddress(input) {
 			// stop all inflight requests
 			spmHandler.Cancel()
 		}
@@ -261,7 +265,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		}
 
 		// break if stop at first match is found or host is unresponsive
-		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
 			break
 		}
 
@@ -277,17 +281,23 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		if input.MetaInput.Input == "" {
 			input.MetaInput.Input = generatedHttpRequest.URL()
 		}
+		updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
+		if request.isUnresponsiveAddress(updatedInput) {
+			// skip on unresponsive host no need to continue
+			spmHandler.Cancel()
+			return nil
+		}
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() {
 				return
 			}
 			// putting ratelimiter here prevents any unnecessary waiting if any
 			request.options.RateLimitTake()
 
 			// after ratelimit take, check if we need to stop
-			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() {
 				return
 			}
 
@@ -370,8 +380,8 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 			return
 		}
 		// marks thsi host as unresponsive if applicable
-		request.markUnresponsiveHost(input, err)
+		request.markUnresponsiveAddress(input, err)
-		if request.isUnresponsiveHost(input) {
+		if request.isUnresponsiveAddress(input) {
 			// stop all inflight requests
 			spmHandler.Cancel()
 		}
@@ -389,7 +399,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		default:
 		}
 
-		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) || spmHandler.Cancelled() {
 			// skip if first match is found
 			break
 		}
@@ -403,11 +413,17 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		if input.MetaInput.Input == "" {
 			input.MetaInput.Input = generatedHttpRequest.URL()
 		}
+		updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
+		if request.isUnresponsiveAddress(updatedInput) {
+			// skip on unresponsive host no need to continue
+			spmHandler.Cancel()
+			return nil
+		}
 		generatedHttpRequest.pipelinedClient = pipeClient
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) {
 				// skip if first match is found
 				return
 			}
@@ -466,7 +482,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
-			ctxWithTimeout, cancel := context.WithTimeout(ctx, httpclientpool.GetHttpTimeout(request.options.Options))
+			ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline)
 			defer cancel()
 
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
@@ -477,6 +493,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 				return true, err
 			}
+			// ideally if http template used a custom port or hostname
+			// we would want to update it in input but currently templateCtx logic
+			// is closely tied to contextargs.Context so we are temporarily creating
+			// a copy and using it to check for host errors etc
+			// but this should be replaced once templateCtx is refactored properly
+			updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
 
 			if generatedHttpRequest.customCancelFunction != nil {
 				defer generatedHttpRequest.customCancelFunction()
@@ -487,7 +509,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				input.MetaInput.Input = generatedHttpRequest.URL()
 			}
 			// Check if hosts keep erroring
-			if request.isUnresponsiveHost(input) {
+			if request.isUnresponsiveAddress(updatedInput) {
 				return true, nil
 			}
 			var gotMatches bool
@@ -527,7 +549,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			}
 			if execReqErr != nil {
 				// if applicable mark the host as unresponsive
-				request.markUnresponsiveHost(input, execReqErr)
+				request.markUnresponsiveAddress(updatedInput, execReqErr)
 				requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
 				request.options.Progress.IncrementFailedRequestsBy(1)
 			} else {
@@ -580,7 +602,7 @@ const drainReqSize = int64(8 * 1024)
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
 	// Check if hosts keep erroring
-	if request.isUnresponsiveHost(input) {
+	if request.isUnresponsiveAddress(input) {
 		return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
 	}
 
@@ -713,7 +735,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			inputUrl = url.String()
 		}
 		formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path)
-		resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
+
+		// send rawhttp request and get response
+		resp, err = httpclientpool.SendRawRequest(generatedRequest.original.rawhttpClient, &httpclientpool.RawHttpRequestOpts{
+			Method:  generatedRequest.rawRequest.Method,
+			URL:     inputUrl,
+			Path:    generatedRequest.rawRequest.Path,
+			Headers: generators.ExpandMapValues(generatedRequest.rawRequest.Headers),
+			Body:    io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)),
+			Options: &options,
+		})
 	} else {
 		//** For Normal requests **//
 		hostname = generatedRequest.request.URL.Host
@@ -731,17 +762,34 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
 				return errSignature
 			}
-
 			httpclient := request.httpClient
+
+			// this will be assigned/updated if this specific request has a custom configuration
+			var modifiedConfig *httpclientpool.Configuration
+
+			// check for cookie related configuration
 			if input.CookieJar != nil {
 				connConfiguration := request.connConfiguration
 				connConfiguration.Connection.SetCookieJar(input.CookieJar)
-				client, err := httpclientpool.Get(request.options.Options, connConfiguration)
+				modifiedConfig = connConfiguration
+			}
+			// check for request updatedTimeout annotation
+			updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout)
+			if ok {
+				if modifiedConfig == nil {
+					modifiedConfig = request.connConfiguration
+				}
+				modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout
+			}
+
+			if modifiedConfig != nil {
+				client, err := httpclientpool.Get(request.options.Options, modifiedConfig)
 				if err != nil {
 					return errors.Wrap(err, "could not get http client")
 				}
 				httpclient = client
 			}
+
 			resp, err = httpclient.Do(generatedRequest.request)
 		}
 	}
@@ -1087,20 +1135,20 @@ func (request *Request) newContext(input *contextargs.Context) context.Context {
 	return input.Context()
 }
 
-// markUnresponsiveHost checks if the error is a unreponsive host error and marks it
+// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it
-func (request *Request) markUnresponsiveHost(input *contextargs.Context, err error) {
+func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) {
 	if err == nil {
 		return
 	}
 	if request.options.HostErrorsCache != nil {
-		request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
+		request.options.HostErrorsCache.MarkFailed(input, err)
 	}
 }
 
-// isUnresponsiveHost checks if the error is a unreponsive based on its execution history
+// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
-func (request *Request) isUnresponsiveHost(input *contextargs.Context) bool {
+func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
 	if request.options.HostErrorsCache != nil {
-		return request.options.HostErrorsCache.Check(input.MetaInput.ID())
+		return request.options.HostErrorsCache.Check(input)
 	}
 	return false
 }

pkg/protocols/http/request_annotations.go

@@ -10,7 +10,9 @@ import (
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/utils/errkit"
 	iputil "github.com/projectdiscovery/utils/ip"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -27,6 +29,15 @@ var (
 	reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`)
 	// @once sets the request to be executed only once for a specific URL
 	reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`)
+
+	// ErrTimeoutAnnotationDeadline is the error returned when a specific amount of time was exceeded for a request
+	// which was alloted using @timeout annotation this usually means that vulnerability was not found
+	// in rare case it could also happen due to network congestion
+	// the assigned class is TemplateLogic since this in almost every case means that server is not vulnerable
+	ErrTimeoutAnnotationDeadline = errkit.New("timeout annotation deadline exceeded").SetKind(nucleierr.ErrTemplateLogic).Build()
+	// ErrRequestTimeoutDeadline is the error returned when a specific amount of time was exceeded for a request
+	// this happens when the request execution exceeds alloted time
+	ErrRequestTimeoutDeadline = errkit.New("request timeout deadline exceeded when notimeout is set").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 type flowMark int
@@ -120,12 +131,16 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 			value := strings.TrimSpace(duration[1])
 			if parsed, err := time.ParseDuration(value); err == nil {
 				//nolint:govet // cancelled automatically by withTimeout
-				ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed)
+				// global timeout is overridden by annotation by replacing context
+				ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline)
+				// add timeout value to context
+				ctx = context.WithValue(ctx, httpclientpool.WithCustomTimeout{}, httpclientpool.WithCustomTimeout{Timeout: parsed})
 				request = request.Clone(ctx)
 			}
 		} else {
 			//nolint:govet // cancelled automatically by withTimeout
-			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), httpclientpool.GetHttpTimeout(r.options.Options))
+			// global timeout is overridden by annotation by replacing context
+			ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline)
 			request = request.Clone(ctx)
 		}
 	}

pkg/protocols/http/request_fuzz.go

@@ -6,7 +6,6 @@ package http
 //		-> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result]
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -133,7 +132,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
 				return request.executeGeneratedFuzzingRequest(gr, input, callback)
 			},
 			Values:      values,
-			BaseRequest: baseRequest.Clone(context.TODO()),
+			BaseRequest: baseRequest.Clone(input.Context()),
 		})
 		if err == nil {
 			applicable = true
@@ -158,7 +157,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
 func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool {
 	hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 	hasInteractMarkers := len(gr.InteractURLs) > 0
-	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
+	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input) {
 		return false
 	}
 	request.options.RateLimitTake()
@@ -201,7 +200,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	}
 	if requestErr != nil {
 		if request.options.HostErrorsCache != nil {
-			request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)
+			request.options.HostErrorsCache.MarkFailed(input, requestErr)
 		}
 		gologger.Verbose().Msgf("[%s] Error occurred in request: %s\n", request.options.TemplateID, requestErr)
 	}

pkg/protocols/javascript/js.go

@@ -32,6 +32,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
 	syncutil "github.com/projectdiscovery/utils/sync"
@@ -360,7 +361,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 	}
 
 	if request.generator != nil && request.Threads > 1 {
-		request.executeRequestParallel(context.Background(), hostPort, hostname, input, payloadValues, callback)
+		request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback)
 		return nil
 	}
 
@@ -387,11 +388,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				}
 				callback(result)
 			}, requestOptions); err != nil {
-				_ = err
+				if errkit.IsNetworkPermanentErr(err) {
-				// Review: should we log error here?
+					// gologger.Verbose().Msgf("Could not execute request: %s\n", err)
-				// it is technically not error as it is expected to fail
+					return err
-				// gologger.Warning().Msgf("Could not execute request: %s\n", err)
+				}
-				// do not return even if error occured
 			}
 			// If this was a match, and we want to stop at first match, skip all further requests.
 			shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
@@ -408,8 +408,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	if threads == 0 {
 		threads = 1
 	}
-	ctx, cancel := context.WithCancel(ctxParent)
+	ctx, cancel := context.WithCancelCause(ctxParent)
-	defer cancel()
+	defer cancel(nil)
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
@@ -453,16 +453,15 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 					}
 					callback(result)
 				}, requestOptions); err != nil {
-					_ = err
+					if errkit.IsNetworkPermanentErr(err) {
-					// Review: should we log error here?
+						cancel(err)
-					// it is technically not error as it is expected to fail
+						return
-					// gologger.Warning().Msgf("Could not execute request: %s\n", err)
+					}
-					// do not return even if error occured
 				}
 				// If this was a match, and we want to stop at first match, skip all further requests.
 
 				if shouldStopAtFirstMatch && gotmatches.Load() {
-					cancel()
+					cancel(nil)
 					return
 				}
 			}()
@@ -507,7 +506,6 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 		results = compiler.ExecuteResult{"success": false, "error": err.Error()}
 	}
 	request.options.Progress.IncrementRequests()
-
 	requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err)
 	gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort)
 

pkg/protocols/network/request.go

@@ -1,7 +1,6 @@
 package network
 
 import (
-	"context"
 	"encoding/hex"
 	"fmt"
 	"net"
@@ -64,7 +63,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err
 			errs = append(errs, err)
 			continue
 		}
-		conn, err := protocolstate.Dialer.Dial(context.TODO(), "tcp", addr)
+		conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr)
 		if err != nil {
 			errs = append(errs, err)
 			continue
@@ -119,6 +118,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata
 func (request *Request) executeOnTarget(input *contextargs.Context, visited mapsutil.Map[string, struct{}], metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	var address string
 	var err error
+	if request.isUnresponsiveAddress(input) {
+		// skip on unresponsive address no need to continue
+		return nil
+	}
 
 	if request.SelfContained {
 		address = ""
@@ -173,6 +176,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return err
 	}
+	updatedTarget := input.Clone()
+	updatedTarget.MetaInput.Input = actualAddress
 
 	// if request threads matches global payload concurrency we follow it
 	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
@@ -206,11 +211,19 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 					m.Unlock()
 				}
 			}
+			if request.isUnresponsiveAddress(updatedTarget) {
+				// skip on unresponsive address no need to continue
+				return nil
+			}
 
 			value = generators.MergeMaps(value, payloads)
 			swg.Add()
 			go func(vars map[string]interface{}) {
 				defer swg.Done()
+				if request.isUnresponsiveAddress(updatedTarget) {
+					// skip on unresponsive address no need to continue
+					return
+				}
 				if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, callback); err != nil {
 					m.Lock()
 					multiErr = multierr.Append(multiErr, err)
@@ -240,13 +253,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 	if host, _, err := net.SplitHostPort(actualAddress); err == nil {
 		hostname = host
 	}
+	updatedTarget := input.Clone()
+	updatedTarget.MetaInput.Input = actualAddress
+
+	if request.isUnresponsiveAddress(updatedTarget) {
+		// skip on unresponsive address no need to continue
+		return nil
+	}
 
 	if shouldUseTLS {
-		conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress)
+		conn, err = request.dialer.DialTLS(input.Context(), "tcp", actualAddress)
 	} else {
-		conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress)
+		conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress)
 	}
 	if err != nil {
+		// adds it to unresponsive address list if applicable
+		request.markUnresponsiveAddress(updatedTarget, err)
 		request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return errors.Wrap(err, "could not connect to server")
@@ -475,3 +497,21 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte
 	}
 	return b[:count], nil
 }
+
+// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) {
+	if err == nil {
+		return
+	}
+	if request.options.HostErrorsCache != nil {
+		request.options.HostErrorsCache.MarkFailed(input, err)
+	}
+}
+
+// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
+	if request.options.HostErrorsCache != nil {
+		return request.options.HostErrorsCache.Check(input)
+	}
+	return false
+}

pkg/protocols/websocket/websocket.go

@@ -1,7 +1,6 @@
 package websocket
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
@@ -228,7 +227,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
 	parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)
 	addressToDial = parsedAddress.String()
 
-	conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial)
+	conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial)
 	if err != nil {
 		requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
 		requestOptions.Progress.IncrementFailedRequestsBy(1)

pkg/templates/cluster.go

@@ -296,7 +296,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 		}
 	})
 	if err != nil && e.options.HostErrorsCache != nil {
-		e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err)
+		e.options.HostErrorsCache.MarkFailed(ctx.Input, err)
 	}
 	return results, err
 }
@@ -330,7 +330,7 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R
 	}
 
 	if err != nil && e.options.HostErrorsCache != nil {
-		e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err)
+		e.options.HostErrorsCache.MarkFailed(ctx.Input, err)
 	}
 	return scanCtx.GenerateResult(), err
 }

pkg/testutils/testutils.go

@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
@@ -27,6 +28,7 @@ import (
 
 // Init initializes the protocols and their configurations
 func Init(options *types.Options) {
+	_ = protocolstate.Init(options)
 	_ = protocolinit.Init(options)
 }
 

pkg/tmplexec/generic/exec.go

@@ -85,7 +85,7 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
 		if err != nil {
 			ctx.LogError(err)
 			if g.options.HostErrorsCache != nil {
-				g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
+				g.options.HostErrorsCache.MarkFailed(ctx.Input, err)
 			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
 		}

pkg/types/nucleierr/kinds.go

@@ -0,0 +1,28 @@
+package nucleierr
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/utils/errkit"
+)
+
+var (
+	// ErrTemplateLogic are errors that occured due to missing variable or something similar in template logic
+	// so this is more of a virtual error that is expected due to template logic
+	ErrTemplateLogic = errkit.NewPrimitiveErrKind("TemplateLogic", "Error expected due to template logic", isTemplateLogicKind)
+)
+
+// isTemplateLogicKind checks if an error is of template logic kind
+func isTemplateLogicKind(err *errkit.ErrorX) bool {
+	if err == nil || err.Cause() == nil {
+		return false
+	}
+	v := err.Cause().Error()
+	switch {
+	case strings.Contains(v, "timeout annotation deadline exceeded"):
+		return true
+	case strings.Contains(v, "stop execution due to unresolved variables"):
+		return true
+	}
+	return false
+}