From c7c35ffb94627bfb5df53e80fd1a080d64f3f9a9 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Thu, 18 Jan 2024 05:53:42 +0530 Subject: [PATCH] fix multiple mem leaks + optimizations (#4630) * fix mem leak * bump version tag * http: add global resp body read limit of 4MB * skip creating templateCtx in normal templates * fix mem leak via retryablehttp , fastdialer * go mod tidy * remove unused var * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- go.mod | 29 +++++++------ go.sum | 63 ++++++++++++++-------------- pkg/catalog/config/constants.go | 2 +- pkg/operators/extractors/compile.go | 4 -- pkg/protocols/code/code.go | 10 +++-- pkg/protocols/dns/request.go | 8 +++- pkg/protocols/file/request.go | 4 +- pkg/protocols/headless/request.go | 9 +++- pkg/protocols/http/build_request.go | 5 ++- pkg/protocols/http/request.go | 24 +++++++---- pkg/protocols/http/utils.go | 3 +- pkg/protocols/javascript/js.go | 6 ++- pkg/protocols/network/request.go | 8 +++- pkg/protocols/offlinehttp/request.go | 4 +- pkg/protocols/protocols.go | 10 +++++ pkg/protocols/ssl/ssl.go | 10 ++++- pkg/protocols/utils/reader.go | 33 +++++++++++++++ 17 files changed, 158 insertions(+), 74 deletions(-) create mode 100644 pkg/protocols/utils/reader.go diff --git a/go.mod b/go.mod index 597c6c8f5..47e414ffb 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.53 + github.com/projectdiscovery/fastdialer v0.0.54 github.com/projectdiscovery/hmap v0.0.34 github.com/projectdiscovery/interactsh v1.1.8 - github.com/projectdiscovery/rawhttp v0.1.34 - github.com/projectdiscovery/retryabledns v1.0.50 - github.com/projectdiscovery/retryablehttp-go v1.0.42 + github.com/projectdiscovery/rawhttp v0.1.33 + github.com/projectdiscovery/retryabledns v1.0.51 + github.com/projectdiscovery/retryablehttp-go v1.0.44 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -39,7 +39,7 @@ require ( github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.19.0 + golang.org/x/net v0.20.0 golang.org/x/oauth2 v0.11.0 golang.org/x/text v0.14.0 gopkg.in/yaml.v2 v2.4.0 @@ -90,7 +90,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74 github.com/projectdiscovery/uncover v1.0.7 - github.com/projectdiscovery/utils v0.0.73-0.20240110205148-46f474b2947f + github.com/projectdiscovery/utils v0.0.74-0.20240115220656-48fef326de18 github.com/projectdiscovery/wappalyzergo v0.0.109 github.com/redis/go-redis/v9 v9.1.0 github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 @@ -112,7 +112,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/andybalholm/brotli v1.0.6 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect @@ -139,7 +139,6 @@ require ( github.com/fatih/color v1.15.0 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gaukas/godicttls v0.0.4 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect @@ -164,7 +163,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.10 // indirect - github.com/klauspost/compress v1.17.3 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect @@ -187,7 +186,7 @@ require ( github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/quic-go/quic-go v0.40.1 // indirect - github.com/refraction-networking/utls v1.5.4 // indirect + github.com/refraction-networking/utls v1.6.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.1 // indirect @@ -221,7 +220,7 @@ require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect - github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/akrylysov/pogreb v0.10.2 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect @@ -279,16 +278,16 @@ require ( github.com/ysmood/leakless v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20231106212110-94c8f62efae4 // indirect + github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect go.etcd.io/bbolt v1.3.8 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 golang.org/x/mod v0.14.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.15.0 // indirect + golang.org/x/tools v0.17.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/go.sum b/go.sum index 68112782c..db2480f33 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= -github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= +github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= @@ -108,8 +108,8 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= @@ -325,8 +325,6 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= -github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -613,8 +611,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= -github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -807,8 +805,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.38 h1:wzObLZ4m4fMtPD0RiirnAp5naF43dJymjCmEeZzJLjM= github.com/projectdiscovery/dsl v0.0.38/go.mod h1:jYaosMHbna8jGxhClWovGFBNJGM19Go0wbk6FfrV/PA= -github.com/projectdiscovery/fastdialer v0.0.53 h1:s9BuOMpeIu8E7muTwYG0aQYpp5TEdlJYy8GBCWO6V+Q= -github.com/projectdiscovery/fastdialer v0.0.53/go.mod h1:Fvbo4+QTc6grX5eGYcEHwJHE9Grq7iATP/nfTcatTKk= +github.com/projectdiscovery/fastdialer v0.0.54 h1:c90JJ2cPlVV9JK/HlgVwiupun0ONa+IXQqwGE1/IMws= +github.com/projectdiscovery/fastdialer v0.0.54/go.mod h1:l1ktd+P7b68NH/0Xu7/EAx3uoNc4iowoYj6jckuY2+k= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -835,14 +833,14 @@ github.com/projectdiscovery/networkpolicy v0.0.7 h1:AwHqBRXBqDQgnWzBMuoJtHBNEYBw github.com/projectdiscovery/networkpolicy v0.0.7/go.mod h1:CK0CnFoLF1Nou6mY7P4WODSAxhPN8g8g7XpapgEP8tI= github.com/projectdiscovery/ratelimit v0.0.24 h1:ydCzcICK8eSYkCeN5NLikGjGayx66ELQmjpqepGC5Cg= github.com/projectdiscovery/ratelimit v0.0.24/go.mod h1:cHow8VY1Dt38/6Cj3b6LmJATnzXrs6838U2GiFYBoXE= -github.com/projectdiscovery/rawhttp v0.1.34 h1:5tWp1kpPRkq6IB4Bx4ddkAATCvXo2WVRg/h1EBsfUKg= -github.com/projectdiscovery/rawhttp v0.1.34/go.mod h1:/98rgouyg6Y6QFwMBD9Hf0fKyq9GMUNDgDovM5mCDMI= +github.com/projectdiscovery/rawhttp v0.1.33 h1:H+QM7zVq/pprGrQrO0JyCSRwrCYdIHbQ3hueh6P5u/4= +github.com/projectdiscovery/rawhttp v0.1.33/go.mod h1:3W9ZZr3feIVbK0VQHZv6c5OKqb+tnTdh3cbNOOO8AY8= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.50 h1:0nM3x29G5LAZ7urfl0jSs501RQ5q57SkPwkdY19ECn8= -github.com/projectdiscovery/retryabledns v1.0.50/go.mod h1:CbQhVC9JjtqU/89gz25gs6UgpQKYwFN2RoWoW5a/M9Q= -github.com/projectdiscovery/retryablehttp-go v1.0.42 h1:NW76U/r0pWNi6iudBqggG69sN8aguuXLLbGRkLvniyo= -github.com/projectdiscovery/retryablehttp-go v1.0.42/go.mod h1:NWR4amTNHwM+ALk1QL1HiyzhFejRTMCHapM+oSoNSv8= +github.com/projectdiscovery/retryabledns v1.0.51 h1:bX/apiRGZwhASBAT7o3qmZ0FznuBlHQlIQdCw1TAzcg= +github.com/projectdiscovery/retryabledns v1.0.51/go.mod h1:rFu1zc7HLHPEipuF91ZNMT1yGG0FKBVUnxnqLJ4OhF4= +github.com/projectdiscovery/retryablehttp-go v1.0.44 h1:hicCe2h6daHt4muPovmffZE3YKBqGioreO6EpIGZ87g= +github.com/projectdiscovery/retryablehttp-go v1.0.44/go.mod h1:7ECXK2cH2/G4sstf8hacyrMdPPJ/3wCAO5tFPZ4iO4s= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -851,8 +849,8 @@ github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74 h1:G0gw+3z github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74/go.mod h1:YH8el7/6pyZbNed1IibjzbGpeigiCVyvE28g5+LsPAw= github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= -github.com/projectdiscovery/utils v0.0.73-0.20240110205148-46f474b2947f h1:gjp9RyGSh507ARIrUUBE0u13mMYPDWZxoZEOAneCT4E= -github.com/projectdiscovery/utils v0.0.73-0.20240110205148-46f474b2947f/go.mod h1:B+4cyms2mQtG6KC2qaV5zEVJgyIWzjCwrgO+kpSR5SY= +github.com/projectdiscovery/utils v0.0.74-0.20240115220656-48fef326de18 h1:hQHfr0YlGGODVMQrN3c41itC477xdFDy/3hJbOfjPqY= +github.com/projectdiscovery/utils v0.0.74-0.20240115220656-48fef326de18/go.mod h1:SEb3ZoGy1nxdnPNXAGhMZNhRcokRkoMEjC6l9H59t1s= github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k= github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -894,8 +892,8 @@ github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpT github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= -github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= -github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= +github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA= +github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1093,8 +1091,8 @@ github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+T github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w= -github.com/zmap/zcrypto v0.0.0-20231106212110-94c8f62efae4 h1:YBEjlA0uAnTqljTgqFgA3NQUrcDSc850G2KuWnZ91UQ= -github.com/zmap/zcrypto v0.0.0-20231106212110-94c8f62efae4/go.mod h1:Z2SNNuFhO+AAsezbGEHTWeW30hHv5niUYT3fwJ61Nl0= +github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s= +github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk= github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo= github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 h1:LaMyYFWQA7kh3ovPfAaFDTKlJu3JGng8khruOtsBVnE= github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706/go.mod h1:re2kMcs84XHb8Xl6RInt0emoKCuphfmfjHYuteviLHQ= @@ -1151,8 +1149,9 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1163,8 +1162,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1246,8 +1245,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1272,8 +1271,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1348,6 +1347,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1361,6 +1361,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1438,8 +1439,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= -golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 7afd7956a..f229aee94 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.1.5` + Version = `v3.1.6-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/operators/extractors/compile.go b/pkg/operators/extractors/compile.go index a70975f77..2b55d374a 100644 --- a/pkg/operators/extractors/compile.go +++ b/pkg/operators/extractors/compile.go @@ -10,10 +10,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" ) -const ( - ExtractedResultsDir = "extracted" -) - // CompileExtractors performs the initial setup operation on an extractor func (e *Extractor) CompileExtractors() error { // Set up the extractor type diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index c681c1c4b..e2cbc4202 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -144,8 +144,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // inject all template context values as gozero env allvars allvars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) - // add template context values - allvars = generators.MergeMaps(allvars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + // add template context values if available + if request.options.HasTemplateCtx(input.MetaInput) { + allvars = generators.MergeMaps(allvars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) variablesMap := request.options.Variables.Evaluate(allvars) @@ -206,7 +208,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data) // add variables from template context before matching/extraction - data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(interactshURLs, data) diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go index 2b6705584..4b364ffa1 100644 --- a/pkg/protocols/dns/request.go +++ b/pkg/protocols/dns/request.go @@ -53,7 +53,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) // merge with metadata (eg. from workflow context) - vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(vars, variablesMap, request.options.Constants) @@ -160,7 +162,9 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad outputEvent[k] = v } // add variables from template context before matching/extraction - outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) dumpResponse(event, request, request.options, response.String(), question) diff --git a/pkg/protocols/file/request.go b/pkg/protocols/file/request.go index bf81a12a0..eb73544f9 100644 --- a/pkg/protocols/file/request.go +++ b/pkg/protocols/file/request.go @@ -248,7 +248,9 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input *contextar dslMap[k] = v } // add template context variables to DSL map - dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug) newOpResult := discardEvent.OperatorsResult if newOpResult != nil { diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go index cfd4491fc..f800c261c 100644 --- a/pkg/protocols/headless/request.go +++ b/pkg/protocols/headless/request.go @@ -54,7 +54,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, vars := protocolutils.GenerateVariablesWithContextArgs(input, false) payloads := generators.BuildPayloadFromOptions(request.options.Options) // add templatecontext variables to varMap - values := generators.MergeMaps(vars, metadata, payloads, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + values := generators.MergeMaps(vars, metadata, payloads) + if request.options.HasTemplateCtx(input.MetaInput) { + values = generators.MergeMaps(values, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } variablesMap := request.options.Variables.Evaluate(values) payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants) @@ -183,7 +186,9 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory()) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } for k, v := range out { outputEvent[k] = v } diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index 7819ebcb8..e74356b0d 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -76,7 +76,10 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, // add template context values to dynamicValues (this takes care of self-contained and other types of requests) // Note: `iterate-all` and flow are mutually exclusive. flow uses templateCtx and iterate-all uses dynamicValues - dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if r.request.options.HasTemplateCtx(input.MetaInput) { + // skip creating template context if not available + dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } if r.request.SelfContained { return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues) } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index d1dd7accf..660b2430e 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -34,6 +34,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" + protocolutil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/rawhttp" @@ -43,7 +44,9 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) -const defaultMaxWorkers = 150 +const ( + defaultMaxWorkers = 150 +) // Type returns the type of the protocol request func (request *Request) Type() templateTypes.ProtocolType { @@ -477,11 +480,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ finalMap["ip"] = input.MetaInput.CustomIP } - for payloadName, payloadValue := range generatedRequest.dynamicValues { - if data, err := expressions.Evaluate(types.ToString(payloadValue), finalMap); err == nil { - generatedRequest.dynamicValues[payloadName] = data - } - } + // we should never evaluate all variables of a template + // for payloadName, payloadValue := range generatedRequest.dynamicValues { + // if data, err := expressions.Evaluate(types.ToString(payloadValue), finalMap); err == nil { + // generatedRequest.dynamicValues[payloadName] = data + // } + // } for payloadName, payloadValue := range generatedRequest.meta { if data, err := expressions.Evaluate(types.ToString(payloadValue), finalMap); err == nil { generatedRequest.meta[payloadName] = data @@ -647,6 +651,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } } } + // global wrap response body reader + if resp != nil && resp.Body != nil { + resp.Body = protocolutil.NewLimitResponseBody(resp.Body) + } if err != nil { // rawhttp doesn't support draining response bodies. if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil && !generatedRequest.original.Pipeline { @@ -769,7 +777,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } diff --git a/pkg/protocols/http/utils.go b/pkg/protocols/http/utils.go index f2b364894..d6b43547b 100644 --- a/pkg/protocols/http/utils.go +++ b/pkg/protocols/http/utils.go @@ -14,6 +14,7 @@ import ( "golang.org/x/text/transform" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + protoUtil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/rawhttp" mapsutil "github.com/projectdiscovery/utils/maps" @@ -64,7 +65,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect break } if redirectResp.Body != nil { - body, _ = io.ReadAll(redirectResp.Body) + body, _ = protoUtil.LimitBodyRead(redirectResp.Body) } respObj := redirectedResponse{ headers: respData, diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index c83367c12..54752d360 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -419,7 +419,11 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte if err != nil { return err } - argsCopy.TemplateCtx = request.options.GetTemplateCtx(input.MetaInput).GetAll() + if request.options.HasTemplateCtx(input.MetaInput) { + argsCopy.TemplateCtx = request.options.GetTemplateCtx(input.MetaInput).GetAll() + } else { + argsCopy.TemplateCtx = map[string]interface{}{} + } var requestData = []byte(request.Code) var interactshURLs []string diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index c896b6551..dedf85915 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -136,7 +136,9 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps } variables := protocolutils.GenerateVariables(address, false, nil) // add template ctx variables to varMap - variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } variablesMap := request.options.Variables.Evaluate(variables) variables = generators.MergeMaps(variablesMap, variables, request.options.Constants) @@ -327,7 +329,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final), response, input.MetaInput.Input, actualAddress) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } outputEvent["ip"] = request.dialer.GetDialedIP(hostname) if request.options.StopAtFirstMatch { outputEvent["stop-at-first-match"] = true diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go index 8d7556252..6191f67be 100644 --- a/pkg/protocols/offlinehttp/request.go +++ b/pkg/protocols/offlinehttp/request.go @@ -89,7 +89,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 37ca4b845..ccc65e5f0 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -125,6 +125,15 @@ func (e *ExecutorOptions) RemoveTemplateCtx(input *contextargs.MetaInput) { } } +// HasTemplateCtx returns true if template context exists for given input +func (e *ExecutorOptions) HasTemplateCtx(input *contextargs.MetaInput) bool { + scanId := input.GetScanHash(e.TemplateID) + if e.templateCtxStore != nil { + return e.templateCtxStore.Has(scanId) + } + return false +} + // GetTemplateCtx returns template context for given input func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contextargs.Context { scanId := input.GetScanHash(e.TemplateID) @@ -132,6 +141,7 @@ func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contexta if !ok { // if template context does not exist create new and add it to store and return it templateCtx = contextargs.New() + templateCtx.MetaInput = input _ = e.templateCtxStore.Set(scanId, templateCtx) } return templateCtx diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 03d0300df..9b185a4f4 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -215,7 +215,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa hostnameVariables := protocolutils.GenerateDNSVariables(hostname) // add template context variables to varMap - values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + values := generators.MergeMaps(payloadValues, hostnameVariables) + if request.options.HasTemplateCtx(input.MetaInput) { + values = generators.MergeMaps(values, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } + variablesMap := request.options.Variables.Evaluate(values) payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants) @@ -322,7 +326,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } // add response fields ^ to template context and merge templatectx variables to output event - data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.HasTemplateCtx(input.MetaInput) { + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + } event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input) diff --git a/pkg/protocols/utils/reader.go b/pkg/protocols/utils/reader.go new file mode 100644 index 000000000..62ad53cd0 --- /dev/null +++ b/pkg/protocols/utils/reader.go @@ -0,0 +1,33 @@ +package utils + +import ( + "io" +) + +var ( + MaxBodyRead = int64(1 << 22) // 4MB using shift operator +) + +var _ io.ReadCloser = &LimitResponseBody{} + +type LimitResponseBody struct { + io.Reader + io.Closer +} + +// NewLimitResponseBody wraps response body with a limit reader. +// thus only allowing MaxBodyRead bytes to be read. i.e 4MB +func NewLimitResponseBody(body io.ReadCloser) io.ReadCloser { + if body == nil { + return nil + } + return &LimitResponseBody{ + Reader: io.LimitReader(body, MaxBodyRead), + Closer: body, + } +} + +// LimitBodyRead limits the body read to MaxBodyRead bytes. +func LimitBodyRead(r io.Reader) ([]byte, error) { + return io.ReadAll(io.LimitReader(r, MaxBodyRead)) +}