mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-18 05:15:24 +00:00
Issue 2254 uncover integration (#2786)
* nuclei -uq 'vuln:CVE-2021-26855' -t cves/2021/ - `nuclei -uq 'vuln:CVE-2021-26855' -t cves/2021/` * Add automatic template execution using metadata - Query uncover after the template is loaded. - Add the received hosts to the input provider from uncover - Make NormalizeStoreInputValue() function public to add hosts from the runner after uncover hosts received. * run go mod tidy * Remove unnecessary comments * Resolve the requested changes - move uncover code to protocols/common/uncover package - Use uncover delay to create uncover rate limiter - Use single ratelimiter object and remove not required ratelimiters - Create Set() method for input provider interface - Rename normalizeStoreInputValue to Set() method * Solved the uncover running twice. - flag StringSliceVarP adds the default value twice in the variable - Check if provider keys exists or not - Add uncover help block to english readme.md * Add uncover field functionality - ./nuclei -uq 'vuln:CVE-2021-26855' -t dns -duc -uf host - ./nuclei -uq 'vuln:CVE-2021-26855' -t dns -duc -uf ip:port * Update error messages and solve nuclei hang for wrong uncover engine - Get uncover engine values from uncover package * Resolve merge conflicts * misc option update * Update logging for templates - remove duplicate env log printing - Log message for template queries Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
This commit is contained in:
Shubham Rasal
GitHub
parent
0c588cf9af
commit
6b142d794a
@ -176,6 +176,14 @@ INTERACTSH:
|
||||
-interactions-cooldown-period int extra time for interaction polling before exiting (default 5)
|
||||
-ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates
|
||||
|
||||
UNCOVER:
|
||||
-uc, -uncover enable uncover engine
|
||||
-uq, -uncover-query string[] uncover search query
|
||||
-ue, -uncover-engine string[] uncover search engine (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye) (default shodan)
|
||||
-uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port")
|
||||
-ul, -uncover-limit int uncover results to return (default 100)
|
||||
-ucd, -uncover-delay int delay between uncover query requests in seconds (0 to disable) (default 1)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute
|
||||
|
||||
@ -18,6 +18,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
@ -213,6 +214,15 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("uncover", "Uncover",
|
||||
flagSet.BoolVarP(&options.Uncover, "uncover", "uc", false, "enable uncover engine"),
|
||||
flagSet.StringSliceVarP(&options.UncoverQuery, "uncover-query", "uq", []string{}, "uncover search query", goflags.FileStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.UncoverEngine, "uncover-engine", "ue", []string{}, fmt.Sprintf("uncover search engine (%s) (default shodan)", uncover.GetUncoverSupportedAgents()), goflags.FileStringSliceOptions),
|
||||
flagSet.StringVarP(&options.UncoverField, "uncover-field", "uf", "ip:port", "uncover fields to return (ip,port,host)"),
|
||||
flagSet.IntVarP(&options.UncoverLimit, "uncover-limit", "ul", 100, "uncover results to return"),
|
||||
flagSet.IntVarP(&options.UncoverDelay, "uncover-delay", "ucd", 1, "delay between uncover query requests in seconds (0 to disable)"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("rate-limit", "Rate-Limit",
|
||||
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
|
||||
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
|
||||
|
||||
29
v2/go.mod
29
v2/go.mod
@ -62,6 +62,10 @@ require (
|
||||
github.com/DataDog/gostackparse v0.6.0
|
||||
github.com/antchfx/xmlquery v1.3.12
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.11
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.38
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.2
|
||||
github.com/docker/go-units v0.5.0
|
||||
github.com/fatih/structs v1.1.0
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
@ -74,9 +78,10 @@ require (
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2
|
||||
github.com/projectdiscovery/goflags v0.1.3
|
||||
github.com/projectdiscovery/nvd v1.0.9
|
||||
github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa
|
||||
github.com/projectdiscovery/ratelimit v0.0.1
|
||||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
|
||||
github.com/projectdiscovery/tlsx v0.0.9
|
||||
github.com/projectdiscovery/uncover v0.0.9
|
||||
github.com/projectdiscovery/utils v0.0.2
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.67
|
||||
github.com/stretchr/testify v1.8.1
|
||||
@ -84,6 +89,26 @@ require (
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.2 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 // indirect
|
||||
github.com/projectdiscovery/sliceutil v0.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
|
||||
@ -199,6 +224,7 @@ require (
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
|
||||
@ -210,7 +236,6 @@ require (
|
||||
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect; indirectdev
|
||||
github.com/projectdiscovery/fileutil v0.0.3 // indirect
|
||||
github.com/projectdiscovery/iputil v0.0.2 // indirect
|
||||
github.com/projectdiscovery/sliceutil v0.0.1 // indirect
|
||||
github.com/sergi/go-diff v1.1.0 // indirect
|
||||
github.com/src-d/gcfg v1.4.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.0 // indirect
|
||||
|
||||
48
v2/go.sum
48
v2/go.sum
@ -116,6 +116,44 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W
|
||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.44.134 h1:TzFxjVHPPsibtkD7y6KHI4V00rEKg4yzNlMNGy2ZHeg=
|
||||
github.com/aws/aws-sdk-go v1.44.134/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 h1:RKci2D7tMwpvGpDNZnGQw9wk6v7o/xSwFcUAuNPoB8k=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9/go.mod h1:vCmV1q1VK8eoQJ5+aYE7PkK1K6v41qJ5pJdK3ggCDvg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.11 h1:9JQUKwRN8oUqeOFIrNaH6RSPmmcNk1+bQrDka/f/bPc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.11/go.mod h1:cw6HIEr0FaZQfcoyRWYZpMfv4qAH19hZFZ5mglwWo3g=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.24 h1:yz4fhoMfgwymG0rU6q5eCydFhYNQxk9yrNjMA7L7xmg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.24/go.mod h1:prZpUfBu1KZLBLVX482Sq4DpDXGugAre08TPEc21GUg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.38 h1:FAncZrGqy2l6JLZTP8fDMrF+BT98kCUSJZJ24oXkoFU=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.38/go.mod h1:8+dzbGcWsHbjeMQ/sGlLTtwykuajgdwkJcND5mIZpto=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 h1:2EXB7dtGwRYIN3XQ9qwIW504DVbKIw3r89xQnonGdsQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16/go.mod h1:XH+3h395e3WVdd6T2Z3mPxuI+x/HVtdqVOREkTiyubs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 h1:dpiPHgmFstgkLG07KaYAewvuptq5kvo52xn7tVSrtrQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10/go.mod h1:9cBNUHI2aW4ho0A5T87O294iPDuuUOSIEDjnd1Lq/z0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20 h1:KSvtm1+fPXE0swe9GPjc6msyrdTT0LB/BP8eLugL1FI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20/go.mod h1:Mp4XI/CkWGD79AQxZ5lIFlgvC0A+gl+4BmyG1F+SfNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 h1:piDBAaWkaxkkVV3xJJbTehXCZRXYs49kvpi/LG6LR2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19/go.mod h1:BmQWRVkLTmyNzYPFAZgon53qKLWBNSvonugD1MrSWUs=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.2 h1:l29X5biLks99HzZzQgC78plJpwiMv/pGNhmaTM2z62A=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.2/go.mod h1:/NHbqPRiwxSPVOB2Xr+StDEH+GWV/64WwnUjv4KYzV0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.2 h1:tpwEMRdMf2UsplengAOnmSIRdvAxf75oUFR+blBr92I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.2/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4=
|
||||
github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk=
|
||||
github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
@ -379,6 +417,8 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO
|
||||
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A=
|
||||
github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ=
|
||||
@ -608,6 +648,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h
|
||||
github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ=
|
||||
github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg=
|
||||
github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ=
|
||||
github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 h1:m1pgJisawU7zP9lKGktOEk6KNrNAR7e4Q07Kt3ox0NM=
|
||||
github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI=
|
||||
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
|
||||
github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M=
|
||||
github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q=
|
||||
@ -642,8 +684,8 @@ github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d61
|
||||
github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
|
||||
github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE=
|
||||
github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
|
||||
github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa h1:XTRcow+zyZUYxOlLsSp1AdM4Jl+i/OpPbOZPsfU/IAI=
|
||||
github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw=
|
||||
github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8=
|
||||
github.com/projectdiscovery/ratelimit v0.0.1/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw=
|
||||
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
|
||||
github.com/projectdiscovery/rawhttp v0.1.2 h1:fCK42+qc5qYR4Dj/BVoukQ182h4n/w0dRcGVO92T7eI=
|
||||
github.com/projectdiscovery/rawhttp v0.1.2/go.mod h1:Q5PDAmKzjAjweEp0CQr9301nyxCOkzA9ImK6qLjgk+8=
|
||||
@ -668,6 +710,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA
|
||||
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
|
||||
github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU=
|
||||
github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI=
|
||||
github.com/projectdiscovery/uncover v0.0.9 h1:cjdL98jZEx10+I/14o6QFsCX6FOucggtl8G5qD7fa10=
|
||||
github.com/projectdiscovery/uncover v0.0.9/go.mod h1:+ZJNjDcznffXM03Ijt3AVjJslCJSvgnR7dRV7ulFdac=
|
||||
github.com/projectdiscovery/utils v0.0.2 h1:lif4OYBqd8jCf0glRBfSs2lT4nMtNjHjeysRw8HIW8M=
|
||||
github.com/projectdiscovery/utils v0.0.2/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ=
|
||||
|
||||
@ -96,6 +96,13 @@ func ParseOptions(options *types.Options) {
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
|
||||
}
|
||||
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validateOptions validates the configuration options passed
|
||||
|
||||
@ -41,6 +41,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||
@ -255,9 +256,9 @@ func New(options *types.Options) (*Runner, error) {
|
||||
}
|
||||
|
||||
if options.RateLimitMinute > 0 {
|
||||
runner.ratelimiter = ratelimit.New(context.Background(), options.RateLimitMinute, time.Minute)
|
||||
runner.ratelimiter = ratelimit.New(context.Background(), int64(options.RateLimitMinute), time.Minute)
|
||||
} else if options.RateLimit > 0 {
|
||||
runner.ratelimiter = ratelimit.New(context.Background(), options.RateLimit, time.Second)
|
||||
runner.ratelimiter = ratelimit.New(context.Background(), int64(options.RateLimit), time.Second)
|
||||
} else {
|
||||
runner.ratelimiter = ratelimit.NewUnlimited(context.Background())
|
||||
}
|
||||
@ -412,6 +413,13 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
store.Load()
|
||||
|
||||
// add the hosts from the metadata queries of loaded templates into input provider
|
||||
if r.options.Uncover && len(r.options.UncoverQuery) == 0 {
|
||||
ret := uncover.GetUncoverTargetsFromMetadata(store.Templates(), r.options.UncoverDelay, r.options.UncoverLimit, r.options.UncoverField)
|
||||
for host := range ret {
|
||||
r.hmapInputProvider.Set(host)
|
||||
}
|
||||
}
|
||||
// list all templates
|
||||
if r.options.TemplateList {
|
||||
r.listAvailableStoreTemplates(store)
|
||||
|
||||
@ -31,6 +31,8 @@ type InputProvider interface {
|
||||
// Scan iterates the input and each found item is passed to the
|
||||
// callback consumer.
|
||||
Scan(callback func(value *contextargs.MetaInput) bool)
|
||||
// Set adds item to input provider
|
||||
Set(value string)
|
||||
}
|
||||
|
||||
// New returns a new Engine instance
|
||||
|
||||
@ -19,6 +19,7 @@ import (
|
||||
asn "github.com/projectdiscovery/mapcidr/asn"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
@ -90,7 +91,7 @@ func (i *Input) initializeInputSources(options *types.Options) error {
|
||||
case asn.IsASN(target):
|
||||
i.expandASNInputValue(target)
|
||||
default:
|
||||
i.normalizeStoreInputValue(target)
|
||||
i.Set(target)
|
||||
}
|
||||
}
|
||||
|
||||
@ -109,6 +110,16 @@ func (i *Input) initializeInputSources(options *types.Options) error {
|
||||
|
||||
i.scanInputFromReader(input)
|
||||
}
|
||||
if options.Uncover && options.UncoverQuery != nil {
|
||||
gologger.Info().Msgf("Running uncover query against: %s", strings.Join(options.UncoverEngine, ","))
|
||||
ch, err := uncover.GetTargetsFromUncover(options.UncoverDelay, options.UncoverLimit, options.UncoverField, options.UncoverEngine, options.UncoverQuery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for c := range ch {
|
||||
i.Set(c)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -123,13 +134,13 @@ func (i *Input) scanInputFromReader(reader io.Reader) {
|
||||
case asn.IsASN(item):
|
||||
i.expandASNInputValue(item)
|
||||
default:
|
||||
i.normalizeStoreInputValue(item)
|
||||
i.Set(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeStoreInputValue normalizes and stores passed input values
|
||||
func (i *Input) normalizeStoreInputValue(value string) {
|
||||
// Set normalizes and stores passed input values
|
||||
func (i *Input) Set(value string) {
|
||||
URL := strings.TrimSpace(value)
|
||||
if URL == "" {
|
||||
return
|
||||
|
||||
@ -87,7 +87,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
input.normalizeStoreInputValue(tt.hostname)
|
||||
input.Set(tt.hostname)
|
||||
// scan
|
||||
got := []string{}
|
||||
input.hostMap.Scan(func(k, v []byte) error {
|
||||
|
||||
@ -19,3 +19,8 @@ func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set adds item to input provider
|
||||
func (s *SimpleInputProvider) Set(value string) {
|
||||
s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value})
|
||||
}
|
||||
|
||||
227
v2/pkg/protocols/common/uncover/uncover.go
Normal file
227
v2/pkg/protocols/common/uncover/uncover.go
Normal file
@ -0,0 +1,227 @@
|
||||
package uncover
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
ucRunner "github.com/projectdiscovery/uncover/runner"
|
||||
"github.com/projectdiscovery/uncover/uncover"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/censys"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/fofa"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/hunter"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/quake"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/shodan"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/shodanidb"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/zoomeye"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
)
|
||||
|
||||
const maxConcurrentAgents = 50
|
||||
|
||||
func GetUncoverSupportedAgents() string {
|
||||
uncoverSupportedAgents := []string{"shodan", "shodan-idb", "fofa", "censys", "quake", "hunter", "zoomeye"}
|
||||
return strings.Join(uncoverSupportedAgents, ",")
|
||||
}
|
||||
|
||||
func GetTargetsFromUncover(delay, limit int, field string, engine, query []string) (chan string, error) {
|
||||
uncoverOptions := &ucRunner.Options{
|
||||
Provider: &ucRunner.Provider{},
|
||||
Delay: delay,
|
||||
Limit: limit,
|
||||
Query: query,
|
||||
Engine: engine,
|
||||
}
|
||||
for _, eng := range engine {
|
||||
err := loadKeys(eng, uncoverOptions)
|
||||
if err != nil {
|
||||
gologger.Error().Label("WRN").Msgf(err.Error())
|
||||
continue
|
||||
}
|
||||
}
|
||||
return getTargets(uncoverOptions, field)
|
||||
}
|
||||
|
||||
func GetUncoverTargetsFromMetadata(templates []*templates.Template, delay, limit int, field string) chan string {
|
||||
ret := make(chan string)
|
||||
var uqMap = make(map[string][]string)
|
||||
var eng, query string
|
||||
for _, template := range templates {
|
||||
for k, v := range template.Info.Metadata {
|
||||
switch k {
|
||||
case "shodan-query":
|
||||
eng = "shodan"
|
||||
case "fofa-query":
|
||||
eng = "fofa"
|
||||
case "censys-query":
|
||||
eng = "censys"
|
||||
case "quake-query":
|
||||
eng = "quake"
|
||||
case "hunter-query":
|
||||
eng = "hunter"
|
||||
case "zoomeye-query":
|
||||
eng = "zoomeye"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
query = fmt.Sprintf("%v", v)
|
||||
uqMap[eng] = append(uqMap[eng], query)
|
||||
}
|
||||
}
|
||||
keys := mapsutil.GetKeys(uqMap)
|
||||
gologger.Info().Msgf("Running uncover query against: %s", strings.Join(keys, ","))
|
||||
var wg sync.WaitGroup
|
||||
go func() {
|
||||
for k, v := range uqMap {
|
||||
wg.Add(1)
|
||||
go func(engine, query []string) {
|
||||
ch, _ := GetTargetsFromUncover(delay, limit, field, engine, query)
|
||||
for c := range ch {
|
||||
ret <- c
|
||||
}
|
||||
wg.Done()
|
||||
}([]string{k}, v)
|
||||
}
|
||||
wg.Wait()
|
||||
close(ret)
|
||||
}()
|
||||
return ret
|
||||
}
|
||||
|
||||
func getTargets(uncoverOptions *ucRunner.Options, field string) (chan string, error) {
|
||||
var rateLimiter *ratelimit.Limiter
|
||||
// create rateLimiter for uncover delay
|
||||
if uncoverOptions.Delay > 0 {
|
||||
rateLimiter = ratelimit.New(context.Background(), 1, time.Duration(uncoverOptions.Delay))
|
||||
} else {
|
||||
rateLimiter = ratelimit.NewUnlimited(context.Background())
|
||||
}
|
||||
var agents []uncover.Agent
|
||||
// declare clients
|
||||
for _, engine := range uncoverOptions.Engine {
|
||||
var (
|
||||
agent uncover.Agent
|
||||
err error
|
||||
)
|
||||
switch engine {
|
||||
case "shodan":
|
||||
agent, err = shodan.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "censys":
|
||||
agent, err = censys.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "fofa":
|
||||
agent, err = fofa.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "shodan-idb":
|
||||
agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "quake":
|
||||
agent, err = quake.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "hunter":
|
||||
agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "zoomeye":
|
||||
agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
default:
|
||||
err = errors.Errorf("%s unknown uncover agent type", engine)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
agents = append(agents, agent)
|
||||
}
|
||||
// enumerate
|
||||
swg := sizedwaitgroup.New(maxConcurrentAgents)
|
||||
ret := make(chan string)
|
||||
go func() {
|
||||
for _, q := range uncoverOptions.Query {
|
||||
uncoverQuery := &uncover.Query{
|
||||
Query: q,
|
||||
Limit: uncoverOptions.Limit,
|
||||
}
|
||||
for _, agent := range agents {
|
||||
swg.Add()
|
||||
go func(agent uncover.Agent, uncoverQuery *uncover.Query) {
|
||||
defer swg.Done()
|
||||
keys := uncoverOptions.Provider.GetKeys()
|
||||
session, err := uncover.NewSession(&keys, uncoverOptions.Retries, uncoverOptions.Timeout)
|
||||
if err != nil {
|
||||
gologger.Error().Label(agent.Name()).Msgf("couldn't create uncover new session: %s", err)
|
||||
}
|
||||
ch, err := agent.Query(session, uncoverQuery)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("%s", err)
|
||||
return
|
||||
}
|
||||
for result := range ch {
|
||||
replacer := strings.NewReplacer(
|
||||
"ip", result.IP,
|
||||
"host", result.Host,
|
||||
"port", fmt.Sprint(result.Port),
|
||||
)
|
||||
ret <- replacer.Replace(field)
|
||||
}
|
||||
}(agent, uncoverQuery)
|
||||
}
|
||||
}
|
||||
swg.Wait()
|
||||
close(ret)
|
||||
}()
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func loadKeys(engine string, options *ucRunner.Options) error {
|
||||
switch engine {
|
||||
case "fofa":
|
||||
if email, exists := os.LookupEnv("FOFA_EMAIL"); exists {
|
||||
if key, exists := os.LookupEnv("FOFA_KEY"); exists {
|
||||
options.Provider.Fofa = append(options.Provider.Fofa, fmt.Sprintf("%s:%s", email, key))
|
||||
} else {
|
||||
return errors.New("missing FOFA_KEY env variable")
|
||||
}
|
||||
} else {
|
||||
return errors.Errorf("FOFA_EMAIL & FOFA_KEY env variables are not configured")
|
||||
}
|
||||
case "shodan":
|
||||
if key, exists := os.LookupEnv("SHODAN_API_KEY"); exists {
|
||||
options.Provider.Shodan = append(options.Provider.Shodan, key)
|
||||
} else {
|
||||
return errors.Errorf("SHODAN_API_KEY env variable is not configured")
|
||||
}
|
||||
case "censys":
|
||||
if id, exists := os.LookupEnv("CENSYS_API_ID"); exists {
|
||||
if secret, exists := os.LookupEnv("CENSYS_API_SECRET"); exists {
|
||||
options.Provider.Censys = append(options.Provider.Censys, fmt.Sprintf("%s:%s", id, secret))
|
||||
} else {
|
||||
return errors.New("missing CENSYS_API_SECRET env variable")
|
||||
}
|
||||
} else {
|
||||
return errors.Errorf("CENSYS_API_ID & CENSYS_API_SECRET env variable is not configured")
|
||||
}
|
||||
case "hunter":
|
||||
if key, exists := os.LookupEnv("HUNTER_API_KEY"); exists {
|
||||
options.Provider.Hunter = append(options.Provider.Hunter, key)
|
||||
} else {
|
||||
return errors.Errorf("HUNTER_API_KEY env variable is not configured")
|
||||
}
|
||||
case "zoomeye":
|
||||
if key, exists := os.LookupEnv("ZOOMEYE_API_KEY"); exists {
|
||||
options.Provider.ZoomEye = append(options.Provider.ZoomEye, key)
|
||||
} else {
|
||||
return errors.Errorf("ZOOMEYE_API_KEY env variable is not configured")
|
||||
}
|
||||
case "quake":
|
||||
if key, exists := os.LookupEnv("QUAKE_TOKEN"); exists {
|
||||
options.Provider.Quake = append(options.Provider.Quake, key)
|
||||
} else {
|
||||
return errors.Errorf("QUAKE_TOKEN env variable is not configured")
|
||||
}
|
||||
default:
|
||||
return errors.Errorf("unknown uncover agent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -91,7 +91,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
|
||||
IssuesClient: nil,
|
||||
Browser: nil,
|
||||
Catalog: disk.NewCatalog(options.TemplatesDirectory),
|
||||
RateLimiter: ratelimit.New(context.Background(), options.RateLimit, time.Second),
|
||||
RateLimiter: ratelimit.New(context.Background(), int64(options.RateLimit), time.Second),
|
||||
}
|
||||
return executerOpts
|
||||
}
|
||||
|
||||
@ -272,6 +272,18 @@ type Options struct {
|
||||
IncludeConditions goflags.StringSlice
|
||||
// Custom Config Directory
|
||||
CustomConfigDir string
|
||||
// Enable uncover egine
|
||||
Uncover bool
|
||||
// Uncover search query
|
||||
UncoverQuery goflags.StringSlice
|
||||
// Uncover search engine
|
||||
UncoverEngine goflags.StringSlice
|
||||
// Uncover search field
|
||||
UncoverField string
|
||||
// Uncover search limit
|
||||
UncoverLimit int
|
||||
// Uncover search delay
|
||||
UncoverDelay int
|
||||
// ConfigPath contains the config path (used by healthcheck)
|
||||
ConfigPath string
|
||||
// ScanAllIPs associated to a dns record
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user