diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 72e57a68e..fd3a68c63 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -188,6 +188,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"), flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), + flagSet.BoolVar(&options.ProxyInternal, "proxy-internal", false, "Proxy all internal requests"), flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), diff --git a/v2/go.mod b/v2/go.mod index 08eff08ea..08c914bbe 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -33,12 +33,12 @@ require ( github.com/projectdiscovery/goflags v0.0.8-0.20220223132346-fcffa47ead36 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v1.0.1-0.20220131074403-ca8bb8f87cd0 + github.com/projectdiscovery/interactsh v1.0.2-0.20220309105842-7aa59db95941 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.8-0.20220225134552-b60c4c126e5b github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 + github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 @@ -88,10 +88,11 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.2.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/c4milo/unpackit v0.1.0 // indirect - github.com/caddyserver/certmagic v0.15.2 // indirect + github.com/caddyserver/certmagic v0.15.3 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -117,7 +118,7 @@ require ( github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.14.1 // indirect + github.com/klauspost/compress v1.14.4 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -151,7 +152,7 @@ require ( github.com/zclconf/go-cty v1.10.0 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.uber.org/zap v1.20.0 // indirect + go.uber.org/zap v1.21.0 // indirect goftp.io/server/v2 v2.0.0 // indirect golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect golang.org/x/mod v0.4.2 // indirect diff --git a/v2/go.sum b/v2/go.sum index 03ef03a85..2855de54e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -82,6 +82,8 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.43.14 h1:ZFvtGVVB5yHskkE/dilXsZR1eLS3K1ibBrWBVYyxrbg= github.com/aws/aws-sdk-go v1.43.14/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= @@ -100,8 +102,8 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= -github.com/caddyserver/certmagic v0.15.2 h1:OMTakTsLM1ZfzMDjwvYprfUgFzpVPh3u87oxMPwmeBc= -github.com/caddyserver/certmagic v0.15.2/go.mod h1:qhkAOthf72ufAcp3Y5jF2RaGE96oip3UbEQRIzwe3/8= +github.com/caddyserver/certmagic v0.15.3 h1:ScY3KVV1eMIUfW74i20kDnD4eWL8T0rG6S6Wnc6nc9U= +github.com/caddyserver/certmagic v0.15.3/go.mod h1:qhkAOthf72ufAcp3Y5jF2RaGE96oip3UbEQRIzwe3/8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -315,8 +317,8 @@ github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.14.1 h1:hLQYb23E8/fO+1u53d02A97a8UnsddcvYzq4ERRU4ds= -github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= @@ -364,7 +366,6 @@ github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= @@ -423,10 +424,6 @@ github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/cryptoutil v0.0.0-20220124150510-1f21e1ec3143 h1:ulWFeH179xgDUfNQT/LyimW1znNlivsqv2d/lNFZU30= -github.com/projectdiscovery/cryptoutil v0.0.0-20220124150510-1f21e1ec3143/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4= -github.com/projectdiscovery/cryptoutil v0.0.0-20220308134124-226941faf57b h1:BTOdZSqMTDGa+RE2d0qyE9SGoj9E1+/gTp6dTHHMA74= -github.com/projectdiscovery/cryptoutil v0.0.0-20220308134124-226941faf57b/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/cryptoutil v1.0.0 h1:5rQfnWDthJ5ZFcqze+rmT1N7l1HJQ6EB26MrjaYB7I0= github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= @@ -452,8 +449,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1: github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= -github.com/projectdiscovery/interactsh v1.0.1-0.20220131074403-ca8bb8f87cd0 h1:Olf2RG9sLqZF157gC664G6A3DU0Fta6VD/OWiNP3LbI= -github.com/projectdiscovery/interactsh v1.0.1-0.20220131074403-ca8bb8f87cd0/go.mod h1:UW8wdok5mrDOXzcHxRjUCCDIScc/3hCpw8QjVDeXHEE= +github.com/projectdiscovery/interactsh v1.0.2-0.20220309105842-7aa59db95941 h1:cFSdFArGFDvrmorGOxc35jPTCPMLb9b+MccLmwQKzA8= +github.com/projectdiscovery/interactsh v1.0.2-0.20220309105842-7aa59db95941/go.mod h1:mMVHcw/S1HpGgvANGEc0syQTPAYzHJtepOv3xrnD+OM= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= @@ -485,8 +482,8 @@ github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d/go.mod github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk= -github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4fvUw6b4sS4GoWbHr60mJo3dI//4mGt3BuLx8Sz9aNw= +github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= @@ -639,8 +636,8 @@ go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6m go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= goftp.io/server/v2 v2.0.0 h1:FF8JKXXKDxAeO1uXEZz7G+IZwCDhl19dpVIlDtp3QAg= goftp.io/server/v2 v2.0.0/go.mod h1:7+H/EIq7tXdfo1Muu5p+l3oQ6rYkDZ8lY7IM5d5kVdQ= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index 6aca4be6a..fba343ce5 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -81,7 +81,7 @@ func processProxyList(options *types.Options) error { func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { if err := testProxyConnection(proxyURL, options.Timeout); err == nil { if types.ProxyURL == "" && types.ProxySocksURL == "" { - assignProxyURL(proxyURL, options) + assignProxyURL(proxyURL) done <- true } } @@ -97,8 +97,7 @@ func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { return nil } -func assignProxyURL(proxyURL url.URL, options *types.Options) { - os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) +func assignProxyURL(proxyURL url.URL) { if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { types.ProxyURL = proxyURL.String() types.ProxySocksURL = "" diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 354514e44..12fa8e267 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -32,6 +32,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" @@ -40,6 +41,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" yamlwrapper "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml" + "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/stringsutil" ) @@ -97,10 +99,23 @@ func New(options *types.Options) (*Runner, error) { runner.catalog = catalog.New(runner.options.TemplatesDirectory) + var httpclient *retryablehttp.Client + if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { + var err error + httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}) + if err != nil { + return nil, err + } + } + reportingOptions, err := createReportingOptions(options) if err != nil { return nil, err } + if reportingOptions != nil && httpclient != nil { + reportingOptions.HttpClient = httpclient + } + if reportingOptions != nil { client, err := reporting.New(reportingOptions, options.ReportingDB) if err != nil { @@ -195,13 +210,16 @@ func New(options *types.Options) (*Runner, error) { opts.Authorization = options.InteractshToken opts.CacheSize = int64(options.InteractionsCacheSize) opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second - opts.ColldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second + opts.CooldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second opts.NoInteractsh = runner.options.NoInteractsh opts.StopAtFirstMatch = runner.options.StopAtFirstMatch opts.Debug = runner.options.Debug opts.DebugRequest = runner.options.DebugRequests opts.DebugResponse = runner.options.DebugResponse + if httpclient != nil { + opts.HTTPClient = httpclient + } interactshClient, err := interactsh.New(opts) if err != nil { gologger.Error().Msgf("Could not create interactsh client: %s", err) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index d0118e5f1..b3e2d2134 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -23,6 +23,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/retryablehttp-go" ) // Client is a wrapped client for interactsh server. @@ -66,7 +67,7 @@ type Options struct { Eviction time.Duration // CooldownPeriod is additional time to wait for interactions after closing // of the poller. - ColldownPeriod time.Duration + CooldownPeriod time.Duration // PollDuration is the time to wait before each poll to the server for interactions. PollDuration time.Duration // Output is the output writer for nuclei @@ -87,6 +88,7 @@ type Options struct { NoColor bool StopAtFirstMatch bool + HTTPClient *retryablehttp.Client } const defaultMaxInteractionsCount = 5000 @@ -110,7 +112,7 @@ func New(options *Options) (*Client, error) { options: options, requests: cache, pollDuration: options.PollDuration, - cooldownDuration: options.ColldownPeriod, + cooldownDuration: options.CooldownPeriod, } return interactClient, nil } @@ -121,7 +123,7 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre ServerURL: client.DefaultOptions.ServerURL, CacheSize: 5000, Eviction: 60 * time.Second, - ColldownPeriod: 5 * time.Second, + CooldownPeriod: 5 * time.Second, PollDuration: 5 * time.Second, Output: output, IssuesClient: reporting, @@ -140,6 +142,7 @@ func (c *Client) firstTimeInitializeClient() error { Token: c.options.Authorization, PersistentSession: false, DisableHTTPFallback: c.options.DisableHttpFallback, + HTTPClient: c.options.HTTPClient, }) if err != nil { return errors.Wrap(err, "could not create client") diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index dd79b67a2..8354f8c4b 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -234,7 +234,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { ServerURL: options.InteractshURL, CacheSize: int64(options.InteractionsCacheSize), Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, + CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, DisableHttpFallback: true, }) diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 3737856ba..96adac1b2 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/retryablehttp-go" ) // Options contains necessary options required for elasticsearch communication @@ -32,7 +33,8 @@ type Options struct { // Password is the password for elasticsearch instance Password string `yaml:"password" validate:"required"` // IndexName is the name of the elasticsearch index - IndexName string `yaml:"index-name" validate:"required"` + IndexName string `yaml:"index-name" validate:"required"` + HttpClient *retryablehttp.Client } type data struct { @@ -51,15 +53,21 @@ type Exporter struct { func New(option *Options) (*Exporter, error) { var ei *Exporter - client := &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 10, - MaxIdleConnsPerHost: 10, - DialContext: protocolstate.Dialer.Dial, - TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, - }, + var client *http.Client + if option.HttpClient != nil { + client = option.HttpClient.HTTPClient + } else { + client = &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 10, + MaxIdleConnsPerHost: 10, + DialContext: protocolstate.Dialer.Dial, + TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, + }, + } } + // preparing url for elasticsearch scheme := "http://" if option.SSL { diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index f51470e1c..d30448e86 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/retryablehttp-go" ) // Options is a configuration file for nuclei reporting module @@ -36,6 +37,7 @@ type Options struct { SarifExporter *sarif.Options `yaml:"sarif"` // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module ElasticsearchExporter *es.Options `yaml:"elasticsearch"` + HttpClient *retryablehttp.Client } // Filter filters the received event and decides whether to perform @@ -108,6 +110,7 @@ type Client struct { func New(options *Options, db string) (*Client, error) { client := &Client{options: options} if options.GitHub != nil { + options.GitHub.HttpClient = options.HttpClient tracker, err := github.New(options.GitHub) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") @@ -115,6 +118,7 @@ func New(options *Options, db string) (*Client, error) { client.trackers = append(client.trackers, tracker) } if options.GitLab != nil { + options.GitLab.HttpClient = options.HttpClient tracker, err := gitlab.New(options.GitLab) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") @@ -122,6 +126,7 @@ func New(options *Options, db string) (*Client, error) { client.trackers = append(client.trackers, tracker) } if options.Jira != nil { + options.Jira.HttpClient = options.HttpClient tracker, err := jira.New(options.Jira) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") @@ -143,6 +148,7 @@ func New(options *Options, db string) (*Client, error) { client.exporters = append(client.exporters, exporter) } if options.ElasticsearchExporter != nil { + options.ElasticsearchExporter.HttpClient = options.HttpClient exporter, err := es.New(options.ElasticsearchExporter) if err != nil { return nil, errors.Wrap(err, "could not create exporting client") diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 352c5183f..41773020e 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -3,6 +3,7 @@ package github import ( "context" "fmt" + "net/http" "net/url" "strings" @@ -13,6 +14,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" ) // Integration is a client for an issue tracker integration @@ -38,6 +41,7 @@ type Options struct { // SeverityAsLabel (optional) sends the severity as the label of the created // issue. SeverityAsLabel bool `yaml:"severity-as-label"` + HttpClient *retryablehttp.Client } // New creates a new issue tracker integration client based on options. @@ -48,6 +52,14 @@ func New(options *Options) (*Integration, error) { ) tc := oauth2.NewClient(ctx, ts) + // patch transport to support proxy - only http + // TODO: investigate if it's possible to reuse existing retryablehttp + if types.ProxyURL != "" { + if proxyURL, err := url.Parse(types.ProxyURL); err == nil { + tc.Transport.(*http.Transport).Proxy = http.ProxyURL(proxyURL) + } + } + client := github.NewClient(tc) if options.BaseURL != "" { parsed, err := url.Parse(options.BaseURL) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 8e2da916c..0758141d6 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/projectdiscovery/retryablehttp-go" ) // Integration is a client for an issue tracker integration @@ -31,6 +32,7 @@ type Options struct { // SeverityAsLabel (optional) sends the severity as the label of the created // issue. SeverityAsLabel bool `yaml:"severity-as-label"` + HttpClient *retryablehttp.Client } // New creates a new issue tracker integration client based on options. @@ -39,6 +41,9 @@ func New(options *Options) (*Integration, error) { if options.BaseURL != "" { gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) } + if options.HttpClient != nil { + gitlabOpts = append(gitlabOpts, gitlab.WithHTTPClient(options.HttpClient.HTTPClient)) + } git, err := gitlab.NewClient(options.Token, gitlabOpts...) if err != nil { return nil, err diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 3c057e3b6..fb2203278 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" ) // Integration is a client for an issue tracker integration @@ -42,6 +43,7 @@ type Options struct { // SeverityAsLabel (optional) sends the severity as the label of the created // issue. SeverityAsLabel bool `yaml:"severity-as-label"` + HttpClient *retryablehttp.Client } // New creates a new issue tracker integration client based on options. @@ -54,6 +56,9 @@ func New(options *Options) (*Integration, error) { Username: username, Password: options.Token, } + if options.HttpClient != nil { + tp.Transport = options.HttpClient.HTTPClient.Transport + } jiraClient, err := jira.NewClient(tp.Client(), options.URL) if err != nil { return nil, err diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go index a45b4eca6..faf9955ff 100644 --- a/v2/pkg/types/proxy.go +++ b/v2/pkg/types/proxy.go @@ -1,10 +1,9 @@ package types const ( - HTTP_PROXY_ENV = "HTTP_PROXY" - SOCKS5 = "socks5" - HTTP = "http" - HTTPS = "https" + SOCKS5 = "socks5" + HTTP = "http" + HTTPS = "https" ) var ( diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index f461518f2..8ecc78528 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -67,6 +67,8 @@ type Options struct { Resume string // Output is the file to write found results to. Output string + // ProxyInternal requests + ProxyInternal bool // List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) Proxy goflags.NormalizedOriginalStringSlice // TemplatesDirectory is the directory to use for storing templates