From b7af565edfda948735938ff736d13b34d7a1f63b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 25 Feb 2021 13:14:42 +0530 Subject: [PATCH] Misc changes --- v2/cmd/nuclei/main.go | 1 + v2/go.mod | 3 ++- v2/go.sum | 6 ++++-- v2/pkg/protocols/headless/engine/http_client.go | 4 +++- v2/pkg/protocols/http/http.go | 6 ++++++ v2/pkg/protocols/http/httpclientpool/clientpool.go | 4 +++- v2/pkg/protocols/http/request.go | 5 ----- v2/pkg/protocols/network/networkclientpool/clientpool.go | 4 +++- v2/pkg/types/types.go | 2 ++ 9 files changed, 24 insertions(+), 11 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 3db0edda5..6fb10683b 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -87,6 +87,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen") set.BoolVarP(&options.Workflows, "w", "workflows", false, "Only run workflow templates with nuclei") set.IntVar(&options.StatsInterval, "stats-interval", 5, "Number of seconds between each stats line") + set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") _ = set.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 88070d24f..0436910a6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,6 +12,7 @@ require ( github.com/golang/protobuf v1.4.3 // indirect github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v32 v32.1.0 + github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/json-iterator/go v1.1.10 @@ -26,7 +27,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.7 github.com/projectdiscovery/collaborator v0.0.2 - github.com/projectdiscovery/fastdialer v0.0.7 + github.com/projectdiscovery/fastdialer v0.0.8 github.com/projectdiscovery/goflags v0.0.2 github.com/projectdiscovery/gologger v1.1.3 github.com/projectdiscovery/hmap v0.0.1 diff --git a/v2/go.sum b/v2/go.sum index d3e93f68a..138eddafd 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -135,6 +135,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -196,8 +198,8 @@ github.com/projectdiscovery/clistats v0.0.7 h1:Q/erjrk2p3BIQq1RaHVtBpgboghNz0u1/ github.com/projectdiscovery/clistats v0.0.7/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s= github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= -github.com/projectdiscovery/fastdialer v0.0.7 h1:OPFwvq4t9vulPRs6rJddH2R2RFCHIj+ZjEvIrq3HTzA= -github.com/projectdiscovery/fastdialer v0.0.7/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= +github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= +github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= github.com/projectdiscovery/goflags v0.0.2 h1:4vB5+mA41xgW6V1y4YD1A+iI8Kq68iTTny50XuSYKdo= github.com/projectdiscovery/goflags v0.0.2/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU= diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index d1e1ad55a..8f69b71dd 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -13,7 +13,9 @@ import ( // newhttpClient creates a new http client for headless communication with a timeout func newhttpClient(options *types.Options) (*http.Client, error) { opts := fastdialer.DefaultOptions - opts.EnableFallback = true + if options.SystemResolvers { + opts.EnableFallback = true + } if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 9930038dc..94d02412c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -3,6 +3,7 @@ package http import ( "strings" + "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -78,6 +79,11 @@ func (r *Request) GetID() string { // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { + // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given + if r.options.Options.RandomAgent { + r.customHeaders["User-Agent"] = uarand.GetRandom() + } + client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{ Threads: r.Threads, MaxRedirects: r.MaxRedirects, diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 07bfde10b..df27e88c2 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -98,10 +98,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if Dialer == nil { opts := fastdialer.DefaultOptions + if options.SystemResolvers { + opts.EnableFallback = true + } if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } - opts.EnableFallback = true Dialer, err = fastdialer.NewDialer(opts) } if err != nil { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index f7b2c9a19..cd8149fbd 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -209,10 +208,6 @@ const drainReqSize = int64(8 * 1024) // executeRequest executes the actual generated request and returns error if occured func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if r.options.Options.RandomAgent { - r.customHeaders["User-Agent"] = uarand.GetRandom() - } r.setCustomHeaders(request) var ( diff --git a/v2/pkg/protocols/network/networkclientpool/clientpool.go b/v2/pkg/protocols/network/networkclientpool/clientpool.go index fff17bc44..4113d3a37 100644 --- a/v2/pkg/protocols/network/networkclientpool/clientpool.go +++ b/v2/pkg/protocols/network/networkclientpool/clientpool.go @@ -17,7 +17,9 @@ func Init(options *types.Options) error { return nil } opts := fastdialer.DefaultOptions - opts.EnableFallback = true + if options.SystemResolvers { + opts.EnableFallback = true + } if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 34a47bb3f..40bdbcda9 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -102,6 +102,8 @@ type Options struct { Workflows bool // StatsInterval is the number of seconds to display stats after StatsInterval int + // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. + SystemResolvers bool InternalResolversList []string // normalized from resolvers flag as well as file provided. }