Merge branch 'dev' of https://github.com/projectdiscovery/nuclei into more-protocols

README.md

@@ -162,16 +162,16 @@ HEADLESS:
    -sc, -system-chrome  Use local installed chrome browser instead of nuclei installed
 
 DEBUG:
-   -debug                     show all requests and responses
-   -debug-req                 show all sent requests
-   -debug-resp                show all received responses
-   -proxy, -proxy-url string  URL of the HTTP proxy server
-   -proxy-socks-url string    URL of the SOCKS proxy server
-   -tlog, -trace-log string   file to write sent requests trace log
-   -version                   show nuclei version
-   -v, -verbose               show verbose output
-   -vv                        display extra verbose information
-   -tv, -templates-version    shows the version of the installed nuclei-templates
+   -debug                    show all requests and responses
+   -debug-req                show all sent requests
+   -debug-resp               show all received responses
+   -p, -proxy string[]       List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
+   -tlog, -trace-log string  file to write sent requests trace log
+   -elog, -error-log string  file to write sent requests error log
+   -version                  show nuclei version
+   -v, -verbose              show verbose output
+   -vv                       display templates loaded for scan
+   -tv, -templates-version   shows the version of the installed nuclei-templates
 
 UPDATE:
    -update                        update nuclei engine to the latest released version

README_CN.md

@@ -121,8 +121,7 @@ nuclei -h
 |templates-version|显示已安装的模板版本|nuclei -templates-version|
 |v|显示发送请求的详细信息|nuclei -v|
 |version|显示nuclei的版本号|nuclei -version|
-|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080|
-|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080|
+|proxy|输入代理地址|nuclei -proxy ./proxy.txt|
 |random-agent|使用随机的UA|nuclei -random-agent|
 |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”|
 

integration_tests/http/get-redirects-chain-headers.yaml

@@ -0,0 +1,23 @@
+id: basic-get-redirects-chain-headers
+
+info:
+  name: Basic GET Redirects Request With Chain header
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    redirects: true
+    max-redirects: 3
+    matchers-condition: and
+    matchers:
+      - type: word
+        part: header
+        words:
+          - "TestRedirectHeaderMatch"
+
+      - type: status
+        status:
+          - 302

v2/cmd/integration-test/http.go

@@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/self-contained.yaml":                     &httpRequestSelContained{},
 	"http/get-case-insensitive.yaml":               &httpGetCaseInsensitive{},
 	"http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
+	"http/get-redirects-chain-headers.yaml":        &httpGetRedirectsChainHeaders{},
 }
 
 type httpInteractshRequest struct{}
@@ -599,3 +600,31 @@ func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error {
 	}
 	return nil
 }
+
+type httpGetRedirectsChainHeaders struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		http.Redirect(w, r, "/redirected", http.StatusFound)
+	})
+	router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.Header().Set("Secret", "TestRedirectHeaderMatch")
+		http.Redirect(w, r, "/final", http.StatusFound)
+	})
+	router.GET("/final", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte("ok"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/integration-test.go

@@ -52,5 +52,5 @@ func main() {
 }
 
 func errIncorrectResultsCount(results []string) error {
-	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+	return fmt.Errorf("incorrect number of results \n\t%s", strings.Join(results, "\n\t"))
 }

v2/cmd/nuclei/main.go

@@ -143,11 +143,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
 		flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
 		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
-
-		/* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
-		   TODO should auto-set the HTTP_PROXY variable for the process? */
-		flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
-		flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
+		flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"),
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),
 		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),

v2/internal/runner/options.go

@@ -3,7 +3,6 @@ package runner
 import (
 	"bufio"
 	"errors"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) {
 
 	// Read the inputs and configure the logging
 	configureOutput(options)
-
 	// Show the user the banner
 	showBanner()
 
@@ -82,15 +80,10 @@ func validateOptions(options *types.Options) error {
 	if options.Verbose && options.Silent {
 		return errors.New("both verbose and silent mode specified")
 	}
-
-	if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil {
+	//loading the proxy server list from file or cli and test the connectivity
+	if err := loadProxyServers(options); err != nil {
 		return err
 	}
-
-	if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil {
-		return err
-	}
-
 	if options.Validate {
 		options.Headless = true // required for correct validation of headless templates
 		validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
@@ -107,19 +100,6 @@ func validateOptions(options *types.Options) error {
 	return nil
 }
 
-func validateProxyURL(proxyURL, message string) error {
-	if proxyURL != "" && !isValidURL(proxyURL) {
-		return errors.New(message)
-	}
-
-	return nil
-}
-
-func isValidURL(urlString string) bool {
-	_, err := url.Parse(urlString)
-	return err == nil
-}
-
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
@@ -165,7 +145,6 @@ func loadResolvers(options *types.Options) {
 
 func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
 	allGivenTemplatePaths := append(templatePaths, workflowPaths...)
-
 	for _, templatePath := range allGivenTemplatePaths {
 		if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
 			fileInfo, err := os.Stat(templatePath)

v2/internal/runner/proxy.go

@@ -0,0 +1,123 @@
+package runner
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/projectdiscovery/fileutil"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var proxyURLList []url.URL
+
+// loadProxyServers load list of proxy servers from file or comma seperated
+func loadProxyServers(options *types.Options) error {
+	if len(options.Proxy) == 0 {
+		return nil
+	}
+	for _, p := range options.Proxy {
+		if proxyURL, err := validateProxyURL(p); err == nil {
+			proxyURLList = append(proxyURLList, proxyURL)
+		} else if fileutil.FileExists(p) {
+			file, err := os.Open(p)
+			if err != nil {
+				return fmt.Errorf("could not open proxy file: %s", err)
+			}
+			defer file.Close()
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				proxy := scanner.Text()
+				if strings.TrimSpace(proxy) == "" {
+					continue
+				}
+				if proxyURL, err := validateProxyURL(proxy); err != nil {
+					return err
+				} else {
+					proxyURLList = append(proxyURLList, proxyURL)
+				}
+			}
+		} else {
+			return fmt.Errorf("invalid proxy file or URL provided for %s", p)
+		}
+	}
+	return processProxyList(options)
+}
+
+func processProxyList(options *types.Options) error {
+	if len(proxyURLList) == 0 {
+		return fmt.Errorf("could not find any valid proxy")
+	} else {
+		done := make(chan bool)
+		exitCounter := make(chan bool)
+		counter := 0
+		for _, url := range proxyURLList {
+			go runProxyConnectivity(url, options, done, exitCounter)
+		}
+		for {
+			select {
+			case <-done:
+				{
+					close(done)
+					return nil
+				}
+			case <-exitCounter:
+				{
+					if counter += 1; counter == len(proxyURLList) {
+						return errors.New("no reachable proxy found")
+					}
+				}
+			}
+		}
+	}
+}
+
+func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) {
+	if err := testProxyConnection(proxyURL, options.Timeout); err == nil {
+		if types.ProxyURL == "" && types.ProxySocksURL == "" {
+			assignProxyURL(proxyURL, options)
+			done <- true
+		}
+	}
+	exitCounter <- true
+}
+
+func testProxyConnection(proxyURL url.URL, timeoutDelay int) error {
+	timeout := time.Duration(timeoutDelay) * time.Second
+	_, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func assignProxyURL(proxyURL url.URL, options *types.Options) {
+	os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String())
+	if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS {
+		types.ProxyURL = proxyURL.String()
+		types.ProxySocksURL = ""
+		gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String())
+	} else if proxyURL.Scheme == types.SOCKS5 {
+		types.ProxyURL = ""
+		types.ProxySocksURL = proxyURL.String()
+		gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String())
+	}
+}
+
+func validateProxyURL(proxy string) (url.URL, error) {
+	if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) {
+		return *url, nil
+	}
+	return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)")
+}
+
+//isSupportedProtocol checks given protocols are supported
+func isSupportedProtocol(value string) bool {
+	return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5
+}

v2/pkg/protocols/dns/operators.go

@@ -14,7 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// Match matches a generic data response again a given matcher
+// Match matches a generic data response against a given matcher
 func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
 	item, ok := request.getMatchPart(matcher.Part, data)
 	if !ok {

v2/pkg/protocols/headless/engine/engine.go

@@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) {
 	} else {
 		chromeLauncher = chromeLauncher.Headless(true)
 	}
-	if options.ProxyURL != "" {
-		chromeLauncher = chromeLauncher.Proxy(options.ProxyURL)
+	if types.ProxyURL != "" {
+		chromeLauncher = chromeLauncher.Proxy(types.ProxyURL)
 	}
 	launcherURL, err := chromeLauncher.Launch()
 	if err != nil {
@@ -88,7 +88,12 @@ func New(options *types.Options) (*Browser, error) {
 	if customAgent == "" {
 		customAgent = uarand.GetRandom()
 	}
-	httpclient := newhttpClient(options)
+
+	httpclient, err := newhttpClient(options)
+	if err != nil {
+		return nil, err
+	}
+
 	engine := &Browser{
 		tempDir:     dataStore,
 		customAgent: customAgent,

v2/pkg/protocols/headless/engine/http_client.go

@@ -4,20 +4,21 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"golang.org/x/net/proxy"
 )
 
 // newhttpClient creates a new http client for headless communication with a timeout
-func newhttpClient(options *types.Options) *http.Client {
+func newhttpClient(options *types.Options) (*http.Client, error) {
 	dialer := protocolstate.Dialer
 
 	// Set the base TLS configuration definition
@@ -27,7 +28,11 @@ func newhttpClient(options *types.Options) *http.Client {
 	}
 
 	// Add the client certificate authentication to the request if it's configured
-	tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+	var err error
+	tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+	if err != nil {
+		return nil, err
+	}
 
 	transport := &http.Transport{
 		DialContext:         dialer.Dial,
@@ -36,15 +41,13 @@ func newhttpClient(options *types.Options) *http.Client {
 		MaxConnsPerHost:     500,
 		TLSClientConfig:     tlsConfig,
 	}
-
-	if options.ProxyURL != "" {
-		if proxyURL, err := url.Parse(options.ProxyURL); err == nil {
+	if types.ProxyURL != "" {
+		if proxyURL, err := url.Parse(types.ProxyURL); err == nil {
 			transport.Proxy = http.ProxyURL(proxyURL)
 		}
-	} else if options.ProxySocksURL != "" {
+	} else if types.ProxySocksURL != "" {
 		var proxyAuth *proxy.Auth
-
-		socksURL, proxyErr := url.Parse(options.ProxySocksURL)
+		socksURL, proxyErr := url.Parse(types.ProxySocksURL)
 		if proxyErr == nil {
 			proxyAuth = &proxy.Auth{}
 			proxyAuth.User = socksURL.User.Username()
@@ -71,5 +74,5 @@ func newhttpClient(options *types.Options) *http.Client {
 		},
 	}
 
-	return httpclient
+	return httpclient, nil
 }

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
@@ -14,6 +13,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+
 	"github.com/pkg/errors"
 	"golang.org/x/net/proxy"
 	"golang.org/x/net/publicsuffix"
@@ -129,9 +130,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		return client, nil
 	}
 	poolMutex.RUnlock()
-
-	if options.ProxyURL != "" {
-		proxyURL, err = url.Parse(options.ProxyURL)
+	if types.ProxyURL != "" {
+		proxyURL, err = url.Parse(types.ProxyURL)
 	}
 	if err != nil {
 		return nil, err
@@ -169,7 +169,10 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	}
 
 	// Add the client certificate authentication to the request if it's configured
-	tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+	tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create client certificate")
+	}
 
 	transport := &http.Transport{
 		DialContext:         Dialer.Dial,
@@ -179,27 +182,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		TLSClientConfig:     tlsConfig,
 		DisableKeepAlives:   disableKeepAlives,
 	}
-
-	// Attempts to overwrite the dial function with the socks proxied version
-	if options.ProxySocksURL != "" {
-		var proxyAuth *proxy.Auth
-
-		socksURL, proxyErr := url.Parse(options.ProxySocksURL)
-		if proxyErr == nil {
-			proxyAuth = &proxy.Auth{}
-			proxyAuth.User = socksURL.User.Username()
-			proxyAuth.Password, _ = socksURL.User.Password()
-		}
-		dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
-		dc := dialer.(interface {
-			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
-		})
-		if proxyErr == nil {
-			transport.DialContext = dc.DialContext
-		}
-	}
 	if proxyURL != nil {
-		transport.Proxy = http.ProxyURL(proxyURL)
+		// Attempts to overwrite the dial function with the socks proxied version
+		if proxyURL.Scheme == types.SOCKS5 {
+			var proxyAuth *proxy.Auth = &proxy.Auth{}
+			proxyAuth.User = proxyURL.User.Username()
+			proxyAuth.Password, _ = proxyURL.User.Password()
+
+			dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct)
+
+			dc := dialer.(interface {
+				DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+			})
+			if proxyErr == nil {
+				transport.DialContext = dc.DialContext
+			}
+		} else {
+			transport.Proxy = http.ProxyURL(proxyURL)
+		}
 	}
 
 	var jar *cookiejar.Jar

v2/pkg/protocols/http/request.go

@@ -105,16 +105,16 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
 	mutex := &sync.Mutex{}
 	for {
 		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
-		if err == io.EOF {
-			break
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+			return err
 		}
 		if reqURL == "" {
 			reqURL = generatedHttpRequest.URL()
 		}
-		if err != nil {
-			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
-			return err
-		}
 		swg.Add()
 		go func(httpRequest *generatedRequest) {
 			defer swg.Done()
@@ -168,18 +168,17 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
 	mutex := &sync.Mutex{}
 	for {
 		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
-		if err == io.EOF {
-			break
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+			return err
 		}
 		if reqURL == "" {
 			reqURL = generatedHttpRequest.URL()
 		}
-		if err != nil {
-			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
-			return err
-		}
 		generatedHttpRequest.pipelinedClient = pipeClient
-
 		swg.Add()
 		go func(httpRequest *generatedRequest) {
 			defer swg.Done()
@@ -222,17 +221,16 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 		hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
 
 		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
-		if err == io.EOF {
-			break
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+			return err
 		}
 		if reqURL == "" {
 			reqURL = generatedHttpRequest.URL()
 		}
-		if err != nil {
-			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
-			return err
-		}
-
 		request.dynamicValues = generatedHttpRequest.dynamicValues
 		// Check if hosts just keep erroring
 		if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) {
@@ -406,7 +404,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		return errors.Wrap(err, "could not dump http response")
 	}
 
-	var data, redirectedResponse []byte
+	var dumpedResponse []redirectedResponse
 	// If the status code is HTTP 101, we should not proceed with reading body.
 	if resp.StatusCode != http.StatusSwitchingProtocols {
 		var bodyReader io.Reader
@@ -415,7 +413,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		} else {
 			bodyReader = resp.Body
 		}
-		data, err = ioutil.ReadAll(bodyReader)
+		data, err := ioutil.ReadAll(bodyReader)
 		if err != nil {
 			// Ignore body read due to server misconfiguration errors
 			if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
@@ -426,96 +424,61 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		}
 		resp.Body.Close()
 
-		redirectedResponse, err = dumpResponseWithRedirectChain(resp, data)
+		dumpedResponse, err = dumpResponseWithRedirectChain(resp, data)
 		if err != nil {
 			return errors.Wrap(err, "could not read http response with redirect chain")
 		}
 	} else {
-		redirectedResponse = dumpedResponseHeaders
+		dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}}
 	}
 
-	// net/http doesn't automatically decompress the response body if an
-	// encoding has been specified by the user in the request so in case we have to
-	// manually do it.
-	dataOrig := data
-	data, err = handleDecompression(resp, data)
-	// in case of error use original data
-	if err != nil {
-		data = dataOrig
-	}
-
-	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
-	dumpedResponseBuilder := &bytes.Buffer{}
-	dumpedResponseBuilder.Write(dumpedResponseHeaders)
-	dumpedResponseBuilder.Write(data)
-	dumpedResponse := dumpedResponseBuilder.Bytes()
-	redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data)
-
-	// Decode gbk response content-types
-	// gb18030 supersedes gb2312
-	responseContentType := resp.Header.Get("Content-Type")
-	if isContentTypeGbk(responseContentType) {
-		dumpedResponse, err = decodegbk(dumpedResponse)
-		if err != nil {
-			return errors.Wrap(err, "could not gbk decode")
-		}
-		redirectedResponse, err = decodegbk(redirectedResponse)
-		if err != nil {
-			return errors.Wrap(err, "could not gbk decode")
+	for _, response := range dumpedResponse {
+		// if nuclei-project is enabled store the response if not previously done
+		if request.options.ProjectFile != nil && !fromCache {
+			if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil {
+				return errors.Wrap(err, "could not store in project file")
+			}
 		}
 
-		// the uncompressed body needs to be decoded to standard utf8
-		data, err = decodegbk(data)
-		if err != nil {
-			return errors.Wrap(err, "could not gbk decode")
+		matchedURL := reqURL
+		if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
+			matchedURL = generatedRequest.rawRequest.FullURL
 		}
-	}
-
-	// if nuclei-project is enabled store the response if not previously done
-	if request.options.ProjectFile != nil && !fromCache {
-		if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil {
-			return errors.Wrap(err, "could not store in project file")
+		if generatedRequest.request != nil {
+			matchedURL = generatedRequest.request.URL.String()
 		}
-	}
+		finalEvent := make(output.InternalEvent)
 
-	matchedURL := reqURL
-	if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
-		matchedURL = generatedRequest.rawRequest.FullURL
-	}
-	if generatedRequest.request != nil {
-		matchedURL = generatedRequest.request.URL.String()
-	}
-	finalEvent := make(output.InternalEvent)
-
-	outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta)
-	if i := strings.LastIndex(hostname, ":"); i != -1 {
-		hostname = hostname[:i]
-	}
-	outputEvent["curl-command"] = curlCommand
-	outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
-	outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
-	for k, v := range previousEvent {
-		finalEvent[k] = v
-	}
-	for k, v := range outputEvent {
-		finalEvent[k] = v
-	}
-	// Add to history the current request number metadata if asked by the user.
-	if request.ReqCondition {
+		outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
+		if i := strings.LastIndex(hostname, ":"); i != -1 {
+			hostname = hostname[:i]
+		}
+		outputEvent["curl-command"] = curlCommand
+		outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
+		for k, v := range previousEvent {
+			finalEvent[k] = v
+		}
 		for k, v := range outputEvent {
-			key := fmt.Sprintf("%s_%d", k, requestCount)
-			previousEvent[key] = v
-			finalEvent[key] = v
+			finalEvent[k] = v
 		}
+		// Add to history the current request number metadata if asked by the user.
+		if request.ReqCondition {
+			for k, v := range outputEvent {
+				key := fmt.Sprintf("%s_%d", k, requestCount)
+				previousEvent[key] = v
+				finalEvent[key] = v
+			}
+		}
+
+		event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
+			internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
+		})
+
+		responseContentType := resp.Header.Get("Content-Type")
+		dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType)
+
+		callback(event)
 	}
-
-	event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
-		internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
-	})
-
-	dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType)
-
-	callback(event)
 	return nil
 }
 

v2/pkg/protocols/http/utils.go

@@ -10,14 +10,21 @@ import (
 	"net/http/httputil"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/stringsutil"
 	"golang.org/x/text/encoding/simplifiedchinese"
 	"golang.org/x/text/transform"
 )
 
+type redirectedResponse struct {
+	headers      []byte
+	body         []byte
+	fullResponse []byte
+	resp         *http.Response
+}
+
 // dumpResponseWithRedirectChain dumps a http response with the
 // complete http redirect chain.
 //
@@ -25,18 +32,23 @@ import (
 // and returns the data to the user for matching and viewing in that order.
 //
 // Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923
-func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, error) {
-	redirects := []string{}
+func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) {
+	var response []redirectedResponse
+
 	respData, err := httputil.DumpResponse(resp, false)
 	if err != nil {
 		return nil, err
 	}
-	redirectChain := &bytes.Buffer{}
-
-	redirectChain.WriteString(tostring.UnsafeToString(respData))
-	redirectChain.Write(body)
-	redirects = append(redirects, redirectChain.String())
-	redirectChain.Reset()
+	respObj := redirectedResponse{
+		headers:      respData,
+		body:         body,
+		resp:         resp,
+		fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
+	}
+	if err := normalizeResponseBody(resp, &respObj); err != nil {
+		return nil, err
+	}
+	response = append(response, respObj)
 
 	var redirectResp *http.Response
 	if resp != nil && resp.Request != nil {
@@ -52,40 +64,51 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, er
 		if redirectResp.Body != nil {
 			body, _ = ioutil.ReadAll(redirectResp.Body)
 		}
-		redirectChain.WriteString(tostring.UnsafeToString(respData))
-		if len(body) > 0 {
-			redirectChain.WriteString(tostring.UnsafeToString(body))
+		respObj := redirectedResponse{
+			headers:      respData,
+			body:         body,
+			resp:         redirectResp,
+			fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
 		}
-		redirects = append(redirects, redirectChain.String())
+		if err := normalizeResponseBody(redirectResp, &respObj); err != nil {
+			return nil, err
+		}
+		response = append(response, respObj)
 		redirectResp = redirectResp.Request.Response
-		redirectChain.Reset()
 	}
-	for i := len(redirects) - 1; i >= 0; i-- {
-		redirectChain.WriteString(redirects[i])
-	}
-	return redirectChain.Bytes(), nil
+	return response, nil
 }
 
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-
-			if i != len(values)-1 {
-				builder.WriteRune('\n')
-				builder.WriteString(header)
-				builder.WriteString(": ")
-			}
-		}
-		builder.WriteRune('\n')
+// normalizeResponseBody performs normalization on the http response object.
+func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error {
+	var err error
+	// net/http doesn't automatically decompress the response body if an
+	// encoding has been specified by the user in the request so in case we have to
+	// manually do it.
+	dataOrig := response.body
+	response.body, err = handleDecompression(resp, response.body)
+	// in case of error use original data
+	if err != nil {
+		response.body = dataOrig
 	}
-	return builder.String()
+	response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body)
+
+	// Decode gbk response content-types
+	// gb18030 supersedes gb2312
+	responseContentType := resp.Header.Get("Content-Type")
+	if isContentTypeGbk(responseContentType) {
+		response.fullResponse, err = decodegbk(response.fullResponse)
+		if err != nil {
+			return errors.Wrap(err, "could not gbk decode")
+		}
+
+		// the uncompressed body needs to be decoded to standard utf8
+		response.body, err = decodegbk(response.body)
+		if err != nil {
+			return errors.Wrap(err, "could not gbk decode")
+		}
+	}
+	return nil
 }
 
 // dump creates a dump of the http request in form of a byte slice

v2/pkg/protocols/offlinehttp/read_response_test.go

@@ -1,12 +1,15 @@
 package offlinehttp
 
 import (
+	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
 	"net/http/httputil"
 	"testing"
 	"time"
 
+	"github.com/julienschmidt/httprouter"
 	"github.com/stretchr/testify/require"
 )
 
@@ -154,11 +157,30 @@ Server: Google Frontend
 	}
 
 	t.Run("test-live-response-with-content-length", func(t *testing.T) {
+		var ts *httptest.Server
+		router := httprouter.New()
+		router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+			w.Header().Add("Server", "Google Frontend")
+			fmt.Fprintf(w, "%s", `<!DOCTYPE html>
+			<html>
+			<head>
+			<title>Firing Range</title>
+			</head>
+			<body>
+			   <h1>Version 0.48</h1>
+			   <h1>What is the Firing Range?</h1>
+			   <p>
+			</body>
+			</html>`)
+		}))
+		ts = httptest.NewServer(router)
+		defer ts.Close()
+
 		client := &http.Client{
 			Timeout: 3 * time.Second,
 		}
 
-		data, err := client.Get("https://golang.org/doc/install")
+		data, err := client.Get(ts.URL)
 		require.Nil(t, err, "could not dial url")
 		defer data.Body.Close()
 

v2/pkg/protocols/utils/utils.go

@@ -3,13 +3,13 @@ package utils
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"io/ioutil"
-	"log"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
 // AddConfiguredClientCertToRequest adds the client certificate authentication to the tls.Config object and returns it
-func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) *tls.Config {
+func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) (*tls.Config, error) {
 	// Build the TLS config with the client certificate if it has been configured with the appropriate options.
 	// Only one of the options needs to be checked since the validation checks in main.go ensure that all three
 	// files are set if any of the client certification configuration options are.
@@ -17,18 +17,18 @@ func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Opti
 		// Load the client certificate using the PEM encoded client certificate and the private key file
 		cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile)
 		if err != nil {
-			log.Fatal(err)
+			return nil, err
 		}
 		tlsConfig.Certificates = []tls.Certificate{cert}
 
 		// Load the certificate authority PEM certificate into the TLS configuration
 		caCert, err := ioutil.ReadFile(options.ClientCAFile)
 		if err != nil {
-			log.Fatal(err)
+			return nil, err
 		}
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
 		tlsConfig.RootCAs = caCertPool
 	}
-	return tlsConfig
+	return tlsConfig, nil
 }

v2/pkg/testutils/testutils.go

@@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{
 	Targets:                    []string{},
 	TargetsFilePath:            "",
 	Output:                     "",
-	ProxyURL:                   "",
-	ProxySocksURL:              "",
+	Proxy:                      []string{},
 	TemplatesDirectory:         "",
 	TraceLogFile:               "",
 	Templates:                  []string{},

v2/pkg/types/proxy.go

@@ -0,0 +1,15 @@
+package types
+
+const (
+	HTTP_PROXY_ENV = "HTTP_PROXY"
+	SOCKS5         = "socks5"
+	HTTP           = "http"
+	HTTPS          = "https"
+)
+
+var (
+	// ProxyURL is the URL for the proxy server
+	ProxyURL string
+	// ProxySocksURL is the URL for the proxy socks server
+	ProxySocksURL string
+)

v2/pkg/types/types.go

@@ -58,10 +58,8 @@ type Options struct {
 	TargetsFilePath string
 	// Output is the file to write found results to.
 	Output string
-	// ProxyURL is the URL for the proxy server
-	ProxyURL string
-	// ProxySocksURL is the URL for the proxy socks server
-	ProxySocksURL string
+	// List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
+	Proxy goflags.NormalizedStringSlice
 	// TemplatesDirectory is the directory to use for storing templates
 	TemplatesDirectory string
 	// TraceLogFile specifies a file to write with the trace of all requests