diff --git a/README.md b/README.md
index ca1988816..e4626a05a 100644
--- a/README.md
+++ b/README.md
@@ -162,16 +162,16 @@ HEADLESS:
-sc, -system-chrome Use local installed chrome browser instead of nuclei installed
DEBUG:
- -debug show all requests and responses
- -debug-req show all sent requests
- -debug-resp show all received responses
- -proxy, -proxy-url string URL of the HTTP proxy server
- -proxy-socks-url string URL of the SOCKS proxy server
- -tlog, -trace-log string file to write sent requests trace log
- -version show nuclei version
- -v, -verbose show verbose output
- -vv display extra verbose information
- -tv, -templates-version shows the version of the installed nuclei-templates
+ -debug show all requests and responses
+ -debug-req show all sent requests
+ -debug-resp show all received responses
+ -p, -proxy string[] List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
+ -tlog, -trace-log string file to write sent requests trace log
+ -elog, -error-log string file to write sent requests error log
+ -version show nuclei version
+ -v, -verbose show verbose output
+ -vv display templates loaded for scan
+ -tv, -templates-version shows the version of the installed nuclei-templates
UPDATE:
-update update nuclei engine to the latest released version
diff --git a/README_CN.md b/README_CN.md
index 04138acd2..0c4aa69c0 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -121,8 +121,7 @@ nuclei -h
|templates-version|显示已安装的模板版本|nuclei -templates-version|
|v|显示发送请求的详细信息|nuclei -v|
|version|显示nuclei的版本号|nuclei -version|
-|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080|
-|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080|
+|proxy|输入代理地址|nuclei -proxy ./proxy.txt|
|random-agent|使用随机的UA|nuclei -random-agent|
|H|自定义请求头|nuclei -H “x-bug-bounty:hacker”|
diff --git a/integration_tests/http/get-redirects-chain-headers.yaml b/integration_tests/http/get-redirects-chain-headers.yaml
new file mode 100644
index 000000000..512073018
--- /dev/null
+++ b/integration_tests/http/get-redirects-chain-headers.yaml
@@ -0,0 +1,23 @@
+id: basic-get-redirects-chain-headers
+
+info:
+ name: Basic GET Redirects Request With Chain header
+ author: pdteam
+ severity: info
+
+requests:
+ - method: GET
+ path:
+ - "{{BaseURL}}"
+ redirects: true
+ max-redirects: 3
+ matchers-condition: and
+ matchers:
+ - type: word
+ part: header
+ words:
+ - "TestRedirectHeaderMatch"
+
+ - type: status
+ status:
+ - 302
\ No newline at end of file
diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go
index 8805fef81..0b2bb247a 100644
--- a/v2/cmd/integration-test/http.go
+++ b/v2/cmd/integration-test/http.go
@@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{
"http/self-contained.yaml": &httpRequestSelContained{},
"http/get-case-insensitive.yaml": &httpGetCaseInsensitive{},
"http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
+ "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{},
}
type httpInteractshRequest struct{}
@@ -599,3 +600,31 @@ func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error {
}
return nil
}
+
+type httpGetRedirectsChainHeaders struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error {
+ router := httprouter.New()
+ router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ http.Redirect(w, r, "/redirected", http.StatusFound)
+ })
+ router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ w.Header().Set("Secret", "TestRedirectHeaderMatch")
+ http.Redirect(w, r, "/final", http.StatusFound)
+ })
+ router.GET("/final", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ _, _ = w.Write([]byte("ok"))
+ })
+ ts := httptest.NewServer(router)
+ defer ts.Close()
+
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+ if err != nil {
+ return err
+ }
+ if len(results) != 1 {
+ return errIncorrectResultsCount(results)
+ }
+ return nil
+}
diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go
index 47005ed18..a44aa4a60 100644
--- a/v2/cmd/integration-test/integration-test.go
+++ b/v2/cmd/integration-test/integration-test.go
@@ -52,5 +52,5 @@ func main() {
}
func errIncorrectResultsCount(results []string) error {
- return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+ return fmt.Errorf("incorrect number of results \n\t%s", strings.Join(results, "\n\t"))
}
diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index 4e99545bb..7f24385fd 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -143,11 +143,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
-
- /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
- TODO should auto-set the HTTP_PROXY variable for the process? */
- flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
- flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
+ flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"),
flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),
flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go
index 4cbcd69f0..b4dc2f3df 100644
--- a/v2/internal/runner/options.go
+++ b/v2/internal/runner/options.go
@@ -3,7 +3,6 @@ package runner
import (
"bufio"
"errors"
- "net/url"
"os"
"path/filepath"
"strings"
@@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) {
// Read the inputs and configure the logging
configureOutput(options)
-
// Show the user the banner
showBanner()
@@ -82,15 +80,10 @@ func validateOptions(options *types.Options) error {
if options.Verbose && options.Silent {
return errors.New("both verbose and silent mode specified")
}
-
- if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil {
+ //loading the proxy server list from file or cli and test the connectivity
+ if err := loadProxyServers(options); err != nil {
return err
}
-
- if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil {
- return err
- }
-
if options.Validate {
options.Headless = true // required for correct validation of headless templates
validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
@@ -107,19 +100,6 @@ func validateOptions(options *types.Options) error {
return nil
}
-func validateProxyURL(proxyURL, message string) error {
- if proxyURL != "" && !isValidURL(proxyURL) {
- return errors.New(message)
- }
-
- return nil
-}
-
-func isValidURL(urlString string) bool {
- _, err := url.Parse(urlString)
- return err == nil
-}
-
// configureOutput configures the output logging levels to be displayed on the screen
func configureOutput(options *types.Options) {
// If the user desires verbose output, show verbose output
@@ -165,7 +145,6 @@ func loadResolvers(options *types.Options) {
func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
allGivenTemplatePaths := append(templatePaths, workflowPaths...)
-
for _, templatePath := range allGivenTemplatePaths {
if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
fileInfo, err := os.Stat(templatePath)
diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go
new file mode 100644
index 000000000..e254472a7
--- /dev/null
+++ b/v2/internal/runner/proxy.go
@@ -0,0 +1,123 @@
+package runner
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "net"
+ "net/url"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/projectdiscovery/fileutil"
+ "github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var proxyURLList []url.URL
+
+// loadProxyServers load list of proxy servers from file or comma seperated
+func loadProxyServers(options *types.Options) error {
+ if len(options.Proxy) == 0 {
+ return nil
+ }
+ for _, p := range options.Proxy {
+ if proxyURL, err := validateProxyURL(p); err == nil {
+ proxyURLList = append(proxyURLList, proxyURL)
+ } else if fileutil.FileExists(p) {
+ file, err := os.Open(p)
+ if err != nil {
+ return fmt.Errorf("could not open proxy file: %s", err)
+ }
+ defer file.Close()
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ proxy := scanner.Text()
+ if strings.TrimSpace(proxy) == "" {
+ continue
+ }
+ if proxyURL, err := validateProxyURL(proxy); err != nil {
+ return err
+ } else {
+ proxyURLList = append(proxyURLList, proxyURL)
+ }
+ }
+ } else {
+ return fmt.Errorf("invalid proxy file or URL provided for %s", p)
+ }
+ }
+ return processProxyList(options)
+}
+
+func processProxyList(options *types.Options) error {
+ if len(proxyURLList) == 0 {
+ return fmt.Errorf("could not find any valid proxy")
+ } else {
+ done := make(chan bool)
+ exitCounter := make(chan bool)
+ counter := 0
+ for _, url := range proxyURLList {
+ go runProxyConnectivity(url, options, done, exitCounter)
+ }
+ for {
+ select {
+ case <-done:
+ {
+ close(done)
+ return nil
+ }
+ case <-exitCounter:
+ {
+ if counter += 1; counter == len(proxyURLList) {
+ return errors.New("no reachable proxy found")
+ }
+ }
+ }
+ }
+ }
+}
+
+func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) {
+ if err := testProxyConnection(proxyURL, options.Timeout); err == nil {
+ if types.ProxyURL == "" && types.ProxySocksURL == "" {
+ assignProxyURL(proxyURL, options)
+ done <- true
+ }
+ }
+ exitCounter <- true
+}
+
+func testProxyConnection(proxyURL url.URL, timeoutDelay int) error {
+ timeout := time.Duration(timeoutDelay) * time.Second
+ _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func assignProxyURL(proxyURL url.URL, options *types.Options) {
+ os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String())
+ if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS {
+ types.ProxyURL = proxyURL.String()
+ types.ProxySocksURL = ""
+ gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String())
+ } else if proxyURL.Scheme == types.SOCKS5 {
+ types.ProxyURL = ""
+ types.ProxySocksURL = proxyURL.String()
+ gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String())
+ }
+}
+
+func validateProxyURL(proxy string) (url.URL, error) {
+ if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) {
+ return *url, nil
+ }
+ return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)")
+}
+
+//isSupportedProtocol checks given protocols are supported
+func isSupportedProtocol(value string) bool {
+ return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5
+}
diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go
index 0cf54988e..d6be2f7df 100644
--- a/v2/pkg/protocols/dns/operators.go
+++ b/v2/pkg/protocols/dns/operators.go
@@ -14,7 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
-// Match matches a generic data response again a given matcher
+// Match matches a generic data response against a given matcher
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := request.getMatchPart(matcher.Part, data)
if !ok {
diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go
index a506cbda4..c435bef40 100644
--- a/v2/pkg/protocols/headless/engine/engine.go
+++ b/v2/pkg/protocols/headless/engine/engine.go
@@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) {
} else {
chromeLauncher = chromeLauncher.Headless(true)
}
- if options.ProxyURL != "" {
- chromeLauncher = chromeLauncher.Proxy(options.ProxyURL)
+ if types.ProxyURL != "" {
+ chromeLauncher = chromeLauncher.Proxy(types.ProxyURL)
}
launcherURL, err := chromeLauncher.Launch()
if err != nil {
@@ -88,7 +88,12 @@ func New(options *types.Options) (*Browser, error) {
if customAgent == "" {
customAgent = uarand.GetRandom()
}
- httpclient := newhttpClient(options)
+
+ httpclient, err := newhttpClient(options)
+ if err != nil {
+ return nil, err
+ }
+
engine := &Browser{
tempDir: dataStore,
customAgent: customAgent,
diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go
index 4ae14cf1e..1f40f98f7 100644
--- a/v2/pkg/protocols/headless/engine/http_client.go
+++ b/v2/pkg/protocols/headless/engine/http_client.go
@@ -4,20 +4,21 @@ import (
"context"
"crypto/tls"
"fmt"
- "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"time"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"golang.org/x/net/proxy"
)
// newhttpClient creates a new http client for headless communication with a timeout
-func newhttpClient(options *types.Options) *http.Client {
+func newhttpClient(options *types.Options) (*http.Client, error) {
dialer := protocolstate.Dialer
// Set the base TLS configuration definition
@@ -27,7 +28,11 @@ func newhttpClient(options *types.Options) *http.Client {
}
// Add the client certificate authentication to the request if it's configured
- tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+ var err error
+ tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+ if err != nil {
+ return nil, err
+ }
transport := &http.Transport{
DialContext: dialer.Dial,
@@ -36,15 +41,13 @@ func newhttpClient(options *types.Options) *http.Client {
MaxConnsPerHost: 500,
TLSClientConfig: tlsConfig,
}
-
- if options.ProxyURL != "" {
- if proxyURL, err := url.Parse(options.ProxyURL); err == nil {
+ if types.ProxyURL != "" {
+ if proxyURL, err := url.Parse(types.ProxyURL); err == nil {
transport.Proxy = http.ProxyURL(proxyURL)
}
- } else if options.ProxySocksURL != "" {
+ } else if types.ProxySocksURL != "" {
var proxyAuth *proxy.Auth
-
- socksURL, proxyErr := url.Parse(options.ProxySocksURL)
+ socksURL, proxyErr := url.Parse(types.ProxySocksURL)
if proxyErr == nil {
proxyAuth = &proxy.Auth{}
proxyAuth.User = socksURL.User.Username()
@@ -71,5 +74,5 @@ func newhttpClient(options *types.Options) *http.Client {
},
}
- return httpclient
+ return httpclient, nil
}
diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go
index 2b7dc0245..191ca3cf0 100644
--- a/v2/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go
@@ -4,7 +4,6 @@ import (
"context"
"crypto/tls"
"fmt"
- "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
"net"
"net/http"
"net/http/cookiejar"
@@ -14,6 +13,8 @@ import (
"sync"
"time"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+
"github.com/pkg/errors"
"golang.org/x/net/proxy"
"golang.org/x/net/publicsuffix"
@@ -129,9 +130,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
return client, nil
}
poolMutex.RUnlock()
-
- if options.ProxyURL != "" {
- proxyURL, err = url.Parse(options.ProxyURL)
+ if types.ProxyURL != "" {
+ proxyURL, err = url.Parse(types.ProxyURL)
}
if err != nil {
return nil, err
@@ -169,7 +169,10 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
}
// Add the client certificate authentication to the request if it's configured
- tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+ tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+ if err != nil {
+ return nil, errors.Wrap(err, "could not create client certificate")
+ }
transport := &http.Transport{
DialContext: Dialer.Dial,
@@ -179,27 +182,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
TLSClientConfig: tlsConfig,
DisableKeepAlives: disableKeepAlives,
}
-
- // Attempts to overwrite the dial function with the socks proxied version
- if options.ProxySocksURL != "" {
- var proxyAuth *proxy.Auth
-
- socksURL, proxyErr := url.Parse(options.ProxySocksURL)
- if proxyErr == nil {
- proxyAuth = &proxy.Auth{}
- proxyAuth.User = socksURL.User.Username()
- proxyAuth.Password, _ = socksURL.User.Password()
- }
- dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
- dc := dialer.(interface {
- DialContext(ctx context.Context, network, addr string) (net.Conn, error)
- })
- if proxyErr == nil {
- transport.DialContext = dc.DialContext
- }
- }
if proxyURL != nil {
- transport.Proxy = http.ProxyURL(proxyURL)
+ // Attempts to overwrite the dial function with the socks proxied version
+ if proxyURL.Scheme == types.SOCKS5 {
+ var proxyAuth *proxy.Auth = &proxy.Auth{}
+ proxyAuth.User = proxyURL.User.Username()
+ proxyAuth.Password, _ = proxyURL.User.Password()
+
+ dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct)
+
+ dc := dialer.(interface {
+ DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+ })
+ if proxyErr == nil {
+ transport.DialContext = dc.DialContext
+ }
+ } else {
+ transport.Proxy = http.ProxyURL(proxyURL)
+ }
}
var jar *cookiejar.Jar
diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go
index 6e1063542..94782c2a9 100644
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@@ -105,16 +105,16 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
mutex := &sync.Mutex{}
for {
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
- if err == io.EOF {
- break
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+ return err
}
if reqURL == "" {
reqURL = generatedHttpRequest.URL()
}
- if err != nil {
- request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
- return err
- }
swg.Add()
go func(httpRequest *generatedRequest) {
defer swg.Done()
@@ -168,18 +168,17 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
mutex := &sync.Mutex{}
for {
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
- if err == io.EOF {
- break
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+ return err
}
if reqURL == "" {
reqURL = generatedHttpRequest.URL()
}
- if err != nil {
- request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
- return err
- }
generatedHttpRequest.pipelinedClient = pipeClient
-
swg.Add()
go func(httpRequest *generatedRequest) {
defer swg.Done()
@@ -222,17 +221,16 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
- if err == io.EOF {
- break
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+ return err
}
if reqURL == "" {
reqURL = generatedHttpRequest.URL()
}
- if err != nil {
- request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
- return err
- }
-
request.dynamicValues = generatedHttpRequest.dynamicValues
// Check if hosts just keep erroring
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) {
@@ -406,7 +404,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
return errors.Wrap(err, "could not dump http response")
}
- var data, redirectedResponse []byte
+ var dumpedResponse []redirectedResponse
// If the status code is HTTP 101, we should not proceed with reading body.
if resp.StatusCode != http.StatusSwitchingProtocols {
var bodyReader io.Reader
@@ -415,7 +413,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
} else {
bodyReader = resp.Body
}
- data, err = ioutil.ReadAll(bodyReader)
+ data, err := ioutil.ReadAll(bodyReader)
if err != nil {
// Ignore body read due to server misconfiguration errors
if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
@@ -426,96 +424,61 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
}
resp.Body.Close()
- redirectedResponse, err = dumpResponseWithRedirectChain(resp, data)
+ dumpedResponse, err = dumpResponseWithRedirectChain(resp, data)
if err != nil {
return errors.Wrap(err, "could not read http response with redirect chain")
}
} else {
- redirectedResponse = dumpedResponseHeaders
+ dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}}
}
- // net/http doesn't automatically decompress the response body if an
- // encoding has been specified by the user in the request so in case we have to
- // manually do it.
- dataOrig := data
- data, err = handleDecompression(resp, data)
- // in case of error use original data
- if err != nil {
- data = dataOrig
- }
-
- // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
- dumpedResponseBuilder := &bytes.Buffer{}
- dumpedResponseBuilder.Write(dumpedResponseHeaders)
- dumpedResponseBuilder.Write(data)
- dumpedResponse := dumpedResponseBuilder.Bytes()
- redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data)
-
- // Decode gbk response content-types
- // gb18030 supersedes gb2312
- responseContentType := resp.Header.Get("Content-Type")
- if isContentTypeGbk(responseContentType) {
- dumpedResponse, err = decodegbk(dumpedResponse)
- if err != nil {
- return errors.Wrap(err, "could not gbk decode")
- }
- redirectedResponse, err = decodegbk(redirectedResponse)
- if err != nil {
- return errors.Wrap(err, "could not gbk decode")
+ for _, response := range dumpedResponse {
+ // if nuclei-project is enabled store the response if not previously done
+ if request.options.ProjectFile != nil && !fromCache {
+ if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil {
+ return errors.Wrap(err, "could not store in project file")
+ }
}
- // the uncompressed body needs to be decoded to standard utf8
- data, err = decodegbk(data)
- if err != nil {
- return errors.Wrap(err, "could not gbk decode")
+ matchedURL := reqURL
+ if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
+ matchedURL = generatedRequest.rawRequest.FullURL
}
- }
-
- // if nuclei-project is enabled store the response if not previously done
- if request.options.ProjectFile != nil && !fromCache {
- if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil {
- return errors.Wrap(err, "could not store in project file")
+ if generatedRequest.request != nil {
+ matchedURL = generatedRequest.request.URL.String()
}
- }
+ finalEvent := make(output.InternalEvent)
- matchedURL := reqURL
- if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
- matchedURL = generatedRequest.rawRequest.FullURL
- }
- if generatedRequest.request != nil {
- matchedURL = generatedRequest.request.URL.String()
- }
- finalEvent := make(output.InternalEvent)
-
- outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta)
- if i := strings.LastIndex(hostname, ":"); i != -1 {
- hostname = hostname[:i]
- }
- outputEvent["curl-command"] = curlCommand
- outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
- outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
- for k, v := range previousEvent {
- finalEvent[k] = v
- }
- for k, v := range outputEvent {
- finalEvent[k] = v
- }
- // Add to history the current request number metadata if asked by the user.
- if request.ReqCondition {
+ outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
+ if i := strings.LastIndex(hostname, ":"); i != -1 {
+ hostname = hostname[:i]
+ }
+ outputEvent["curl-command"] = curlCommand
+ outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
+ for k, v := range previousEvent {
+ finalEvent[k] = v
+ }
for k, v := range outputEvent {
- key := fmt.Sprintf("%s_%d", k, requestCount)
- previousEvent[key] = v
- finalEvent[key] = v
+ finalEvent[k] = v
}
+ // Add to history the current request number metadata if asked by the user.
+ if request.ReqCondition {
+ for k, v := range outputEvent {
+ key := fmt.Sprintf("%s_%d", k, requestCount)
+ previousEvent[key] = v
+ finalEvent[key] = v
+ }
+ }
+
+ event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
+ internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
+ })
+
+ responseContentType := resp.Header.Get("Content-Type")
+ dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType)
+
+ callback(event)
}
-
- event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
- internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
- })
-
- dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType)
-
- callback(event)
return nil
}
diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go
index 89c35a233..5eae57efe 100644
--- a/v2/pkg/protocols/http/utils.go
+++ b/v2/pkg/protocols/http/utils.go
@@ -10,14 +10,21 @@ import (
"net/http/httputil"
"strings"
+ "github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
- "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/stringsutil"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
)
+type redirectedResponse struct {
+ headers []byte
+ body []byte
+ fullResponse []byte
+ resp *http.Response
+}
+
// dumpResponseWithRedirectChain dumps a http response with the
// complete http redirect chain.
//
@@ -25,18 +32,23 @@ import (
// and returns the data to the user for matching and viewing in that order.
//
// Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923
-func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, error) {
- redirects := []string{}
+func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) {
+ var response []redirectedResponse
+
respData, err := httputil.DumpResponse(resp, false)
if err != nil {
return nil, err
}
- redirectChain := &bytes.Buffer{}
-
- redirectChain.WriteString(tostring.UnsafeToString(respData))
- redirectChain.Write(body)
- redirects = append(redirects, redirectChain.String())
- redirectChain.Reset()
+ respObj := redirectedResponse{
+ headers: respData,
+ body: body,
+ resp: resp,
+ fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
+ }
+ if err := normalizeResponseBody(resp, &respObj); err != nil {
+ return nil, err
+ }
+ response = append(response, respObj)
var redirectResp *http.Response
if resp != nil && resp.Request != nil {
@@ -52,40 +64,51 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, er
if redirectResp.Body != nil {
body, _ = ioutil.ReadAll(redirectResp.Body)
}
- redirectChain.WriteString(tostring.UnsafeToString(respData))
- if len(body) > 0 {
- redirectChain.WriteString(tostring.UnsafeToString(body))
+ respObj := redirectedResponse{
+ headers: respData,
+ body: body,
+ resp: redirectResp,
+ fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
}
- redirects = append(redirects, redirectChain.String())
+ if err := normalizeResponseBody(redirectResp, &respObj); err != nil {
+ return nil, err
+ }
+ response = append(response, respObj)
redirectResp = redirectResp.Request.Response
- redirectChain.Reset()
}
- for i := len(redirects) - 1; i >= 0; i-- {
- redirectChain.WriteString(redirects[i])
- }
- return redirectChain.Bytes(), nil
+ return response, nil
}
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
- builder := &strings.Builder{}
-
- for header, values := range headers {
- builder.WriteString(header)
- builder.WriteString(": ")
-
- for i, value := range values {
- builder.WriteString(value)
-
- if i != len(values)-1 {
- builder.WriteRune('\n')
- builder.WriteString(header)
- builder.WriteString(": ")
- }
- }
- builder.WriteRune('\n')
+// normalizeResponseBody performs normalization on the http response object.
+func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error {
+ var err error
+ // net/http doesn't automatically decompress the response body if an
+ // encoding has been specified by the user in the request so in case we have to
+ // manually do it.
+ dataOrig := response.body
+ response.body, err = handleDecompression(resp, response.body)
+ // in case of error use original data
+ if err != nil {
+ response.body = dataOrig
}
- return builder.String()
+ response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body)
+
+ // Decode gbk response content-types
+ // gb18030 supersedes gb2312
+ responseContentType := resp.Header.Get("Content-Type")
+ if isContentTypeGbk(responseContentType) {
+ response.fullResponse, err = decodegbk(response.fullResponse)
+ if err != nil {
+ return errors.Wrap(err, "could not gbk decode")
+ }
+
+ // the uncompressed body needs to be decoded to standard utf8
+ response.body, err = decodegbk(response.body)
+ if err != nil {
+ return errors.Wrap(err, "could not gbk decode")
+ }
+ }
+ return nil
}
// dump creates a dump of the http request in form of a byte slice
diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go
index 4a75d7263..f47b345dc 100644
--- a/v2/pkg/protocols/offlinehttp/read_response_test.go
+++ b/v2/pkg/protocols/offlinehttp/read_response_test.go
@@ -1,12 +1,15 @@
package offlinehttp
import (
+ "fmt"
"io/ioutil"
"net/http"
+ "net/http/httptest"
"net/http/httputil"
"testing"
"time"
+ "github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/require"
)
@@ -154,11 +157,30 @@ Server: Google Frontend
}
t.Run("test-live-response-with-content-length", func(t *testing.T) {
+ var ts *httptest.Server
+ router := httprouter.New()
+ router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+ w.Header().Add("Server", "Google Frontend")
+ fmt.Fprintf(w, "%s", `
+
+
+ Firing Range
+
+
+ Version 0.48
+ What is the Firing Range?
+
+
+ `)
+ }))
+ ts = httptest.NewServer(router)
+ defer ts.Close()
+
client := &http.Client{
Timeout: 3 * time.Second,
}
- data, err := client.Get("https://golang.org/doc/install")
+ data, err := client.Get(ts.URL)
require.Nil(t, err, "could not dial url")
defer data.Body.Close()
diff --git a/v2/pkg/protocols/utils/utils.go b/v2/pkg/protocols/utils/utils.go
index 8d43fd478..d139f1287 100644
--- a/v2/pkg/protocols/utils/utils.go
+++ b/v2/pkg/protocols/utils/utils.go
@@ -3,13 +3,13 @@ package utils
import (
"crypto/tls"
"crypto/x509"
- "github.com/projectdiscovery/nuclei/v2/pkg/types"
"io/ioutil"
- "log"
+
+ "github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// AddConfiguredClientCertToRequest adds the client certificate authentication to the tls.Config object and returns it
-func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) *tls.Config {
+func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) (*tls.Config, error) {
// Build the TLS config with the client certificate if it has been configured with the appropriate options.
// Only one of the options needs to be checked since the validation checks in main.go ensure that all three
// files are set if any of the client certification configuration options are.
@@ -17,18 +17,18 @@ func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Opti
// Load the client certificate using the PEM encoded client certificate and the private key file
cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile)
if err != nil {
- log.Fatal(err)
+ return nil, err
}
tlsConfig.Certificates = []tls.Certificate{cert}
// Load the certificate authority PEM certificate into the TLS configuration
caCert, err := ioutil.ReadFile(options.ClientCAFile)
if err != nil {
- log.Fatal(err)
+ return nil, err
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
tlsConfig.RootCAs = caCertPool
}
- return tlsConfig
+ return tlsConfig, nil
}
diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go
index 2f27ed901..8e0c5923f 100644
--- a/v2/pkg/testutils/testutils.go
+++ b/v2/pkg/testutils/testutils.go
@@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{
Targets: []string{},
TargetsFilePath: "",
Output: "",
- ProxyURL: "",
- ProxySocksURL: "",
+ Proxy: []string{},
TemplatesDirectory: "",
TraceLogFile: "",
Templates: []string{},
diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go
new file mode 100644
index 000000000..a45b4eca6
--- /dev/null
+++ b/v2/pkg/types/proxy.go
@@ -0,0 +1,15 @@
+package types
+
+const (
+ HTTP_PROXY_ENV = "HTTP_PROXY"
+ SOCKS5 = "socks5"
+ HTTP = "http"
+ HTTPS = "https"
+)
+
+var (
+ // ProxyURL is the URL for the proxy server
+ ProxyURL string
+ // ProxySocksURL is the URL for the proxy socks server
+ ProxySocksURL string
+)
diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
index 4080eab89..65958c7af 100644
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@@ -58,10 +58,8 @@ type Options struct {
TargetsFilePath string
// Output is the file to write found results to.
Output string
- // ProxyURL is the URL for the proxy server
- ProxyURL string
- // ProxySocksURL is the URL for the proxy socks server
- ProxySocksURL string
+ // List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
+ Proxy goflags.NormalizedStringSlice
// TemplatesDirectory is the directory to use for storing templates
TemplatesDirectory string
// TraceLogFile specifies a file to write with the trace of all requests