From a52e466b54806ada72f4a1c086ea20d551d14068 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 19 Nov 2020 01:27:06 +0100 Subject: [PATCH 1/2] adding gunzip in response dump --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/executer/executer_http.go | 17 +++++++++++++---- v2/pkg/requests/util.go | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index bdd2a3ed6..34f6c37d3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -24,6 +24,7 @@ require ( github.com/projectdiscovery/retryabledns v1.0.5 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 + github.com/segmentio/ksuid v1.0.3 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.5.1 go.uber.org/ratelimit v0.1.0 diff --git a/v2/go.sum b/v2/go.sum index 397397fa1..348d66b42 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,6 +86,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= +github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index bb177547a..27c0a96c5 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -1,6 +1,7 @@ package executer import ( + "bytes" "context" "crypto/tls" "fmt" @@ -458,14 +459,14 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, duration := time.Since(timeStart) + // Dump response - Step 1 - Decompression not yet handled + var dumpedResponse []byte if e.debug { - dumpedResponse, dumpErr := httputil.DumpResponse(resp, true) + var dumpErr error + dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) if dumpErr != nil { return errors.Wrap(dumpErr, "could not dump http response") } - - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) } data, err := ioutil.ReadAll(resp.Body) @@ -485,11 +486,19 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request // so in case we have to manually do it + dataOrig := data data, err = requests.HandleDecompression(request, data) if err != nil { return errors.Wrap(err, "could not decompress http body") } + // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) + if e.debug { + dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) + gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID) + fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) + } + // if nuclei-project is enabled store the response if not previously done if e.pf != nil && !fromcache { err := e.pf.Set(dumpedRequest, resp, data) diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go index da4692018..4c59c406e 100644 --- a/v2/pkg/requests/util.go +++ b/v2/pkg/requests/util.go @@ -35,7 +35,7 @@ func HandleDecompression(r *HTTPRequest, bodyOrig []byte) (bodyDec []byte, err e return bodyOrig, nil } - encodingHeader := strings.ToLower(r.Request.Header.Get("Accept-Encoding")) + encodingHeader := strings.TrimSpace(strings.ToLower(r.Request.Header.Get("Accept-Encoding"))) if encodingHeader == "gzip" { gzipreader, err := gzip.NewReader(bytes.NewReader(bodyOrig)) if err != nil { From 3739d105f1e034261c72ef71e71a9cc62bd839c1 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 20 Nov 2020 10:19:51 +0100 Subject: [PATCH 2/2] adding missing encoding --- v2/pkg/requests/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go index 4c59c406e..bdd633a83 100644 --- a/v2/pkg/requests/util.go +++ b/v2/pkg/requests/util.go @@ -36,7 +36,7 @@ func HandleDecompression(r *HTTPRequest, bodyOrig []byte) (bodyDec []byte, err e } encodingHeader := strings.TrimSpace(strings.ToLower(r.Request.Header.Get("Accept-Encoding"))) - if encodingHeader == "gzip" { + if encodingHeader == "gzip" || encodingHeader == "gzip, deflate" { gzipreader, err := gzip.NewReader(bytes.NewReader(bodyOrig)) if err != nil { return bodyDec, err