From cdd54acf70efee7795caf7a664d4a5996e9a7dba Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Sat, 16 Sep 2023 14:20:35 +0530 Subject: [PATCH] use CL instead of TE + unit test (#4154) * force transfer encoding + unit test * fix nil panic in integration_test --- v2/pkg/protocols/http/build_request.go | 11 ++++ v2/pkg/protocols/http/request.go | 36 ++++++++++- v2/pkg/protocols/http/request_test.go | 90 ++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 2d7b4aece..c121b7228 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -4,6 +4,8 @@ import ( "bufio" "context" "fmt" + "net/http" + "strconv" "strings" "time" @@ -276,6 +278,9 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st if len(r.options.Options.CustomHeaders) > 0 { _ = rawRequestData.TryFillCustomHeaders(r.options.Options.CustomHeaders) } + if rawRequestData.Data != "" && !stringsutil.EqualFoldAny(rawRequestData.Method, http.MethodHead, http.MethodGet) && rawRequestData.Headers["Transfer-Encoding"] != "chunked" { + rawRequestData.Headers["Content-Length"] = strconv.Itoa(len(rawRequestData.Data)) + } unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs} return unsafeReq, nil } @@ -288,6 +293,12 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st if err != nil { return nil, err } + + // force transfer encoding if conditions are met + if len(rawRequestData.Data) > 0 && req.Header.Get("Transfer-Encoding") != "chunked" && !stringsutil.EqualFoldAny(rawRequestData.Method, http.MethodGet, http.MethodHead) { + req.ContentLength = int64(len(rawRequestData.Data)) + } + // override the body with a new one that will be used to read the request body in parallel threads // for race condition testing if r.request.Threads > 0 && r.request.Race { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 209e8f1b2..a1c642cb1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httputil" + "strconv" "strings" "sync" "time" @@ -36,6 +37,7 @@ import ( templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/utils/reader" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" @@ -490,6 +492,32 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // Dump request for variables checks // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function if !generatedRequest.original.Race { + + // change encoding type to content-length unless transfer-encoding header is manually set + if generatedRequest.request != nil && !stringsutil.EqualFoldAny(generatedRequest.request.Method, http.MethodGet, http.MethodHead) && generatedRequest.request.Body != nil && generatedRequest.request.Header.Get("Transfer-Encoding") != "chunked" { + var newReqBody *reader.ReusableReadCloser + newReqBody, ok := generatedRequest.request.Body.(*reader.ReusableReadCloser) + if !ok { + newReqBody, err = reader.NewReusableReadCloser(generatedRequest.request.Body) + } + if err == nil { + // update the request body with the reusable reader + generatedRequest.request.Body = newReqBody + // get content length + length, _ := io.Copy(io.Discard, newReqBody) + generatedRequest.request.ContentLength = length + } else { + // log error and continue + gologger.Verbose().Msgf("[%v] Could not read request body while forcing transfer encoding: %s\n", request.options.TemplateID, err) + err = nil + } + } + + // do the same for unsafe requests + if generatedRequest.rawRequest != nil && !stringsutil.EqualFoldAny(generatedRequest.rawRequest.Method, http.MethodGet, http.MethodHead) && generatedRequest.rawRequest.Data != "" && generatedRequest.rawRequest.Headers["Transfer-Encoding"] != "chunked" { + generatedRequest.rawRequest.Headers["Content-Length"] = strconv.Itoa(len(generatedRequest.rawRequest.Data)) + } + var dumpError error // TODO: dump is currently not working with post-processors - somehow it alters the signature dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input) @@ -514,6 +542,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ var hostname string timeStart := time.Now() if generatedRequest.original.Pipeline { + // if request is a pipeline request, use the pipelined client if generatedRequest.rawRequest != nil { formedURL = generatedRequest.rawRequest.FullURL if parsed, parseErr := urlutil.ParseURL(formedURL, true); parseErr == nil { @@ -524,6 +553,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) } } else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil { + // if request is a unsafe request, use the rawhttp client formedURL = generatedRequest.rawRequest.FullURL // use request url as matched url if empty if formedURL == "" { @@ -545,11 +575,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ options.SNI = request.options.Options.SNI inputUrl := input.MetaInput.Input if url, err := urlutil.ParseURL(inputUrl, false); err == nil { - inputUrl = fmt.Sprintf("%s://%s", url.Scheme, url.Host) + url.Path = "" + url.Params = urlutil.NewOrderedParams() // donot include query params + // inputUrl should only contain scheme://host:port + inputUrl = url.String() } formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path) resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) } else { + //** For Normal requests **// hostname = generatedRequest.request.URL.Host formedURL = generatedRequest.request.URL.String() // if nuclei-project is available check if the request was already sent previously diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index 01b2fd2bc..514bdae0a 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -94,3 +94,93 @@ Disallow: /c`)) require.NotNil(t, finalEvent, "could not get event output from request") require.Equal(t, 3, matchCount, "could not get correct match count") } + +func TestDisableTE(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "http-disable-transfer-encoding" + + // in raw request format + request := &Request{ + ID: templateID, + Raw: []string{ + `POST / HTTP/1.1 + Host: {{Hostname}} + User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + Accept-Language: en-US,en;q=0.5 + + login=1&username=admin&password=admin + `, + }, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{{ + Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher}, + Status: []int{200}, + }}, + }, + } + + // in base request format + request2 := &Request{ + ID: templateID, + Method: HTTPMethodTypeHolder{MethodType: HTTPPost}, + Path: []string{"{{BaseURL}}"}, + Body: "login=1&username=admin&password=admin", + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{{ + Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher}, + Status: []int{200}, + }}, + }, + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.TransferEncoding) > 0 || r.ContentLength <= 0 { + t.Error("Transfer-Encoding header should not be set") + } + })) + defer ts.Close() + + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile http raw request") + + err = request2.Compile(executerOpts) + require.Nil(t, err, "could not compile http base request") + + var finalEvent *output.InternalWrappedEvent + var matchCount int + t.Run("test", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + ctxArgs := contextargs.NewWithInput(ts.URL) + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { + if event.OperatorsResult != nil && event.OperatorsResult.Matched { + matchCount++ + } + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + + t.Run("test2", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + ctxArgs := contextargs.NewWithInput(ts.URL) + err := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { + if event.OperatorsResult != nil && event.OperatorsResult.Matched { + matchCount++ + } + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 2, matchCount, "could not get correct match count") +}