use CL instead of TE + unit test (#4154)

* force transfer encoding + unit test

* fix nil panic in integration_test

v2/pkg/protocols/http/build_request.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -276,6 +278,9 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		if len(r.options.Options.CustomHeaders) > 0 {
 			_ = rawRequestData.TryFillCustomHeaders(r.options.Options.CustomHeaders)
 		}
+		if rawRequestData.Data != "" && !stringsutil.EqualFoldAny(rawRequestData.Method, http.MethodHead, http.MethodGet) && rawRequestData.Headers["Transfer-Encoding"] != "chunked" {
+			rawRequestData.Headers["Content-Length"] = strconv.Itoa(len(rawRequestData.Data))
+		}
 		unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs}
 		return unsafeReq, nil
 	}
@@ -288,6 +293,12 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 	if err != nil {
 		return nil, err
 	}
+
+	// force transfer encoding if conditions are met
+	if len(rawRequestData.Data) > 0 && req.Header.Get("Transfer-Encoding") != "chunked" && !stringsutil.EqualFoldAny(rawRequestData.Method, http.MethodGet, http.MethodHead) {
+		req.ContentLength = int64(len(rawRequestData.Data))
+	}
+
 	// override the body with a new one that will be used to read the request body in parallel threads
 	// for race condition testing
 	if r.request.Threads > 0 && r.request.Race {

v2/pkg/protocols/http/request.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httputil"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -36,6 +37,7 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
+	"github.com/projectdiscovery/utils/reader"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -490,6 +492,32 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	// Dump request for variables checks
 	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
 	if !generatedRequest.original.Race {
+
+		// change encoding type to content-length unless transfer-encoding header is manually set
+		if generatedRequest.request != nil && !stringsutil.EqualFoldAny(generatedRequest.request.Method, http.MethodGet, http.MethodHead) && generatedRequest.request.Body != nil && generatedRequest.request.Header.Get("Transfer-Encoding") != "chunked" {
+			var newReqBody *reader.ReusableReadCloser
+			newReqBody, ok := generatedRequest.request.Body.(*reader.ReusableReadCloser)
+			if !ok {
+				newReqBody, err = reader.NewReusableReadCloser(generatedRequest.request.Body)
+			}
+			if err == nil {
+				// update the request body with the reusable reader
+				generatedRequest.request.Body = newReqBody
+				// get content length
+				length, _ := io.Copy(io.Discard, newReqBody)
+				generatedRequest.request.ContentLength = length
+			} else {
+				// log error and continue
+				gologger.Verbose().Msgf("[%v] Could not read request body while forcing transfer encoding: %s\n", request.options.TemplateID, err)
+				err = nil
+			}
+		}
+
+		// do the same for unsafe requests
+		if generatedRequest.rawRequest != nil && !stringsutil.EqualFoldAny(generatedRequest.rawRequest.Method, http.MethodGet, http.MethodHead) && generatedRequest.rawRequest.Data != "" && generatedRequest.rawRequest.Headers["Transfer-Encoding"] != "chunked" {
+			generatedRequest.rawRequest.Headers["Content-Length"] = strconv.Itoa(len(generatedRequest.rawRequest.Data))
+		}
+
 		var dumpError error
 		// TODO: dump is currently not working with post-processors - somehow it alters the signature
 		dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input)
@@ -514,6 +542,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	var hostname string
 	timeStart := time.Now()
 	if generatedRequest.original.Pipeline {
+		// if request is a pipeline request, use the pipelined client
 		if generatedRequest.rawRequest != nil {
 			formedURL = generatedRequest.rawRequest.FullURL
 			if parsed, parseErr := urlutil.ParseURL(formedURL, true); parseErr == nil {
@@ -524,6 +553,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
 		}
 	} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
+		// if request is a unsafe request, use the rawhttp client
 		formedURL = generatedRequest.rawRequest.FullURL
 		// use request url as matched url if empty
 		if formedURL == "" {
@@ -545,11 +575,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		options.SNI = request.options.Options.SNI
 		inputUrl := input.MetaInput.Input
 		if url, err := urlutil.ParseURL(inputUrl, false); err == nil {
-			inputUrl = fmt.Sprintf("%s://%s", url.Scheme, url.Host)
+			url.Path = ""
+			url.Params = urlutil.NewOrderedParams() // donot include query params
+			// inputUrl should only contain scheme://host:port
+			inputUrl = url.String()
 		}
 		formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path)
 		resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
 	} else {
+		//** For Normal requests **//
 		hostname = generatedRequest.request.URL.Host
 		formedURL = generatedRequest.request.URL.String()
 		// if nuclei-project is available check if the request was already sent previously

v2/pkg/protocols/http/request_test.go

@@ -94,3 +94,93 @@ Disallow: /c`))
 	require.NotNil(t, finalEvent, "could not get event output from request")
 	require.Equal(t, 3, matchCount, "could not get correct match count")
 }
+
+func TestDisableTE(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "http-disable-transfer-encoding"
+
+	// in raw request format
+	request := &Request{
+		ID: templateID,
+		Raw: []string{
+			`POST / HTTP/1.1
+			Host: {{Hostname}}
+			User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0
+			Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+			Accept-Language: en-US,en;q=0.5
+
+			login=1&username=admin&password=admin
+			`,
+		},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},
+				Status: []int{200},
+			}},
+		},
+	}
+
+	// in base request format
+	request2 := &Request{
+		ID:     templateID,
+		Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
+		Path:   []string{"{{BaseURL}}"},
+		Body:   "login=1&username=admin&password=admin",
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},
+				Status: []int{200},
+			}},
+		},
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if len(r.TransferEncoding) > 0 || r.ContentLength <= 0 {
+			t.Error("Transfer-Encoding header should not be set")
+		}
+	}))
+	defer ts.Close()
+
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile http raw request")
+
+	err = request2.Compile(executerOpts)
+	require.Nil(t, err, "could not compile http base request")
+
+	var finalEvent *output.InternalWrappedEvent
+	var matchCount int
+	t.Run("test", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		ctxArgs := contextargs.NewWithInput(ts.URL)
+		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
+			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+				matchCount++
+			}
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute network request")
+	})
+
+	t.Run("test2", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		ctxArgs := contextargs.NewWithInput(ts.URL)
+		err := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
+			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+				matchCount++
+			}
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute network request")
+	})
+
+	require.NotNil(t, finalEvent, "could not get event output from request")
+	require.Equal(t, 2, matchCount, "could not get correct match count")
+}