From e7cffad31205bcd059daec488c7addb95dca084e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 23 Aug 2022 12:45:55 +0530 Subject: [PATCH] Fixed request annotation based timeout bugs + tests + misc (#2476) --- .../http/annotation-timeout.yaml | 18 ++++++++ v2/cmd/integration-test/http.go | 21 +++++++++ v2/pkg/protocols/http/build_request.go | 9 +++- v2/pkg/protocols/http/request_annotations.go | 4 +- .../http/request_annotations_test.go | 45 +++++++++++++++++++ 5 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 integration_tests/http/annotation-timeout.yaml create mode 100644 v2/pkg/protocols/http/request_annotations_test.go diff --git a/integration_tests/http/annotation-timeout.yaml b/integration_tests/http/annotation-timeout.yaml new file mode 100644 index 000000000..ddb0b3e6a --- /dev/null +++ b/integration_tests/http/annotation-timeout.yaml @@ -0,0 +1,18 @@ +id: annotation-timeout + +info: + name: Basic Annotation Timeout + author: pdteam + severity: info + +requests: + - raw: + - | + @timeout: 5s + GET / HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: word + words: + - "This is test matcher text" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c4ca5e9a4..0fdf9f566 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -11,6 +11,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/julienschmidt/httprouter" @@ -52,6 +53,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-sni.yaml": &customCLISNI{}, "http/redirect-match-url.yaml": &httpRedirectMatchURL{}, "http/get-sni-unsafe.yaml": &customCLISNIUnsafe{}, + "http/annotation-timeout.yaml": &annotationTimeout{}, } type httpInteractshRequest struct{} @@ -909,3 +911,22 @@ func (h *customCLISNIUnsafe) Execute(filePath string) error { } return expectResultsCount(results, 1) } + +type annotationTimeout struct{} + +// Execute executes a test case and returns an error if occurred +func (h *annotationTimeout) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + time.Sleep(4 * time.Second) + fmt.Fprintf(w, "This is test matcher text") + }) + ts := httptest.NewTLSServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-timeout", "1") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 84fb06f27..fc8968be2 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -26,6 +26,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/stringsutil" ) var ( @@ -112,10 +113,15 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st if isRawRequest { // Get the hostname from the URL section to build the request. reader := bufio.NewReader(strings.NewReader(data)) + read_line: s, err := reader.ReadString('\n') if err != nil { return nil, fmt.Errorf("could not read request: %w", err) } + // ignore all annotations + if stringsutil.HasPrefixAny(s, "@") { + goto read_line + } parts := strings.Split(s, " ") if len(parts) < 3 { @@ -288,8 +294,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest } if reqWithAnnotations, hasAnnotations := r.request.parseAnnotations(rawRequest, req); hasAnnotations { - req = reqWithAnnotations - request = request.WithContext(req.Context()) + request.Request = reqWithAnnotations } return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil diff --git a/v2/pkg/protocols/http/request_annotations.go b/v2/pkg/protocols/http/request_annotations.go index b14831cb7..0b15c7dac 100644 --- a/v2/pkg/protocols/http/request_annotations.go +++ b/v2/pkg/protocols/http/request_annotations.go @@ -102,12 +102,12 @@ func (r *Request) parseAnnotations(rawRequest string, request *http.Request) (*h value := strings.TrimSpace(duration[1]) if parsed, err := time.ParseDuration(value); err == nil { //nolint:govet // cancelled automatically by withTimeout - ctx, _ := context.WithTimeout(request.Context(), parsed) + ctx, _ := context.WithTimeout(context.Background(), parsed) request = request.Clone(ctx) } } else { //nolint:govet // cancelled automatically by withTimeout - ctx, _ := context.WithTimeout(request.Context(), time.Duration(r.options.Options.Timeout)*time.Second) + ctx, _ := context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second) request = request.Clone(ctx) } } diff --git a/v2/pkg/protocols/http/request_annotations_test.go b/v2/pkg/protocols/http/request_annotations_test.go new file mode 100644 index 000000000..682a923fa --- /dev/null +++ b/v2/pkg/protocols/http/request_annotations_test.go @@ -0,0 +1,45 @@ +package http + +import ( + "context" + "net/http" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/stretchr/testify/require" +) + +func TestRequestParseAnnotationsTimeout(t *testing.T) { + t.Run("positive", func(t *testing.T) { + request := &Request{ + connConfiguration: &httpclientpool.Configuration{NoTimeout: true}, + } + rawRequest := `@timeout: 2s + GET / HTTP/1.1 + Host: {{Hostname}}` + + httpReq, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.Nil(t, err, "could not create http request") + + newRequest, modified := request.parseAnnotations(rawRequest, httpReq) + require.True(t, modified, "could not get correct modified value") + _, deadlined := newRequest.Context().Deadline() + require.True(t, deadlined, "could not get set request deadline") + }) + + t.Run("negative", func(t *testing.T) { + request := &Request{ + connConfiguration: &httpclientpool.Configuration{}, + } + rawRequest := `GET / HTTP/1.1 + Host: {{Hostname}}` + + httpReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://example.com", nil) + require.Nil(t, err, "could not create http request") + + newRequest, modified := request.parseAnnotations(rawRequest, httpReq) + require.False(t, modified, "could not get correct modified value") + _, deadlined := newRequest.Context().Deadline() + require.False(t, deadlined, "could not get set request deadline") + }) +}