From 25e7799c094e87f6679b92753e41f2ee4795c492 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Sat, 30 Mar 2024 23:50:31 +0530 Subject: [PATCH] req_url_pattern for vuln_hash calculation + unit test (#4964) --- internal/runner/runner.go | 5 ++ pkg/output/output.go | 3 ++ pkg/protocols/http/build_request.go | 42 ++++++++++++++++- pkg/protocols/http/request.go | 14 +++++- pkg/protocols/http/request_test.go | 72 +++++++++++++++++++++++++++++ pkg/protocols/protocols.go | 3 ++ pkg/templates/compile.go | 11 ++++- pkg/templates/preprocessors.go | 14 +++--- 8 files changed, 154 insertions(+), 10 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 309c1c951..5598e774a 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -439,6 +439,11 @@ func (r *Runner) RunEnumeration() error { Parser: r.parser, } + if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" { + // Go StdLib style experimental/debug feature switch + executorOpts.ExportReqURLPattern = true + } + if len(r.options.SecretsFile) > 0 && !r.options.Validate { authTmplStore, err := GetAuthTmplStore(*r.options, r.catalog, executorOpts) if err != nil { diff --git a/pkg/output/output.go b/pkg/output/output.go index 6899d6cf1..4a85c3248 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -171,6 +171,9 @@ type ResultEvent struct { // IssueTrackers is the metadata for issue trackers IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"` + // ReqURLPattern when enabled contains base URL pattern that was used to generate the request + // must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true + ReqURLPattern string `json:"req_url_pattern,omitempty"` FileToIndexPosition map[string]int `json:"-"` Error string `json:"error,omitempty"` diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index 7408925a2..bae9a37af 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -32,6 +32,10 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) +const ( + ReqURLPatternKey = "req_url_pattern" +) + // ErrEvalExpression var ( ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions") @@ -48,6 +52,36 @@ type generatedRequest struct { dynamicValues map[string]interface{} interactshURLs []string customCancelFunction context.CancelFunc + // requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash) + // ex: {{BaseURL}}/api/exp?param={{randstr}} + requestURLPattern string +} + +// setReqURLPattern sets the url request pattern for the generated request +func (gr *generatedRequest) setReqURLPattern(reqURLPattern string) { + data := strings.Split(reqURLPattern, "\n") + if len(data) > 1 { + reqURLPattern = strings.TrimSpace(data[0]) + // this is raw request (if it has 3 parts after strings.Fields then its valid only use 2nd part) + parts := strings.Fields(reqURLPattern) + if len(parts) >= 3 { + // remove first and last and use all in between + parts = parts[1 : len(parts)-1] + reqURLPattern = strings.Join(parts, " ") + } + } else { + reqURLPattern = strings.TrimSpace(reqURLPattern) + } + + // now urlRequestPattern is generated replace preprocessor values with actual placeholders + // that were used (these are called generated 'constants' and contains {{}} in var name) + for k, v := range gr.original.options.Constants { + if strings.HasPrefix(k, "{{") && strings.HasSuffix(k, "}}") { + // this takes care of all preprocessors ( currently we have randstr and its variations) + reqURLPattern = strings.ReplaceAll(reqURLPattern, fmt.Sprint(v), k) + } + } + gr.requestURLPattern = reqURLPattern } // ApplyAuth applies the auth provider to the generated request @@ -96,7 +130,13 @@ func (r *requestGenerator) Total() int { // Make creates a http request for the provided input. // It returns ErrNoMoreRequests as error when all the requests have been exhausted. -func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { +func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (gr *generatedRequest, err error) { + origReqData := reqData + defer func() { + if gr != nil { + gr.setReqURLPattern(origReqData) + } + }() // value of `reqData` depends on the type of request specified in template // 1. If request is raw request = reqData contains raw request (i.e http request dump) // 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 0202f20e8..fbe5585f4 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -495,8 +495,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if event == nil { event := &output.InternalWrappedEvent{ InternalEvent: map[string]interface{}{ - "template-id": request.options.TemplateID, - "host": input.MetaInput.Input, + "template-id": request.options.TemplateID, + "host": input.MetaInput.Input, + ReqURLPatternKey: generatedRequest.requestURLPattern, }, } if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() { @@ -515,6 +516,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if event.InternalEvent["host"] == nil { event.InternalEvent["host"] = input.MetaInput.Input } + event.InternalEvent[ReqURLPatternKey] = generatedRequest.requestURLPattern }() request.setCustomHeaders(generatedRequest) @@ -843,6 +845,14 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ event.UsesInteractsh = true } + // if requrlpattern is enabled, only then it is reflected in result event else it is empty string + // consult @Ice3man543 before changing this logic (context: vuln_hash) + if request.options.ExportReqURLPattern { + for _, v := range event.Results { + v.ReqURLPattern = generatedRequest.requestURLPattern + } + } + responseContentType := respChain.Response().Header.Get("Content-Type") isResponseTruncated := request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize dumpResponse(event, request, respChain.FullResponse().Bytes(), formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input) diff --git a/pkg/protocols/http/request_test.go b/pkg/protocols/http/request_test.go index 3d9697125..cd8e860b0 100644 --- a/pkg/protocols/http/request_test.go +++ b/pkg/protocols/http/request_test.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" ) @@ -184,3 +185,74 @@ func TestDisableTE(t *testing.T) { require.NotNil(t, finalEvent, "could not get event output from request") require.Equal(t, 2, matchCount, "could not get correct match count") } + +// consult @Ice3man543 before making any breaking changes to this test (context: vuln_hash) +func TestReqURLPattern(t *testing.T) { + options := testutils.DefaultOptions + + // assume this was a preprocessor + // {{randstr}} => 2eNU2kbrOcUDzhnUL1RGvSo1it7 + testutils.Init(options) + templateID := "testing-http" + request := &Request{ + ID: templateID, + Raw: []string{ + `GET /{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data=2eNU2kbrOcUDzhnUL1RGvSo1it7 HTTP/1.1 + Host: {{Hostname}} + User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + Accept-Language: en-US,en;q=0.5 + `, + }, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{{ + Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher}, + DSL: []string{"true"}, + }}, + }, + IterateAll: true, + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // always return 200 + w.WriteHeader(200) + _, _ = w.Write([]byte(`match`)) + })) + defer ts.Close() + + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + client, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress)) + executerOpts.Interactsh = client + defer client.Close() + executerOpts.ExportReqURLPattern = true + + // this is how generated constants are added to template + // generated constants are preprocessors that are executed while loading once + executerOpts.Constants = map[string]interface{}{ + "{{randstr}}": "2eNU2kbrOcUDzhnUL1RGvSo1it7", + } + + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + var finalEvent *output.InternalWrappedEvent + var matchCount int + t.Run("test", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + ctxArgs := contextargs.NewWithInput(ts.URL) + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { + if event.OperatorsResult != nil && event.OperatorsResult.Matched { + matchCount++ + } + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 1, matchCount, "could not get correct match count") + require.NotEmpty(t, finalEvent.Results[0].ReqURLPattern, "could not get req url pattern") + require.Equal(t, `/{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data={{randstr}}`, finalEvent.Results[0].ReqURLPattern) +} diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 5cdc2a50d..f9d906420 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -123,6 +123,9 @@ type ExecutorOptions struct { //TemporaryDirectory is the directory to store temporary files TemporaryDirectory string Parser parser.Parser + // ExportReqURLPattern exports the request URL pattern + // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request + ExportReqURLPattern bool } // GetThreadsForPayloadRequests returns the number of threads to use as default for diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index f831485d8..eda128c72 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp" "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec" @@ -295,14 +296,22 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option SignatureStats[Unsigned].Add(1) } + generatedConstants := map[string]interface{}{} // ==== execute preprocessors ====== for _, v := range allPreprocessors { - data = v.Process(data) + var replaced map[string]interface{} + data, replaced = v.ProcessNReturnData(data) + // preprocess kind of act like a constant and are generated while loading + // and stay constant for the template lifecycle + generatedConstants = generators.MergeMaps(generatedConstants, replaced) } reParsed, err := parseTemplate(data, options) if err != nil { return nil, err } + // add generated constants to constants map and executer options + reParsed.Constants = generators.MergeMaps(reParsed.Constants, generatedConstants) + reParsed.Options.Constants = reParsed.Constants reParsed.Verified = isVerified return reParsed, nil } diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go index d7e5864d6..054a0bc41 100644 --- a/pkg/templates/preprocessors.go +++ b/pkg/templates/preprocessors.go @@ -10,7 +10,7 @@ import ( type Preprocessor interface { // Process processes the data and returns the processed data. - Process(data []byte) []byte + ProcessNReturnData(data []byte) ([]byte, map[string]interface{}) // Exists check if the preprocessor exists in the data. Exists(data []byte) bool } @@ -39,10 +39,10 @@ var _ Preprocessor = &randStrPreprocessor{} type randStrPreprocessor struct{} -// Process processes the data and returns the processed data. -func (r *randStrPreprocessor) Process(data []byte) []byte { +// ProcessNReturnData processes the data and returns the key-value pairs of generated/replaced data. +func (r *randStrPreprocessor) ProcessNReturnData(data []byte) ([]byte, map[string]interface{}) { foundMap := make(map[string]struct{}) - + dataMap := make(map[string]interface{}) for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) { if len(expression) != 2 { continue @@ -57,10 +57,12 @@ func (r *randStrPreprocessor) Process(data []byte) []byte { } foundMap[value] = struct{}{} if strings.EqualFold(value, "randstr") || strings.HasPrefix(value, "randstr_") { - data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(ksuid.New().String())) + randStr := ksuid.New().String() + data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(randStr)) + dataMap[expression[0]] = randStr } } - return data + return data, dataMap } // Exists check if the preprocessor exists in the data.