req_url_pattern for vuln_hash calculation + unit test (#4964)

internal/runner/runner.go

@@ -439,6 +439,11 @@ func (r *Runner) RunEnumeration() error {
 		Parser:             r.parser,
 	}
 
+	if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" {
+		// Go StdLib style experimental/debug feature switch
+		executorOpts.ExportReqURLPattern = true
+	}
+
 	if len(r.options.SecretsFile) > 0 && !r.options.Validate {
 		authTmplStore, err := GetAuthTmplStore(*r.options, r.catalog, executorOpts)
 		if err != nil {

pkg/output/output.go

@@ -171,6 +171,9 @@ type ResultEvent struct {
 
 	// IssueTrackers is the metadata for issue trackers
 	IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"`
+	// ReqURLPattern when enabled contains base URL pattern that was used to generate the request
+	// must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true
+	ReqURLPattern string `json:"req_url_pattern,omitempty"`
 
 	FileToIndexPosition map[string]int `json:"-"`
 	Error               string         `json:"error,omitempty"`

pkg/protocols/http/build_request.go

@@ -32,6 +32,10 @@ import (
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
+const (
+	ReqURLPatternKey = "req_url_pattern"
+)
+
 // ErrEvalExpression
 var (
 	ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions")
@@ -48,6 +52,36 @@ type generatedRequest struct {
 	dynamicValues        map[string]interface{}
 	interactshURLs       []string
 	customCancelFunction context.CancelFunc
+	// requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash)
+	// ex: {{BaseURL}}/api/exp?param={{randstr}}
+	requestURLPattern string
+}
+
+// setReqURLPattern sets the url request pattern for the generated request
+func (gr *generatedRequest) setReqURLPattern(reqURLPattern string) {
+	data := strings.Split(reqURLPattern, "\n")
+	if len(data) > 1 {
+		reqURLPattern = strings.TrimSpace(data[0])
+		// this is raw request (if it has 3 parts after strings.Fields then its valid only use 2nd part)
+		parts := strings.Fields(reqURLPattern)
+		if len(parts) >= 3 {
+			// remove first and last and use all in between
+			parts = parts[1 : len(parts)-1]
+			reqURLPattern = strings.Join(parts, " ")
+		}
+	} else {
+		reqURLPattern = strings.TrimSpace(reqURLPattern)
+	}
+
+	// now urlRequestPattern is generated replace preprocessor values with actual placeholders
+	// that were used (these are called generated 'constants' and contains {{}} in var name)
+	for k, v := range gr.original.options.Constants {
+		if strings.HasPrefix(k, "{{") && strings.HasSuffix(k, "}}") {
+			// this takes care of all preprocessors ( currently we have randstr and its variations)
+			reqURLPattern = strings.ReplaceAll(reqURLPattern, fmt.Sprint(v), k)
+		}
+	}
+	gr.requestURLPattern = reqURLPattern
 }
 
 // ApplyAuth applies the auth provider to the generated request
@@ -96,7 +130,13 @@ func (r *requestGenerator) Total() int {
 
 // Make creates a http request for the provided input.
 // It returns ErrNoMoreRequests as error when all the requests have been exhausted.
-func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
+func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (gr *generatedRequest, err error) {
+	origReqData := reqData
+	defer func() {
+		if gr != nil {
+			gr.setReqURLPattern(origReqData)
+		}
+	}()
 	// value of `reqData` depends on the type of request specified in template
 	// 1. If request is raw request =  reqData contains raw request (i.e http request dump)
 	// 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path

pkg/protocols/http/request.go

@@ -495,8 +495,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if event == nil {
 			event := &output.InternalWrappedEvent{
 				InternalEvent: map[string]interface{}{
-					"template-id": request.options.TemplateID,
-					"host":        input.MetaInput.Input,
+					"template-id":    request.options.TemplateID,
+					"host":           input.MetaInput.Input,
+					ReqURLPatternKey: generatedRequest.requestURLPattern,
 				},
 			}
 			if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
@@ -515,6 +516,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if event.InternalEvent["host"] == nil {
 			event.InternalEvent["host"] = input.MetaInput.Input
 		}
+		event.InternalEvent[ReqURLPatternKey] = generatedRequest.requestURLPattern
 	}()
 
 	request.setCustomHeaders(generatedRequest)
@@ -843,6 +845,14 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			event.UsesInteractsh = true
 		}
 
+		// if requrlpattern is enabled, only then it is reflected in result event else it is empty string
+		// consult @Ice3man543 before changing this logic (context: vuln_hash)
+		if request.options.ExportReqURLPattern {
+			for _, v := range event.Results {
+				v.ReqURLPattern = generatedRequest.requestURLPattern
+			}
+		}
+
 		responseContentType := respChain.Response().Header.Get("Content-Type")
 		isResponseTruncated := request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize
 		dumpResponse(event, request, respChain.FullResponse().Bytes(), formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)

pkg/protocols/http/request_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
@@ -184,3 +185,74 @@ func TestDisableTE(t *testing.T) {
 	require.NotNil(t, finalEvent, "could not get event output from request")
 	require.Equal(t, 2, matchCount, "could not get correct match count")
 }
+
+// consult @Ice3man543 before making any breaking changes to this test (context: vuln_hash)
+func TestReqURLPattern(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	// assume this was a preprocessor
+	// {{randstr}} => 2eNU2kbrOcUDzhnUL1RGvSo1it7
+	testutils.Init(options)
+	templateID := "testing-http"
+	request := &Request{
+		ID: templateID,
+		Raw: []string{
+			`GET /{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data=2eNU2kbrOcUDzhnUL1RGvSo1it7 HTTP/1.1
+			Host: {{Hostname}}
+			User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0
+			Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+			Accept-Language: en-US,en;q=0.5
+			`,
+		},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
+				DSL:  []string{"true"},
+			}},
+		},
+		IterateAll: true,
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// always return 200
+		w.WriteHeader(200)
+		_, _ = w.Write([]byte(`match`))
+	}))
+	defer ts.Close()
+
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	client, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress))
+	executerOpts.Interactsh = client
+	defer client.Close()
+	executerOpts.ExportReqURLPattern = true
+
+	// this is how generated constants are added to template
+	// generated constants are preprocessors that are executed while loading once
+	executerOpts.Constants = map[string]interface{}{
+		"{{randstr}}": "2eNU2kbrOcUDzhnUL1RGvSo1it7",
+	}
+
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile network request")
+
+	var finalEvent *output.InternalWrappedEvent
+	var matchCount int
+	t.Run("test", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		ctxArgs := contextargs.NewWithInput(ts.URL)
+		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
+			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+				matchCount++
+			}
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute network request")
+	})
+	require.NotNil(t, finalEvent, "could not get event output from request")
+	require.Equal(t, 1, matchCount, "could not get correct match count")
+	require.NotEmpty(t, finalEvent.Results[0].ReqURLPattern, "could not get req url pattern")
+	require.Equal(t, `/{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data={{randstr}}`, finalEvent.Results[0].ReqURLPattern)
+}

pkg/protocols/protocols.go

@@ -123,6 +123,9 @@ type ExecutorOptions struct {
 	//TemporaryDirectory is the directory to store temporary files
 	TemporaryDirectory string
 	Parser             parser.Parser
+	// ExportReqURLPattern exports the request URL pattern
+	// in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request
+	ExportReqURLPattern bool
 }
 
 // GetThreadsForPayloadRequests returns the number of threads to use as default for

pkg/templates/compile.go

@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec"
@@ -295,14 +296,22 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
 		SignatureStats[Unsigned].Add(1)
 	}
 
+	generatedConstants := map[string]interface{}{}
 	// ==== execute preprocessors ======
 	for _, v := range allPreprocessors {
-		data = v.Process(data)
+		var replaced map[string]interface{}
+		data, replaced = v.ProcessNReturnData(data)
+		// preprocess kind of act like a constant and are generated while loading
+		// and stay constant for the template lifecycle
+		generatedConstants = generators.MergeMaps(generatedConstants, replaced)
 	}
 	reParsed, err := parseTemplate(data, options)
 	if err != nil {
 		return nil, err
 	}
+	// add generated constants to constants map and executer options
+	reParsed.Constants = generators.MergeMaps(reParsed.Constants, generatedConstants)
+	reParsed.Options.Constants = reParsed.Constants
 	reParsed.Verified = isVerified
 	return reParsed, nil
 }

pkg/templates/preprocessors.go

@@ -10,7 +10,7 @@ import (
 
 type Preprocessor interface {
 	// Process processes the data and returns the processed data.
-	Process(data []byte) []byte
+	ProcessNReturnData(data []byte) ([]byte, map[string]interface{})
 	// Exists check if the preprocessor exists in the data.
 	Exists(data []byte) bool
 }
@@ -39,10 +39,10 @@ var _ Preprocessor = &randStrPreprocessor{}
 
 type randStrPreprocessor struct{}
 
-// Process processes the data and returns the processed data.
-func (r *randStrPreprocessor) Process(data []byte) []byte {
+// ProcessNReturnData processes the data and returns the key-value pairs of generated/replaced data.
+func (r *randStrPreprocessor) ProcessNReturnData(data []byte) ([]byte, map[string]interface{}) {
 	foundMap := make(map[string]struct{})
-
+	dataMap := make(map[string]interface{})
 	for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) {
 		if len(expression) != 2 {
 			continue
@@ -57,10 +57,12 @@ func (r *randStrPreprocessor) Process(data []byte) []byte {
 		}
 		foundMap[value] = struct{}{}
 		if strings.EqualFold(value, "randstr") || strings.HasPrefix(value, "randstr_") {
-			data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(ksuid.New().String()))
+			randStr := ksuid.New().String()
+			data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(randStr))
+			dataMap[expression[0]] = randStr
 		}
 	}
-	return data
+	return data, dataMap
 }
 
 // Exists check if the preprocessor exists in the data.