feat(fuzz): eval variables (#6358)

* feat(fuzz): eval vars for rule keys & values

Signed-off-by: Dwi Siswanto <git@dw1.io>

* chore: re-fmt fuzzing/dast errors

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test(fuzz): adds `TestEvaluateVariables`

Signed-off-by: Dwi Siswanto <git@dw1.io>

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>

pkg/fuzz/execute.go

@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
 	"github.com/projectdiscovery/retryablehttp-go"
 	errorutil "github.com/projectdiscovery/utils/errors"
@@ -23,7 +24,7 @@ import (
 )
 
 var (
-	ErrRuleNotApplicable = errorutil.NewWithFmt("rule not applicable : %v")
+	ErrRuleNotApplicable = errorutil.NewWithFmt("rule not applicable: %v")
 )
 
 // IsErrRuleNotApplicable checks if an error is due to rule not applicable
@@ -189,6 +190,33 @@ mainLoop:
 	return nil
 }
 
+// evaluateVars evaluates variables in a string using available executor options
+func (rule *Rule) evaluateVars(input string) (string, error) {
+	if rule.options == nil {
+		return input, nil
+	}
+
+	data := generators.MergeMaps(
+		rule.options.Variables.GetAll(),
+		rule.options.Constants,
+		rule.options.Options.Vars.AsMap(),
+	)
+
+	exprs := expressions.FindExpressions(input, marker.ParenthesisOpen, marker.ParenthesisClose, data)
+
+	err := expressions.ContainsUnresolvedVariables(exprs...)
+	if err != nil {
+		return input, err
+	}
+
+	eval, err := expressions.Evaluate(input, data)
+	if err != nil {
+		return input, err
+	}
+
+	return eval, nil
+}
+
 // evaluateVarsWithInteractsh evaluates the variables with Interactsh URLs and updates them accordingly.
 func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string) (map[string]interface{}, []string) {
 	// Check if Interactsh options are configured
@@ -341,23 +369,47 @@ func (rule *Rule) Compile(generator *generators.PayloadGenerator, options *proto
 	if len(rule.Keys) > 0 {
 		rule.keysMap = make(map[string]struct{})
 	}
+
+	// eval vars in "keys"
 	for _, key := range rule.Keys {
-		rule.keysMap[strings.ToLower(key)] = struct{}{}
+		evaluatedKey, err := rule.evaluateVars(key)
+		if err != nil {
+			return errors.Wrap(err, "could not evaluate key")
+		}
+
+		rule.keysMap[strings.ToLower(evaluatedKey)] = struct{}{}
 	}
+
+	// eval vars in "values"
 	for _, value := range rule.ValuesRegex {
-		compiled, err := regexp.Compile(value)
+		evaluatedValue, err := rule.evaluateVars(value)
+		if err != nil {
+			return errors.Wrap(err, "could not evaluate value regex")
+		}
+
+		compiled, err := regexp.Compile(evaluatedValue)
 		if err != nil {
 			return errors.Wrap(err, "could not compile value regex")
 		}
+
 		rule.valuesRegex = append(rule.valuesRegex, compiled)
 	}
+
+	// eval vars in "keys-regex"
 	for _, value := range rule.KeysRegex {
-		compiled, err := regexp.Compile(value)
+		evaluatedValue, err := rule.evaluateVars(value)
+		if err != nil {
+			return errors.Wrap(err, "could not evaluate key regex")
+		}
+
+		compiled, err := regexp.Compile(evaluatedValue)
 		if err != nil {
 			return errors.Wrap(err, "could not compile key regex")
 		}
+
 		rule.keysRegex = append(rule.keysRegex, compiled)
 	}
+
 	if rule.ruleType != replaceRegexRuleType {
 		if rule.ReplaceRegex != "" {
 			return errors.Errorf("replace-regex is only applicable for replace and replace-regex rule types")
@@ -366,11 +418,19 @@ func (rule *Rule) Compile(generator *generators.PayloadGenerator, options *proto
 		if rule.ReplaceRegex == "" {
 			return errors.Errorf("replace-regex is required for replace-regex rule type")
 		}
-		compiled, err := regexp.Compile(rule.ReplaceRegex)
+
+		evalReplaceRegex, err := rule.evaluateVars(rule.ReplaceRegex)
+		if err != nil {
+			return errors.Wrap(err, "could not evaluate replace regex")
+		}
+
+		compiled, err := regexp.Compile(evalReplaceRegex)
 		if err != nil {
 			return errors.Wrap(err, "could not compile replace regex")
 		}
+
 		rule.replaceRegex = compiled
 	}
+
 	return nil
 }

pkg/fuzz/fuzz_test.go

@@ -3,6 +3,11 @@ package fuzz
 import (
 	"testing"
 
+	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/stretchr/testify/require"
 )
 
@@ -37,3 +42,219 @@ func TestRuleMatchKeyOrValue(t *testing.T) {
 		require.False(t, result, "could not get correct result")
 	})
 }
+
+func TestEvaluateVariables(t *testing.T) {
+	t.Run("keys", func(t *testing.T) {
+		rule := &Rule{
+			Keys: []string{"{{foo_var}}"},
+			Part: "query",
+		}
+
+		// mock
+		templateVars := variables.Variable{
+			InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
+		}
+		templateVars.Set("foo_var", "foo_var_value")
+
+		constants := map[string]interface{}{
+			"const_key": "const_value",
+		}
+
+		options := &types.Options{}
+
+		// runtime vars (to simulate CLI)
+		runtimeVars := goflags.RuntimeMap{}
+		_ = runtimeVars.Set("runtime_key=runtime_value")
+		options.Vars = runtimeVars
+
+		executorOpts := &protocols.ExecutorOptions{
+			Variables: templateVars,
+			Constants: constants,
+			Options:   options,
+		}
+
+		err := rule.Compile(nil, executorOpts)
+		require.NoError(t, err, "could not compile rule")
+
+		result := rule.matchKeyOrValue("foo_var_value", "test_value")
+		require.True(t, result, "should match evaluated variable key")
+
+		result = rule.matchKeyOrValue("{{foo_var}}", "test_value")
+		require.False(t, result, "should not match unevaluated variable key")
+	})
+
+	t.Run("keys-regex", func(t *testing.T) {
+		rule := &Rule{
+			KeysRegex: []string{"^{{foo_var}}"},
+			Part:      "query",
+		}
+
+		templateVars := variables.Variable{
+			InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
+		}
+		templateVars.Set("foo_var", "foo_var_value")
+
+		executorOpts := &protocols.ExecutorOptions{
+			Variables: templateVars,
+			Constants: map[string]interface{}{},
+			Options:   &types.Options{},
+		}
+
+		err := rule.Compile(nil, executorOpts)
+		require.NoError(t, err, "could not compile rule")
+
+		result := rule.matchKeyOrValue("foo_var_value", "test_value")
+		require.True(t, result, "should match evaluated variable in regex")
+
+		result = rule.matchKeyOrValue("other_key", "test_value")
+		require.False(t, result, "should not match non-matching key")
+	})
+
+	t.Run("values-regex", func(t *testing.T) {
+		rule := &Rule{
+			ValuesRegex: []string{"{{foo_var}}"},
+			Part:        "query",
+		}
+
+		templateVars := variables.Variable{
+			InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
+		}
+		templateVars.Set("foo_var", "test_pattern")
+
+		executorOpts := &protocols.ExecutorOptions{
+			Variables: templateVars,
+			Constants: map[string]interface{}{},
+			Options:   &types.Options{},
+		}
+
+		err := rule.Compile(nil, executorOpts)
+		require.NoError(t, err, "could not compile rule")
+
+		result := rule.matchKeyOrValue("test_key", "test_pattern")
+		require.True(t, result, "should match evaluated variable in values regex")
+
+		result = rule.matchKeyOrValue("test_key", "other_value")
+		require.False(t, result, "should not match non-matching value")
+	})
+
+	// complex vars w/ consts and runtime vars
+	t.Run("complex-variables", func(t *testing.T) {
+		rule := &Rule{
+			Keys: []string{"{{template_var}}", "{{const_key}}", "{{runtime_key}}"},
+			Part: "query",
+		}
+
+		templateVars := variables.Variable{
+			InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
+		}
+		templateVars.Set("template_var", "template_value")
+
+		constants := map[string]interface{}{
+			"const_key": "const_value",
+		}
+
+		options := &types.Options{}
+		runtimeVars := goflags.RuntimeMap{}
+		_ = runtimeVars.Set("runtime_key=runtime_value")
+		options.Vars = runtimeVars
+
+		executorOpts := &protocols.ExecutorOptions{
+			Variables: templateVars,
+			Constants: constants,
+			Options:   options,
+		}
+
+		err := rule.Compile(nil, executorOpts)
+		require.NoError(t, err, "could not compile rule")
+
+		result := rule.matchKeyOrValue("template_value", "test")
+		require.True(t, result, "should match template variable")
+
+		result = rule.matchKeyOrValue("const_value", "test")
+		require.True(t, result, "should match constant")
+
+		result = rule.matchKeyOrValue("runtime_value", "test")
+		require.True(t, result, "should match runtime variable")
+
+		result = rule.matchKeyOrValue("{{template_var}}", "test")
+		require.False(t, result, "should not match unevaluated template variable")
+	})
+
+	t.Run("invalid-variables", func(t *testing.T) {
+		rule := &Rule{
+			Keys: []string{"{{nonexistent_var}}"},
+			Part: "query",
+		}
+
+		executorOpts := &protocols.ExecutorOptions{
+			Variables: variables.Variable{
+				InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
+			},
+			Constants: map[string]interface{}{},
+			Options:   &types.Options{},
+		}
+
+		err := rule.Compile(nil, executorOpts)
+		if err != nil {
+			require.Contains(t, err.Error(), "unresolved", "error should mention unresolved variables")
+		} else {
+			result := rule.matchKeyOrValue("some_key", "some_value")
+			require.False(t, result, "should not match when variables are unresolved")
+		}
+	})
+
+	t.Run("evaluateVars-function", func(t *testing.T) {
+		rule := &Rule{}
+
+		templateVars := variables.Variable{
+			InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
+		}
+		templateVars.Set("test_var", "test_value")
+
+		constants := map[string]interface{}{
+			"const_var": "const_value",
+		}
+
+		options := &types.Options{}
+		runtimeVars := goflags.RuntimeMap{}
+		_ = runtimeVars.Set("runtime_var=runtime_value")
+		options.Vars = runtimeVars
+
+		executorOpts := &protocols.ExecutorOptions{
+			Variables: templateVars,
+			Constants: constants,
+			Options:   options,
+		}
+
+		rule.options = executorOpts
+
+		// Test simple var substitution
+		result, err := rule.evaluateVars("{{test_var}}")
+		require.NoError(t, err, "should evaluate template variable")
+		require.Equal(t, "test_value", result, "should return evaluated value")
+
+		// Test constant substitution
+		result, err = rule.evaluateVars("{{const_var}}")
+		require.NoError(t, err, "should evaluate constant")
+		require.Equal(t, "const_value", result, "should return constant value")
+
+		// Test runtime var substitution
+		result, err = rule.evaluateVars("{{runtime_var}}")
+		require.NoError(t, err, "should evaluate runtime variable")
+		require.Equal(t, "runtime_value", result, "should return runtime value")
+
+		// Test mixed content
+		result, err = rule.evaluateVars("prefix-{{test_var}}-suffix")
+		require.NoError(t, err, "should evaluate mixed content")
+		require.Equal(t, "prefix-test_value-suffix", result, "should return mixed evaluated content")
+
+		// Test unresolved var - should either fail during evaluation or return original string
+		result2, err := rule.evaluateVars("{{nonexistent}}")
+		if err != nil {
+			require.Contains(t, err.Error(), "unresolved", "should fail for unresolved variable")
+		} else {
+			// If no error, it should return the original unresolved variable
+			require.Equal(t, "{{nonexistent}}", result2, "should return original string for unresolved variable")
+		}
+	})
+}

pkg/protocols/http/request_fuzz.go

@@ -76,7 +76,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 			if errors.Is(err, ErrMissingVars) {
 				return err
 			}
-			gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err)
+			gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed: %s\n", request.options.TemplateID, err)
 		}
 		return nil
 	}
@@ -103,13 +103,13 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 		// in case of any error, return it
 		if fuzz.IsErrRuleNotApplicable(err) {
 			// log and fail silently
-			gologger.Verbose().Msgf("[%s] fuzz: rule not applicable : %s\n", request.options.TemplateID, err)
+			gologger.Verbose().Msgf("[%s] fuzz: %s\n", request.options.TemplateID, err)
 			return nil
 		}
 		if errors.Is(err, ErrMissingVars) {
 			return err
 		}
-		gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err)
+		gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed: %s\n", request.options.TemplateID, err)
 	}
 	return nil
 }
@@ -158,7 +158,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
 			continue
 		}
 		if fuzz.IsErrRuleNotApplicable(err) {
-			gologger.Verbose().Msgf("[%s] fuzz: rule not applicable : %s\n", request.options.TemplateID, err)
+			gologger.Verbose().Msgf("[%s] fuzz: %s\n", request.options.TemplateID, err)
 			continue
 		}
 		if err == types.ErrNoMoreRequests {
@@ -168,8 +168,9 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
 	}
 
 	if !applicable {
-		return fuzz.ErrRuleNotApplicable.Msgf(fmt.Sprintf("no rule was applicable for this request: %v", input.MetaInput.Input))
+		return fmt.Errorf("no rule was applicable for this request: %v", input.MetaInput.Input)
 	}
+
 	return nil
 }