Merge branch 'dev' into deserialization-helpers

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15-alpine as build-env
+FROM golang:1.16.6-alpine as build-env
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
 FROM alpine:latest

v2/pkg/catalog/loader/loader.go

@@ -38,6 +38,8 @@ type Store struct {
 
 	templates []*templates.Template
 	workflows []*templates.Template
+
+	preprocessor templates.Preprocessor
 }
 
 // New creates a new template store based on provided configuration
@@ -76,6 +78,11 @@ func (s *Store) Workflows() []*templates.Template {
 	return s.workflows
 }
 
+// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
+func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
+	s.preprocessor = preprocessor
+}
+
 // Load loads all the templates from a store, performs filtering and returns
 // the complete compiled templates for a nuclei execution configuration.
 func (s *Store) Load() {
@@ -105,7 +112,7 @@ func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
 			gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
 			continue
 		}
-		_, err = templates.Parse(k, s.config.ExecutorOptions)
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
 		if err != nil {
 			if strings.Contains(err.Error(), "cannot create template executer") {
 				continue
@@ -129,7 +136,7 @@ func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
 			notErrored = false
 			gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
 		}
-		_, err = templates.Parse(k, s.config.ExecutorOptions)
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
 		if err != nil {
 			if strings.Contains(err.Error(), "cannot create template executer") {
 				continue
@@ -156,7 +163,7 @@ func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
 			gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
 		}
 		if loaded {
-			parsed, err := templates.Parse(k, s.config.ExecutorOptions)
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
 			if err != nil {
 				gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
 			} else if parsed != nil {
@@ -179,7 +186,7 @@ func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
 			gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
 		}
 		if loaded {
-			parsed, err := templates.Parse(k, s.config.ExecutorOptions)
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
 			if err != nil {
 				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
 			} else if parsed != nil {

v2/pkg/operators/common/dsl/dsl.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"html"
 	"math"
@@ -29,113 +30,88 @@ const (
 	withMaxRandArgsSize  = withCutSetArgsSize
 )
 
-// HelperFunctions contains the dsl helper functions
+var functions = map[string]govaluate.ExpressionFunction{
-func HelperFunctions() map[string]govaluate.ExpressionFunction {
+	"len": func(args ...interface{}) (interface{}, error) {
-	functions := make(map[string]govaluate.ExpressionFunction)
-
-	functions["len"] = func(args ...interface{}) (interface{}, error) {
 		length := len(types.ToString(args[0]))
 		return float64(length), nil
-	}
+	},
-
+	"toupper": func(args ...interface{}) (interface{}, error) {
-	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
 		return strings.ToUpper(types.ToString(args[0])), nil
-	}
+	},
-
+	"tolower": func(args ...interface{}) (interface{}, error) {
-	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
 		return strings.ToLower(types.ToString(args[0])), nil
-	}
+	},
-
+	"replace": func(args ...interface{}) (interface{}, error) {
-	functions["replace"] = func(args ...interface{}) (interface{}, error) {
 		return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
-	}
+	},
-
+	"replace_regex": func(args ...interface{}) (interface{}, error) {
-	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(types.ToString(args[1]))
 		if err != nil {
 			return nil, err
 		}
 		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
-	}
+	},
-
+	"trim": func(args ...interface{}) (interface{}, error) {
-	functions["trim"] = func(args ...interface{}) (interface{}, error) {
 		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
-	}
+	},
-
+	"trimleft": func(args ...interface{}) (interface{}, error) {
-	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
+	},
-
+	"trimright": func(args ...interface{}) (interface{}, error) {
-	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
+	},
-
+	"trimspace": func(args ...interface{}) (interface{}, error) {
-	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSpace(types.ToString(args[0])), nil
-	}
+	},
-
+	"trimprefix": func(args ...interface{}) (interface{}, error) {
-	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
+	},
-
+	"trimsuffix": func(args ...interface{}) (interface{}, error) {
-	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
+	},
-
+	"reverse": func(args ...interface{}) (interface{}, error) {
-	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
 		return reverseString(types.ToString(args[0])), nil
-	}
+	},
-
 	// encoding
-	functions["base64"] = func(args ...interface{}) (interface{}, error) {
+	"base64": func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 
 		return sEnc, nil
-	}
+	},
-
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
-	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
+	"base64_py": func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 		return deserialization.InsertInto(sEnc, 76, '\n'), nil
-	}
+	},
-
+	"base64_decode": func(args ...interface{}) (interface{}, error) {
-	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
 		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
-	}
+	},
-
+	"url_encode": func(args ...interface{}) (interface{}, error) {
-	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
 		return url.PathEscape(types.ToString(args[0])), nil
-	}
+	},
-
+	"url_decode": func(args ...interface{}) (interface{}, error) {
-	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
 		return url.PathUnescape(types.ToString(args[0]))
-	}
+	},
-
+	"hex_encode": func(args ...interface{}) (interface{}, error) {
-	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
 		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
-	}
+	},
-
+	"hex_decode": func(args ...interface{}) (interface{}, error) {
-	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
 		hx, _ := hex.DecodeString(types.ToString(args[0]))
 		return string(hx), nil
-	}
+	},
-
+	"html_escape": func(args ...interface{}) (interface{}, error) {
-	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
 		return html.EscapeString(types.ToString(args[0])), nil
-	}
+	},
-
+	"html_unescape": func(args ...interface{}) (interface{}, error) {
-	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
 		return html.UnescapeString(types.ToString(args[0])), nil
-	}
+	},
-
 	// hashing
-	functions["md5"] = func(args ...interface{}) (interface{}, error) {
+	"md5": func(args ...interface{}) (interface{}, error) {
 		hash := md5.Sum([]byte(types.ToString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
-	}
+	},
-
+	"sha256": func(args ...interface{}) (interface{}, error) {
-	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
 		_, err := h.Write([]byte(types.ToString(args[0])))
 
@@ -143,9 +119,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
-	}
+	},
-
+	"sha1": func(args ...interface{}) (interface{}, error) {
-	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
 		_, err := h.Write([]byte(types.ToString(args[0])))
 
@@ -153,27 +128,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
-	}
+	},
-
+	"mmh3": func(args ...interface{}) (interface{}, error) {
-	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
 		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
-	}
+	},
-
 	// search
-	functions["contains"] = func(args ...interface{}) (interface{}, error) {
+	"contains": func(args ...interface{}) (interface{}, error) {
 		return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
+	},
-
+	"regex": func(args ...interface{}) (interface{}, error) {
-	functions["regex"] = func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(types.ToString(args[0]))
 		if err != nil {
 			return nil, err
 		}
 		return compiled.MatchString(types.ToString(args[1])), nil
-	}
+	},
-
 	// random generators
-	functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
+	"rand_char": func(args ...interface{}) (interface{}, error) {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
@@ -184,9 +155,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return chars[rand.Intn(len(chars))], nil
-	}
+	},
-
+	"rand_base": func(args ...interface{}) (interface{}, error) {
-	functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		base := letters + numbers
@@ -202,9 +172,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		base = trimAll(base, bad)
 		return randSeq(base, l), nil
-	}
+	},
-
+	"rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) {
-	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := letters + numbers
@@ -217,9 +186,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
+	},
-
+	"rand_text_alpha": func(args ...interface{}) (interface{}, error) {
-	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := letters
@@ -232,9 +200,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
+	},
-
+	"rand_text_numeric": func(args ...interface{}) (interface{}, error) {
-	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := numbers
@@ -247,9 +214,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
+	},
-
+	"rand_int": func(args ...interface{}) (interface{}, error) {
-	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
 		min := 0
 		max := math.MaxInt32
 
@@ -260,17 +226,15 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			max = args[1].(int)
 		}
 		return rand.Intn(max-min) + min, nil
-	}
+	},
-
 	// Time Functions
-	functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
+	"waitfor": func(args ...interface{}) (interface{}, error) {
 		seconds := args[0].(float64)
 		time.Sleep(time.Duration(seconds) * time.Second)
 		return true, nil
-	}
+	},
-
 	// deserialization Functions
-	functions["generate_java_gadget"] = func(args ...interface{}) (interface{}, error) {
+  "generate_java_gadget": func(args ...interface{}) (interface{}, error) {
 		gadget := args[0].(string)
 		cmd := args[1].(string)
 
@@ -280,10 +244,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
 		return data, nil
-	}
+	},
+}
+
+// HelperFunctions returns the dsl helper functions
+func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	return functions
 }
 
+// AddHelperFunction allows creation of additiona helper functions to be supported with templates
+func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
+	if _, ok := functions[key]; !ok {
+		functions[key] = value
+		return nil
+	}
+	return errors.New("duplicate helper function key defined")
+}
+
 func reverseString(s string) string {
 	runes := []rune(s)
 	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {

v2/pkg/protocols/dns/dns.go

@@ -118,6 +118,8 @@ func questionTypeToInt(questionType string) uint16 {
 		question = dns.TypeMX
 	case "TXT":
 		question = dns.TypeTXT
+	case "DS":
+		question = dns.TypeDS
 	case "AAAA":
 		question = dns.TypeAAAA
 	}

v2/pkg/protocols/http/request.go

@@ -263,19 +263,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 		err           error
 	)
 
-	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
-	if !request.original.Race {
-		dumpedRequest, err = dump(request, reqURL)
-		if err != nil {
-			return err
-		}
-
-		if r.options.Options.Debug || r.options.Options.DebugRequests {
-			gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
-			gologger.Print().Msgf("%s", string(dumpedRequest))
-		}
-	}
-
 	var formedURL string
 	var hostname string
 	timeStart := time.Now()
@@ -314,6 +301,20 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 			resp, err = r.httpClient.Do(request.request)
 		}
 	}
+
+	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
+	if !request.original.Race {
+		dumpedRequest, err = dump(request, reqURL)
+		if err != nil {
+			return err
+		}
+
+		if r.options.Options.Debug || r.options.Options.DebugRequests {
+			gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
+			gologger.Print().Msgf("%s", string(dumpedRequest))
+		}
+	}
+
 	if resp == nil {
 		err = errors.New("no response got for request")
 	}

v2/pkg/templates/compile.go

@@ -18,7 +18,7 @@ import (
 
 // Parse parses a yaml request template file
 //nolint:gocritic // this cannot be passed by pointer
-func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error) {
+func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
 	template := &Template{}
 
 	f, err := os.Open(filePath)
@@ -33,6 +33,10 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
 	}
 
 	data = template.expandPreprocessors(data)
+	if preprocessor != nil {
+		data = preprocessor.Process(data)
+	}
+
 	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
 	if err != nil {
 		return nil, err
@@ -63,7 +67,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
 		if err != nil {
 			return nil, errors.Wrap(err, "could not create workflow loader")
 		}
-		compileWorkflow(&options, compiled, loader)
+		compileWorkflow(preprocessor, &options, compiled, loader)
 		template.CompiledWorkflow = compiled
 		template.CompiledWorkflow.Options = &options
 	}

v2/pkg/templates/preprocessors.go

@@ -8,6 +8,10 @@ import (
 	"github.com/segmentio/ksuid"
 )
 
+type Preprocessor interface {
+	Process(data []byte) []byte
+}
+
 var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`)
 
 // expandPreprocessors expands the pre-processors if any for a template data.

v2/pkg/templates/workflows.go

@@ -8,9 +8,9 @@ import (
 )
 
 // compileWorkflow compiles the workflow for execution
-func compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Workflow, loader compile.WorkflowLoader) {
+func compileWorkflow(preprocessor Preprocessor, options *protocols.ExecuterOptions, workflow *workflows.Workflow, loader compile.WorkflowLoader) {
 	for _, workflow := range workflow.Workflows {
-		if err := parseWorkflow(workflow, options, loader); err != nil {
+		if err := parseWorkflow(preprocessor, workflow, options, loader); err != nil {
 			gologger.Warning().Msgf("Could not parse workflow: %v\n", err)
 			continue
 		}
@@ -18,24 +18,24 @@ func compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Wor
 }
 
 // parseWorkflow parses and compiles all templates in a workflow recursively
-func parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader compile.WorkflowLoader) error {
+func parseWorkflow(preprocessor Preprocessor, workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader compile.WorkflowLoader) error {
 	shouldNotValidate := false
 
 	if len(workflow.Subtemplates) > 0 || len(workflow.Matchers) > 0 {
 		shouldNotValidate = true
 	}
-	if err := parseWorkflowTemplate(workflow, options, loader, shouldNotValidate); err != nil {
+	if err := parseWorkflowTemplate(workflow, preprocessor, options, loader, shouldNotValidate); err != nil {
 		return err
 	}
 	for _, subtemplates := range workflow.Subtemplates {
-		if err := parseWorkflow(subtemplates, options, loader); err != nil {
+		if err := parseWorkflow(preprocessor, subtemplates, options, loader); err != nil {
 			gologger.Warning().Msgf("Could not parse workflow: %v\n", err)
 			continue
 		}
 	}
 	for _, matcher := range workflow.Matchers {
 		for _, subtemplates := range matcher.Subtemplates {
-			if err := parseWorkflow(subtemplates, options, loader); err != nil {
+			if err := parseWorkflow(preprocessor, subtemplates, options, loader); err != nil {
 				gologger.Warning().Msgf("Could not parse workflow: %v\n", err)
 				continue
 			}
@@ -45,7 +45,7 @@ func parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.Exec
 }
 
 // parseWorkflowTemplate parses a workflow template creating an executer
-func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader compile.WorkflowLoader, noValidate bool) error {
+func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Preprocessor, options *protocols.ExecuterOptions, loader compile.WorkflowLoader, noValidate bool) error {
 	var paths []string
 
 	if len(workflow.Tags) > 0 {
@@ -68,7 +68,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protoc
 			Interactsh:   options.Interactsh,
 			ProjectFile:  options.ProjectFile,
 		}
-		template, err := Parse(path, opts)
+		template, err := Parse(path, preprocessor, opts)
 		if err != nil {
 			gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err)
 			continue