Optimize template validation

v2/internal/runner/options.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
@@ -91,6 +92,11 @@ func validateOptions(options *types.Options) error {
 	if err != nil {
 		return err
 	}
+
+	if options.Validate {
+		validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
+	}
+
 	return nil
 }
 
@@ -149,3 +155,21 @@ func loadResolvers(options *types.Options) {
 		}
 	}
 }
+
+func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
+	allGivenTemplatePaths := append(templatePaths, workflowPaths...)
+
+	for _, templatePath := range allGivenTemplatePaths {
+		if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
+			fileInfo, err := os.Stat(templatePath)
+			if err == nil && fileInfo.IsDir() {
+				relativizedPath, err2 := filepath.Rel(templatesDirectory, templatePath)
+				if err2 != nil || (len(relativizedPath) >= 2 && relativizedPath[:2] == "..") {
+					gologger.Warning().Msgf("The given path (%s) is outside the default template directory path (%s)! "+
+						"Referenced sub-templates with relative paths in workflows will be resolved against the default template directory.", templatePath, templatesDirectory)
+					break
+				}
+			}
+		}
+	}
+}

v2/internal/runner/runner.go

@@ -340,8 +340,8 @@ func (r *Runner) RunEnumeration() error {
 		return errors.Wrap(err, "could not load templates from config")
 	}
 	if r.options.Validate {
-		if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) {
+		if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
-			return errors.New("an error occurred during templates validation")
+			return err
 		}
 		gologger.Info().Msgf("All templates validated successfully\n")
 		return nil // exit

v2/pkg/catalog/loader/loader.go

@@ -1,6 +1,7 @@
 package loader
 
 import (
+	"errors"
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
@@ -93,49 +94,66 @@ func (store *Store) Load() {
 
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
-func (store *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
+func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
 	templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
 	workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
 
 	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
 	filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
 
-	notErrored := true
+	if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) {
-	errorValidationFunc := func(message string, template string, err error) {
+		return nil
-		if strings.Contains(err.Error(), "cannot create template executer") {
+	}
-			return
+	return errors.New("an error occurred during templates validation")
+}
+
+func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
+	areWorkflowsValid := true
+	for workflowPath := range filteredWorkflowPaths {
+		if _, err := parsers.LoadWorkflow(workflowPath, store.tagFilter); err != nil {
+			if isParsingError("Error occurred loading workflow %s: %s\n", workflowPath, err) {
+				areWorkflowsValid = false
+				continue
+			}
+		}
+
+		if _, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions); err != nil {
+			if isParsingError("Error occurred parsing workflow %s: %s\n", workflowPath, err) {
+				areWorkflowsValid = false
+			}
+		}
+	}
+	return areWorkflowsValid
+}
+
+func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
+	areTemplatesValid := true
+	for templatePath := range filteredTemplatePaths {
+		if _, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil); err != nil {
+			if isParsingError("Error occurred loading template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
+				continue
+			}
+		}
+
+		if _, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions); err != nil {
+			if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
+			}
+		}
+	}
+	return areTemplatesValid
+}
+
+func isParsingError(message string, template string, err error) bool {
+	if strings.Contains(err.Error(), templates.TemplateExecuterCreationErrorMessage) {
+		return false
 	}
 	if err == filter.ErrExcluded {
-			return
+		return false
 	}
-		notErrored = false
 	gologger.Error().Msgf(message, template, err)
-	}
+	return true
-	for templatePath := range filteredTemplatePaths {
-		_, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
-		if err != nil {
-			errorValidationFunc("Error occurred loading template %s: %s\n", templatePath, err)
-			continue
-		}
-		_, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
-		if err != nil {
-			errorValidationFunc("Error occurred parsing template %s: %s\n", templatePath, err)
-			continue
-		}
-	}
-	for workflowPath := range filteredWorkflowPaths {
-		_, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
-		if err != nil {
-			errorValidationFunc("Error occurred loading workflow %s: %s\n", workflowPath, err)
-			continue
-		}
-		_, err = templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
-		if err != nil {
-			errorValidationFunc("Error occurred parsing workflow %s: %s\n", workflowPath, err)
-			continue
-		}
-	}
-	return notErrored
 }
 
 // LoadTemplates takes a list of templates and returns paths for them

v2/pkg/parsers/parser.go

@@ -87,8 +87,19 @@ func validateMandatoryInfoFields(info *model.Info) error {
 
 var fieldErrorRegexp = regexp.MustCompile(`not found in`)
 
+var parsedTemplatesCache = make(map[string]*parsedTemplateErrHolder, 2500)
+
+type parsedTemplateErrHolder struct {
+	template *templates.Template
+	err      error
+}
+
 // ParseTemplate parses a template and returns a *templates.Template structure
 func ParseTemplate(templatePath string) (*templates.Template, error) {
+	if value, found := parsedTemplatesCache[templatePath]; found {
+		return value.template, value.err
+	}
+
 	f, err := os.Open(templatePath)
 	if err != nil {
 		return nil, err
@@ -102,12 +113,15 @@ func ParseTemplate(templatePath string) (*templates.Template, error) {
 
 	template := &templates.Template{}
 	err = yaml.UnmarshalStrict(data, template)
+
 	if err != nil {
 		if fieldErrorRegexp.MatchString(err.Error()) {
 			gologger.Warning().Msgf("Unrecognized fields in template %s: %s", templatePath, err)
-			return template, nil
+			parsedTemplatesCache[templatePath] = &parsedTemplateErrHolder{template: template, err: err}
+			return template, err
 		}
 		return nil, err
 	}
+	parsedTemplatesCache[templatePath] = &parsedTemplateErrHolder{template: template, err: nil}
 	return template, nil
 }

v2/pkg/parsers/workflow_loader.go

@@ -29,13 +29,21 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
 	return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
 }
 
+var loadedWorkflowTemplateCache = make(map[string]struct{})
+
 func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
 	includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
 	templatePathMap := w.pathFilter.Match(includedTemplates)
 
 	loadedTemplates := make([]string, 0, len(templatePathMap))
 	for templatePath := range templatePathMap {
+
+		if _, found := loadedWorkflowTemplateCache[templatePath]; found {
+			continue
+		}
+
 		loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
+		loadedWorkflowTemplateCache[templatePath] = struct{}{}
 		if err != nil {
 			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
 		} else if loaded {

v2/pkg/templates/compile.go

@@ -17,9 +17,18 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 )
 
+const TemplateExecuterCreationErrorMessage = "cannot create template executer"
+
+var parsedTemplatesCache = make(map[string]*Template, 2500)
+
 // Parse parses a yaml request template file
 //nolint:gocritic // this cannot be passed by pointer
+// TODO make sure reading from the disk the template parsing happens once: see parsers.ParseTemplate vs templates.Parse
 func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
+	if value, found := parsedTemplatesCache[filePath]; found {
+		return value, nil
+	}
+
 	template := &Template{}
 
 	f, err := os.Open(filePath)
@@ -127,8 +136,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 		template.TotalRequests += template.Executer.Requests()
 	}
 	if template.Executer == nil && template.CompiledWorkflow == nil {
-		return nil, errors.New("cannot create template executer")
+		return nil, errors.New(TemplateExecuterCreationErrorMessage)
 	}
 	template.Path = filePath
+
+	parsedTemplatesCache[filePath] = template
 	return template, nil
 }