From 36b1c08edcf8f778658bc6894dcf943a7d0400bd Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 27 Aug 2021 17:06:06 +0300 Subject: [PATCH] Optimize template validation --- v2/internal/runner/options.go | 24 +++++++++ v2/internal/runner/runner.go | 4 +- v2/pkg/catalog/loader/loader.go | 82 +++++++++++++++++++------------ v2/pkg/parsers/parser.go | 16 +++++- v2/pkg/parsers/workflow_loader.go | 8 +++ v2/pkg/templates/compile.go | 13 ++++- 6 files changed, 111 insertions(+), 36 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f60a3b6a6..b72e1a7f1 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -5,6 +5,7 @@ import ( "errors" "net/url" "os" + "path/filepath" "strings" "github.com/projectdiscovery/gologger" @@ -91,6 +92,11 @@ func validateOptions(options *types.Options) error { if err != nil { return err } + + if options.Validate { + validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) + } + return nil } @@ -149,3 +155,21 @@ func loadResolvers(options *types.Options) { } } } + +func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { + allGivenTemplatePaths := append(templatePaths, workflowPaths...) + + for _, templatePath := range allGivenTemplatePaths { + if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { + fileInfo, err := os.Stat(templatePath) + if err == nil && fileInfo.IsDir() { + relativizedPath, err2 := filepath.Rel(templatesDirectory, templatePath) + if err2 != nil || (len(relativizedPath) >= 2 && relativizedPath[:2] == "..") { + gologger.Warning().Msgf("The given path (%s) is outside the default template directory path (%s)! "+ + "Referenced sub-templates with relative paths in workflows will be resolved against the default template directory.", templatePath, templatesDirectory) + break + } + } + } + } +} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 76699f2dd..cc9607ea3 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -340,8 +340,8 @@ func (r *Runner) RunEnumeration() error { return errors.Wrap(err, "could not load templates from config") } if r.options.Validate { - if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) { - return errors.New("an error occurred during templates validation") + if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil { + return err } gologger.Info().Msgf("All templates validated successfully\n") return nil // exit diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 2d2ae364c..4c1c726fc 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -1,6 +1,7 @@ package loader import ( + "errors" "strings" "github.com/projectdiscovery/gologger" @@ -93,49 +94,66 @@ func (store *Store) Load() { // ValidateTemplates takes a list of templates and validates them // erroring out on discovering any faulty templates. -func (store *Store) ValidateTemplates(templatesList, workflowsList []string) bool { +func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error { templatePaths := store.config.Catalog.GetTemplatesPath(templatesList) workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList) filteredTemplatePaths := store.pathFilter.Match(templatePaths) filteredWorkflowPaths := store.pathFilter.Match(workflowPaths) - notErrored := true - errorValidationFunc := func(message string, template string, err error) { - if strings.Contains(err.Error(), "cannot create template executer") { - return - } - if err == filter.ErrExcluded { - return - } - notErrored = false - gologger.Error().Msgf(message, template, err) - } - for templatePath := range filteredTemplatePaths { - _, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil) - if err != nil { - errorValidationFunc("Error occurred loading template %s: %s\n", templatePath, err) - continue - } - _, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) - if err != nil { - errorValidationFunc("Error occurred parsing template %s: %s\n", templatePath, err) - continue - } + if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) { + return nil } + return errors.New("an error occurred during templates validation") +} + +func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool { + areWorkflowsValid := true for workflowPath := range filteredWorkflowPaths { - _, err := parsers.LoadWorkflow(workflowPath, store.tagFilter) - if err != nil { - errorValidationFunc("Error occurred loading workflow %s: %s\n", workflowPath, err) - continue + if _, err := parsers.LoadWorkflow(workflowPath, store.tagFilter); err != nil { + if isParsingError("Error occurred loading workflow %s: %s\n", workflowPath, err) { + areWorkflowsValid = false + continue + } } - _, err = templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions) - if err != nil { - errorValidationFunc("Error occurred parsing workflow %s: %s\n", workflowPath, err) - continue + + if _, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions); err != nil { + if isParsingError("Error occurred parsing workflow %s: %s\n", workflowPath, err) { + areWorkflowsValid = false + } } } - return notErrored + return areWorkflowsValid +} + +func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool { + areTemplatesValid := true + for templatePath := range filteredTemplatePaths { + if _, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil); err != nil { + if isParsingError("Error occurred loading template %s: %s\n", templatePath, err) { + areTemplatesValid = false + continue + } + } + + if _, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions); err != nil { + if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) { + areTemplatesValid = false + } + } + } + return areTemplatesValid +} + +func isParsingError(message string, template string, err error) bool { + if strings.Contains(err.Error(), templates.TemplateExecuterCreationErrorMessage) { + return false + } + if err == filter.ErrExcluded { + return false + } + gologger.Error().Msgf(message, template, err) + return true } // LoadTemplates takes a list of templates and returns paths for them diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 5371aa901..e38f045d9 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -87,8 +87,19 @@ func validateMandatoryInfoFields(info *model.Info) error { var fieldErrorRegexp = regexp.MustCompile(`not found in`) +var parsedTemplatesCache = make(map[string]*parsedTemplateErrHolder, 2500) + +type parsedTemplateErrHolder struct { + template *templates.Template + err error +} + // ParseTemplate parses a template and returns a *templates.Template structure func ParseTemplate(templatePath string) (*templates.Template, error) { + if value, found := parsedTemplatesCache[templatePath]; found { + return value.template, value.err + } + f, err := os.Open(templatePath) if err != nil { return nil, err @@ -102,12 +113,15 @@ func ParseTemplate(templatePath string) (*templates.Template, error) { template := &templates.Template{} err = yaml.UnmarshalStrict(data, template) + if err != nil { if fieldErrorRegexp.MatchString(err.Error()) { gologger.Warning().Msgf("Unrecognized fields in template %s: %s", templatePath, err) - return template, nil + parsedTemplatesCache[templatePath] = &parsedTemplateErrHolder{template: template, err: err} + return template, err } return nil, err } + parsedTemplatesCache[templatePath] = &parsedTemplateErrHolder{template: template, err: nil} return template, nil } diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index e3f9ab8b2..6769f3e20 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -29,13 +29,21 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil } +var loadedWorkflowTemplateCache = make(map[string]struct{}) + func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string { includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory}) templatePathMap := w.pathFilter.Match(includedTemplates) loadedTemplates := make([]string, 0, len(templatePathMap)) for templatePath := range templatePathMap { + + if _, found := loadedWorkflowTemplateCache[templatePath]; found { + continue + } + loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags) + loadedWorkflowTemplateCache[templatePath] = struct{}{} if err != nil { gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err) } else if loaded { diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 653727f37..6c6107cc1 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -17,9 +17,18 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) +const TemplateExecuterCreationErrorMessage = "cannot create template executer" + +var parsedTemplatesCache = make(map[string]*Template, 2500) + // Parse parses a yaml request template file //nolint:gocritic // this cannot be passed by pointer +// TODO make sure reading from the disk the template parsing happens once: see parsers.ParseTemplate vs templates.Parse func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) { + if value, found := parsedTemplatesCache[filePath]; found { + return value, nil + } + template := &Template{} f, err := os.Open(filePath) @@ -127,8 +136,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.TotalRequests += template.Executer.Requests() } if template.Executer == nil && template.CompiledWorkflow == nil { - return nil, errors.New("cannot create template executer") + return nil, errors.New(TemplateExecuterCreationErrorMessage) } template.Path = filePath + + parsedTemplatesCache[filePath] = template return template, nil }