External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 18:05:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize template validation

Browse Source
This commit is contained in:
forgedhallpass 2021-08-27 17:06:06 +03:00
parent 095e78e431
commit 36b1c08edc
6 changed files with 111 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
v2/internal/runner/options.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/projectdiscovery/gologger"
@ -91,6 +92,11 @@ func validateOptions(options *types.Options) error {
if err != nil {
return err
}
if options.Validate {
validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
}
return nil
}
@ -149,3 +155,21 @@ func loadResolvers(options *types.Options) {
}
}
}
func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
allGivenTemplatePaths := append(templatePaths, workflowPaths...)
for _, templatePath := range allGivenTemplatePaths {
if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
fileInfo, err := os.Stat(templatePath)
if err == nil && fileInfo.IsDir() {
relativizedPath, err2 := filepath.Rel(templatesDirectory, templatePath)
if err2 != nil || (len(relativizedPath) >= 2 && relativizedPath[:2] == "..") {
gologger.Warning().Msgf("The given path (%s) is outside the default template directory path (%s)! "+
"Referenced sub-templates with relative paths in workflows will be resolved against the default template directory.", templatePath, templatesDirectory)
break
}
}
}
}
}

4
v2/internal/runner/runner.go
View File

@ -340,8 +340,8 @@ func (r *Runner) RunEnumeration() error {
return errors.Wrap(err, "could not load templates from config")
}
if r.options.Validate {
if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) {
return errors.New("an error occurred during templates validation")
if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
return err
}
gologger.Info().Msgf("All templates validated successfully\n")
return nil // exit

82
v2/pkg/catalog/loader/loader.go
View File

@ -1,6 +1,7 @@
package loader
import (
"errors"
"strings"
"github.com/projectdiscovery/gologger"
@ -93,49 +94,66 @@ func (store *Store) Load() {
// ValidateTemplates takes a list of templates and validates them
// erroring out on discovering any faulty templates.
func (store *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
notErrored := true
errorValidationFunc := func(message string, template string, err error) {
if strings.Contains(err.Error(), "cannot create template executer") {
return
}
if err == filter.ErrExcluded {
return
}
notErrored = false
gologger.Error().Msgf(message, template, err)
}
for templatePath := range filteredTemplatePaths {
_, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
if err != nil {
errorValidationFunc("Error occurred loading template %s: %s\n", templatePath, err)
continue
}
_, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
errorValidationFunc("Error occurred parsing template %s: %s\n", templatePath, err)
continue
}
if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) {
return nil
}
return errors.New("an error occurred during templates validation")
}
func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
areWorkflowsValid := true
for workflowPath := range filteredWorkflowPaths {
_, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
if err != nil {
errorValidationFunc("Error occurred loading workflow %s: %s\n", workflowPath, err)
continue
if _, err := parsers.LoadWorkflow(workflowPath, store.tagFilter); err != nil {
if isParsingError("Error occurred loading workflow %s: %s\n", workflowPath, err) {
areWorkflowsValid = false
continue
}
}
_, err = templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
errorValidationFunc("Error occurred parsing workflow %s: %s\n", workflowPath, err)
continue
if _, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions); err != nil {
if isParsingError("Error occurred parsing workflow %s: %s\n", workflowPath, err) {
areWorkflowsValid = false
}
}
}
return notErrored
return areWorkflowsValid
}
func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
areTemplatesValid := true
for templatePath := range filteredTemplatePaths {
if _, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil); err != nil {
if isParsingError("Error occurred loading template %s: %s\n", templatePath, err) {
areTemplatesValid = false
continue
}
}
if _, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions); err != nil {
if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
areTemplatesValid = false
}
}
}
return areTemplatesValid
}
func isParsingError(message string, template string, err error) bool {
if strings.Contains(err.Error(), templates.TemplateExecuterCreationErrorMessage) {
return false
}
if err == filter.ErrExcluded {
return false
}
gologger.Error().Msgf(message, template, err)
return true
}
// LoadTemplates takes a list of templates and returns paths for them

16
v2/pkg/parsers/parser.go
View File

@ -87,8 +87,19 @@ func validateMandatoryInfoFields(info *model.Info) error {
var fieldErrorRegexp = regexp.MustCompile(`not found in`)
var parsedTemplatesCache = make(map[string]*parsedTemplateErrHolder, 2500)
type parsedTemplateErrHolder struct {
template *templates.Template
err error
}
// ParseTemplate parses a template and returns a *templates.Template structure
func ParseTemplate(templatePath string) (*templates.Template, error) {
if value, found := parsedTemplatesCache[templatePath]; found {
return value.template, value.err
}
f, err := os.Open(templatePath)
if err != nil {
return nil, err
@ -102,12 +113,15 @@ func ParseTemplate(templatePath string) (*templates.Template, error) {
template := &templates.Template{}
err = yaml.UnmarshalStrict(data, template)
if err != nil {
if fieldErrorRegexp.MatchString(err.Error()) {
gologger.Warning().Msgf("Unrecognized fields in template %s: %s", templatePath, err)
return template, nil
parsedTemplatesCache[templatePath] = &parsedTemplateErrHolder{template: template, err: err}
return template, err
}
return nil, err
}
parsedTemplatesCache[templatePath] = &parsedTemplateErrHolder{template: template, err: nil}
return template, nil
}

8
v2/pkg/parsers/workflow_loader.go
View File

@ -29,13 +29,21 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
}
var loadedWorkflowTemplateCache = make(map[string]struct{})
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
templatePathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatePathMap))
for templatePath := range templatePathMap {
if _, found := loadedWorkflowTemplateCache[templatePath]; found {
continue
}
loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
loadedWorkflowTemplateCache[templatePath] = struct{}{}
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if loaded {

13
v2/pkg/templates/compile.go
View File

@ -17,9 +17,18 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
)
const TemplateExecuterCreationErrorMessage = "cannot create template executer"
var parsedTemplatesCache = make(map[string]*Template, 2500)
// Parse parses a yaml request template file
//nolint:gocritic // this cannot be passed by pointer
// TODO make sure reading from the disk the template parsing happens once: see parsers.ParseTemplate vs templates.Parse
func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
if value, found := parsedTemplatesCache[filePath]; found {
return value, nil
}
template := &Template{}
f, err := os.Open(filePath)
@ -127,8 +136,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
template.TotalRequests += template.Executer.Requests()
}
if template.Executer == nil && template.CompiledWorkflow == nil {
return nil, errors.New("cannot create template executer")
return nil, errors.New(TemplateExecuterCreationErrorMessage)
}
template.Path = filePath
parsedTemplatesCache[filePath] = template
return template, nil
}