From 53b167064a9e3cd6785a084f307077523b05964b Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 2 Aug 2025 15:54:15 +0530 Subject: [PATCH] feat: loading templates performance improvements --- internal/runner/options.go | 5 ++-- pkg/operators/matchers/validate.go | 23 +++++++++++------- pkg/templates/parser.go | 36 ++++++++++++++++++++-------- pkg/templates/templates.go | 13 ++++------ pkg/templates/validator_singleton.go | 7 ++++++ 5 files changed, 56 insertions(+), 28 deletions(-) create mode 100644 pkg/templates/validator_singleton.go diff --git a/internal/runner/options.go b/internal/runner/options.go index bd6b92bc0..48d26e9f4 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -39,6 +39,8 @@ const ( DefaultDumpTrafficOutputFolder = "output" ) +var validateOptions = validator.New() + func ConfigureOptions() error { // with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions // if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read @@ -138,8 +140,7 @@ func ParseOptions(options *types.Options) { // validateOptions validates the configuration options passed func ValidateOptions(options *types.Options) error { - validate := validator.New() - if err := validate.Struct(options); err != nil { + if err := validateOptions.Struct(options); err != nil { if _, ok := err.(*validator.InvalidValidationError); ok { return err } diff --git a/pkg/operators/matchers/validate.go b/pkg/operators/matchers/validate.go index 0f6a5b916..7dd5a1388 100644 --- a/pkg/operators/matchers/validate.go +++ b/pkg/operators/matchers/validate.go @@ -8,22 +8,29 @@ import ( "github.com/antchfx/xpath" sliceutil "github.com/projectdiscovery/utils/slice" - "gopkg.in/yaml.v3" ) var commonExpectedFields = []string{"Type", "Condition", "Name", "MatchAll", "Negative", "Internal"} // Validate perform initial validation on the matcher structure func (matcher *Matcher) Validate() error { - // uses yaml marshaling to convert the struct to map[string]interface to have same field names + // Build a map of YAML‐tag names that are actually set (non-zero) in the matcher. matcherMap := make(map[string]interface{}) - marshaledMatcher, err := yaml.Marshal(matcher) - if err != nil { - return err - } - if err := yaml.Unmarshal(marshaledMatcher, &matcherMap); err != nil { - return err + val := reflect.ValueOf(*matcher) + typ := reflect.TypeOf(*matcher) + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + // skip internal / unexported or opt-out fields + yamlTag := strings.Split(field.Tag.Get("yaml"), ",")[0] + if yamlTag == "" || yamlTag == "-" { + continue + } + if val.Field(i).IsZero() { + continue + } + matcherMap[yamlTag] = struct{}{} } + var err error var expectedFields []string switch matcher.matcherType { diff --git a/pkg/templates/parser.go b/pkg/templates/parser.go index 56b64c237..3a2cabc2a 100644 --- a/pkg/templates/parser.go +++ b/pkg/templates/parser.go @@ -131,13 +131,13 @@ func (p *Parser) ParseTemplate(templatePath string, catalog catalog.Catalog) (an _ = reader.Close() }() - data, err := io.ReadAll(reader) - if err != nil { - return nil, err - } - - // pre-process directives only for local files + // For local YAML files, check if preprocessing is needed + var data []byte if fileutil.FileExists(templatePath) && config.GetTemplateFormatFromExt(templatePath) == config.YAML { + data, err = io.ReadAll(reader) + if err != nil { + return nil, err + } data, err = yamlutil.PreProcess(data) if err != nil { return nil, err @@ -148,12 +148,28 @@ func (p *Parser) ParseTemplate(templatePath string, catalog catalog.Catalog) (an switch config.GetTemplateFormatFromExt(templatePath) { case config.JSON: + if data == nil { + data, err = io.ReadAll(reader) + if err != nil { + return nil, err + } + } err = json.Unmarshal(data, template) case config.YAML: - if p.NoStrictSyntax { - err = yaml.Unmarshal(data, template) + if data != nil { + // Already read and preprocessed + if p.NoStrictSyntax { + err = yaml.Unmarshal(data, template) + } else { + err = yaml.UnmarshalStrict(data, template) + } } else { - err = yaml.UnmarshalStrict(data, template) + // Stream directly from reader + decoder := yaml.NewDecoder(reader) + if !p.NoStrictSyntax { + decoder.SetStrict(true) + } + err = decoder.Decode(template) } default: err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath) @@ -162,7 +178,7 @@ func (p *Parser) ParseTemplate(templatePath string, catalog catalog.Catalog) (an return nil, err } - p.parsedTemplatesCache.Store(templatePath, template, data, nil) + p.parsedTemplatesCache.Store(templatePath, template, nil, nil) // don't keep raw bytes to save memory return template, nil } diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 7726fc7c9..fbbfd44e9 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -7,7 +7,6 @@ import ( "strconv" "strings" - validate "github.com/go-playground/validator/v10" "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/code" @@ -310,10 +309,8 @@ func (template *Template) validateAllRequestIDs() { // MarshalYAML forces recursive struct validation during marshal operation func (template *Template) MarshalYAML() ([]byte, error) { out, marshalErr := yaml.Marshal(template) - // Review: we are adding requestIDs for templateContext - // if we are using this method then we might need to purge manually added IDS that start with `templatetype_` - // this is only applicable if there are more than 1 request fields in protocol - errValidate := validate.New().Struct(template) + // Use shared validator to avoid rebuilding struct cache for every template marshal + errValidate := tplValidator.Struct(template) return out, multierr.Append(marshalErr, errValidate) } @@ -354,7 +351,7 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error if len(alias.RequestsWithTCP) > 0 { template.RequestsNetwork = alias.RequestsWithTCP } - err = validate.New().Struct(template) + err = tplValidator.Struct(template) if err != nil { return err } @@ -525,7 +522,7 @@ func (template *Template) hasMultipleRequests() bool { func (template *Template) MarshalJSON() ([]byte, error) { type TemplateAlias Template //avoid recursion out, marshalErr := json.Marshal((*TemplateAlias)(template)) - errValidate := validate.New().Struct(template) + errValidate := tplValidator.Struct(template) return out, multierr.Append(marshalErr, errValidate) } @@ -538,7 +535,7 @@ func (template *Template) UnmarshalJSON(data []byte) error { return err } *template = Template(*alias) - err = validate.New().Struct(template) + err = tplValidator.Struct(template) if err != nil { return err } diff --git a/pkg/templates/validator_singleton.go b/pkg/templates/validator_singleton.go new file mode 100644 index 000000000..9679ac210 --- /dev/null +++ b/pkg/templates/validator_singleton.go @@ -0,0 +1,7 @@ +package templates + +import ( + validate "github.com/go-playground/validator/v10" +) + +var tplValidator = validate.New()