From d988de45f645de6556339ba477a46ce0638e240d Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 15 Mar 2024 00:01:09 +0100 Subject: [PATCH] merge --- go.sum | 3 - pkg/catalog/loader/loader.go | 5 +- pkg/loader/workflow/workflow_loader.go | 2 +- pkg/parsers/parser.go | 199 ------------------------- pkg/templates/parser_stats.go | 1 + pkg/templates/parser_test.go | 8 +- pkg/templates/stats.go | 15 ++ pkg/templates/tag_filter.go | 4 +- pkg/templates/tag_filter_test.go | 28 ++-- 9 files changed, 39 insertions(+), 226 deletions(-) delete mode 100644 pkg/parsers/parser.go create mode 100644 pkg/templates/stats.go diff --git a/go.sum b/go.sum index 88b979180..04ac047cd 100644 --- a/go.sum +++ b/go.sum @@ -890,8 +890,6 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo= github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4= -github.com/projectdiscovery/utils v0.0.84-0.20240311212130-16ce15974a4a h1:njYY24OsTQJ80L8O+QxcYFljiVl83xp/BWz3dsIJF30= -github.com/projectdiscovery/utils v0.0.84-0.20240311212130-16ce15974a4a/go.mod h1:bvcudEteeZ5MIZeBCXEfpcgj9h3tyB9qtnmc7zQR92w= github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42 h1:l22rSOP8i6HXu1QfAtIot8NvmJgUmBHEn6Mih7s8Gak= github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42/go.mod h1:VsoXXTuNAAziuodKWakLyurVXaV4tNTJU4Eo8umyr3Q= github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA= @@ -1383,7 +1381,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 459c91c82..1884fa338 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -16,7 +16,6 @@ import ( cfg "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" - "github.com/projectdiscovery/nuclei/v3/pkg/parsers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -114,7 +113,7 @@ func NewConfig(options *types.Options, catalog catalog.Catalog, executerOpts pro // New creates a new template store based on provided configuration func New(cfg *Config) (*Store, error) { - tagFilter, err := templates.New(&filter.Config{ + tagFilter, err := templates.NewTagFilter(&templates.TagFilterConfig{ Tags: cfg.Tags, ExcludeTags: cfg.ExcludeTags, Authors: cfg.Authors, @@ -408,7 +407,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ } if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { // donot include headless template in final list if headless flag is not set - stats.Increment(parsers.HeadlessFlagWarningStats) + stats.Increment(templates.HeadlessFlagWarningStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } diff --git a/pkg/loader/workflow/workflow_loader.go b/pkg/loader/workflow/workflow_loader.go index 5eff92fe3..eb798279d 100644 --- a/pkg/loader/workflow/workflow_loader.go +++ b/pkg/loader/workflow/workflow_loader.go @@ -17,7 +17,7 @@ type workflowLoader struct { // NewLoader returns a new workflow loader structure func NewLoader(options *protocols.ExecutorOptions) (model.WorkflowLoader, error) { - tagFilter, err := templates.NewTagFilter(&templates.Config{ + tagFilter, err := templates.NewTagFilter(&templates.TagFilterConfig{ Authors: options.Options.Authors, Tags: options.Options.Tags, ExcludeTags: options.Options.ExcludeTags, diff --git a/pkg/parsers/parser.go b/pkg/parsers/parser.go deleted file mode 100644 index 25e01b63f..000000000 --- a/pkg/parsers/parser.go +++ /dev/null @@ -1,199 +0,0 @@ -package parsers - -import ( - "encoding/json" - "fmt" - "regexp" - "strings" - - "github.com/projectdiscovery/nuclei/v3/pkg/catalog" - "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" - "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/templates/cache" - "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" - "github.com/projectdiscovery/nuclei/v3/pkg/utils" - "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" - "gopkg.in/yaml.v2" -) - -const ( - errMandatoryFieldMissingFmt = "mandatory '%s' field is missing" - errInvalidFieldFmt = "invalid field format for '%s' (allowed format is %s)" - warningFieldMissingFmt = "field '%s' is missing" - CouldNotLoadTemplate = "Could not load template %s: %s" - LoadedWithWarnings = "Loaded template %s: with syntax warning : %s" -) - -// LoadTemplate returns true if the template is valid and matches the filtering criteria. -func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string, catalog catalog.Catalog) (bool, error) { - template, templateParseError := ParseTemplate(templatePath, catalog) - if templateParseError != nil { - return false, fmt.Errorf(CouldNotLoadTemplate, templatePath, templateParseError) - } - - if len(template.Workflows) > 0 { - return false, nil - } - - validationError := validateTemplateMandatoryFields(template) - if validationError != nil { - stats.Increment(SyntaxErrorStats) - return false, fmt.Errorf(CouldNotLoadTemplate, templatePath, validationError) - } - - ret, err := isTemplateInfoMetadataMatch(tagFilter, template, extraTags) - if err != nil { - return ret, fmt.Errorf(CouldNotLoadTemplate, templatePath, err) - } - // if template loaded then check the template for optional fields to add warnings - if ret { - validationWarning := validateTemplateOptionalFields(template) - if validationWarning != nil { - stats.Increment(SyntaxWarningStats) - return ret, fmt.Errorf(LoadedWithWarnings, templatePath, validationWarning) - } - } - return ret, nil -} - -// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. -func LoadWorkflow(templatePath string, catalog catalog.Catalog) (bool, error) { - template, templateParseError := ParseTemplate(templatePath, catalog) - if templateParseError != nil { - return false, templateParseError - } - - if len(template.Workflows) > 0 { - if validationError := validateTemplateMandatoryFields(template); validationError != nil { - stats.Increment(SyntaxErrorStats) - return false, validationError - } - return true, nil - } - - return false, nil -} - -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, template *templates.Template, extraTags []string) (bool, error) { - match, err := tagFilter.Match(template, extraTags) - - if err == filter.ErrExcluded { - return false, filter.ErrExcluded - } - - return match, err -} - -// validateTemplateMandatoryFields validates the mandatory fields of a template -// return error from this function will cause hard fail and not proceed further -func validateTemplateMandatoryFields(template *templates.Template) error { - info := template.Info - - var errors []string - - if utils.IsBlank(info.Name) { - errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "name")) - } - - if info.Authors.IsEmpty() { - errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "author")) - } - - if template.ID == "" { - errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "id")) - } else if !templateIDRegexp.MatchString(template.ID) { - errors = append(errors, fmt.Sprintf(errInvalidFieldFmt, "id", templateIDRegexp.String())) - } - - if len(errors) > 0 { - return fmt.Errorf(strings.Join(errors, ", ")) - } - - return nil -} - -// validateTemplateOptionalFields validates the optional fields of a template -// return error from this function will throw a warning and proceed further -func validateTemplateOptionalFields(template *templates.Template) error { - info := template.Info - - var warnings []string - - if template.Type() != types.WorkflowProtocol && utils.IsBlank(info.SeverityHolder.Severity.String()) { - warnings = append(warnings, fmt.Sprintf(warningFieldMissingFmt, "severity")) - } - - if len(warnings) > 0 { - return fmt.Errorf(strings.Join(warnings, ", ")) - } - - return nil -} - -var ( - parsedTemplatesCache *cache.Templates - ShouldValidate bool - NoStrictSyntax bool - templateIDRegexp = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`) -) - -const ( - SyntaxWarningStats = "syntax-warnings" - SyntaxErrorStats = "syntax-errors" - RuntimeWarningsStats = "runtime-warnings" - UnsignedCodeWarning = "unsigned-warnings" - HeadlessFlagWarningStats = "headless-flag-missing-warnings" - TemplatesExecutedStats = "templates-executed" - CodeFlagWarningStats = "code-flag-missing-warnings" - FuzzFlagWarningStats = "fuzz-flag-missing-warnings" - // Note: this is redefined in workflows.go to avoid circular dependency, so make sure to keep it in sync - SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates -) - -func init() { - parsedTemplatesCache = cache.New() - config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache) - - stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") - stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") - stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") - stats.NewEntry(UnsignedCodeWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") - stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.") - stats.NewEntry(CodeFlagWarningStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.") - stats.NewEntry(TemplatesExecutedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore") - stats.NewEntry(FuzzFlagWarningStats, "Excluded %d fuzz template[s] (disabled as default), use -fuzz option to run fuzz templates.") - stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]") -} - -// ParseTemplate parses a template and returns a *templates.Template structure -func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Template, error) { - if value, err := parsedTemplatesCache.Has(templatePath); value != nil { - return value.(*templates.Template), err - } - data, err := utils.ReadFromPathOrURL(templatePath, catalog) - if err != nil { - return nil, err - } - - template := &templates.Template{} - - switch config.GetTemplateFormatFromExt(templatePath) { - case config.JSON: - err = json.Unmarshal(data, template) - case config.YAML: - if NoStrictSyntax { - err = yaml.Unmarshal(data, template) - } else { - err = yaml.UnmarshalStrict(data, template) - } - default: - err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath) - } - if err != nil { - return nil, err - } - - parsedTemplatesCache.Store(templatePath, template, nil) - return template, nil -} diff --git a/pkg/templates/parser_stats.go b/pkg/templates/parser_stats.go index 1b555c166..473439b6f 100644 --- a/pkg/templates/parser_stats.go +++ b/pkg/templates/parser_stats.go @@ -8,5 +8,6 @@ const ( HeadlessFlagWarningStats = "headless-flag-missing-warnings" TemplatesExecutedStats = "templates-executed" CodeFlagWarningStats = "code-flag-missing-warnings" + FuzzFlagWarningStats = "fuzz-flag-missing-warnings" SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates ) diff --git a/pkg/templates/parser_test.go b/pkg/templates/parser_test.go index bd6aac1ba..7ceb1bfa3 100644 --- a/pkg/templates/parser_test.go +++ b/pkg/templates/parser_test.go @@ -21,7 +21,7 @@ func TestLoadTemplate(t *testing.T) { name string template *Template templateErr error - filter Config + filter TagFilterConfig expectedErr error isValid bool @@ -79,7 +79,7 @@ func TestLoadTemplate(t *testing.T) { // should be error because the template is loaded expectedErr: errors.New("field 'severity' is missing"), isValid: true, - filter: Config{IncludeIds: []string{"CVE-2021-27330"}}, + filter: TagFilterConfig{IncludeIds: []string{"CVE-2021-27330"}}, }, { name: "template-without-severity-with-diff-filter-id", @@ -91,7 +91,7 @@ func TestLoadTemplate(t *testing.T) { }, }, isValid: false, - filter: Config{IncludeIds: []string{"another-id"}}, + filter: TagFilterConfig{IncludeIds: []string{"another-id"}}, // no error because the template is not loaded expectedErr: nil, }, @@ -143,7 +143,7 @@ func TestLoadTemplate(t *testing.T) { } p.parsedTemplatesCache.Store(name, template, nil, nil) - tagFilter, err := NewTagFilter(&Config{}) + tagFilter, err := NewTagFilter(&TagFilterConfig{}) require.Nil(t, err) success, err := p.LoadTemplate(name, tagFilter, nil, catalog) if tc.success { diff --git a/pkg/templates/stats.go b/pkg/templates/stats.go new file mode 100644 index 000000000..25a61e6e0 --- /dev/null +++ b/pkg/templates/stats.go @@ -0,0 +1,15 @@ +package templates + +import "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" + +func init() { + stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") + stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") + stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") + stats.NewEntry(UnsignedCodeWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") + stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.") + stats.NewEntry(CodeFlagWarningStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.") + stats.NewEntry(TemplatesExecutedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore") + stats.NewEntry(FuzzFlagWarningStats, "Excluded %d fuzz template[s] (disabled as default), use -fuzz option to run fuzz templates.") + stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]") +} diff --git a/pkg/templates/tag_filter.go b/pkg/templates/tag_filter.go index 212edf851..f63836212 100644 --- a/pkg/templates/tag_filter.go +++ b/pkg/templates/tag_filter.go @@ -347,7 +347,7 @@ func isConditionMatch(tagFilter *TagFilter, template *Template) bool { return true } -type Config struct { +type TagFilterConfig struct { Tags []string ExcludeTags []string Authors []string @@ -364,7 +364,7 @@ type Config struct { // New returns a tag filter for nuclei tag based execution // // It takes into account Tags, Severities, ExcludeSeverities, Authors, IncludeTags, ExcludeTags, Conditions. -func NewTagFilter(config *Config) (*TagFilter, error) { +func NewTagFilter(config *TagFilterConfig) (*TagFilter, error) { filter := &TagFilter{ allowedTags: make(map[string]struct{}), authors: make(map[string]struct{}), diff --git a/pkg/templates/tag_filter_test.go b/pkg/templates/tag_filter_test.go index 36804d3e5..de702d6b2 100644 --- a/pkg/templates/tag_filter_test.go +++ b/pkg/templates/tag_filter_test.go @@ -34,7 +34,7 @@ func TestTagBasedFilter(t *testing.T) { return dummyTemplate } - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Tags: []string{"cves", "2021", "jira"}, }) require.Nil(t, err) @@ -61,7 +61,7 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("not-match-excludes", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ ExcludeTags: []string{"dos"}, }) require.Nil(t, err) @@ -71,7 +71,7 @@ func TestTagBasedFilter(t *testing.T) { require.Equal(t, ErrExcluded, err, "could not get correct error") }) t.Run("match-includes", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Tags: []string{"cves", "fuzz"}, ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, @@ -83,7 +83,7 @@ func TestTagBasedFilter(t *testing.T) { require.True(t, matched, "could not get correct match") }) t.Run("match-includes", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ IncludeTags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) @@ -94,7 +94,7 @@ func TestTagBasedFilter(t *testing.T) { require.True(t, matched, "could not get correct match") }) t.Run("match-author", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Authors: []string{"pdteam"}, }) require.Nil(t, err) @@ -103,7 +103,7 @@ func TestTagBasedFilter(t *testing.T) { require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Severities: severity.Severities{severity.High}, }) require.Nil(t, err) @@ -112,7 +112,7 @@ func TestTagBasedFilter(t *testing.T) { require.True(t, matched, "could not get correct match") }) t.Run("match-id", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ IncludeIds: []string{"cve-test"}, }) require.Nil(t, err) @@ -121,7 +121,7 @@ func TestTagBasedFilter(t *testing.T) { require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ ExcludeSeverities: severity.Severities{severity.Low}, }) require.Nil(t, err) @@ -133,7 +133,7 @@ func TestTagBasedFilter(t *testing.T) { require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) @@ -143,7 +143,7 @@ func TestTagBasedFilter(t *testing.T) { require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Authors: []string{"pdteam"}, Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, @@ -164,7 +164,7 @@ func TestTagBasedFilter(t *testing.T) { require.False(t, matched, "could not get correct match") }) t.Run("match-type", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ Protocols: []types.ProtocolType{types.HTTPProtocol}, }) require.Nil(t, err) @@ -174,7 +174,7 @@ func TestTagBasedFilter(t *testing.T) { require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-id", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ ExcludeIds: []string{"cve-test"}, }) require.Nil(t, err) @@ -186,7 +186,7 @@ func TestTagBasedFilter(t *testing.T) { require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-type", func(t *testing.T) { - filter, err := NewTagFilter(&Config{ + filter, err := NewTagFilter(&TagFilterConfig{ ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol}, }) require.Nil(t, err) @@ -268,7 +268,7 @@ func TestTagBasedFilter(t *testing.T) { func testAdvancedFiltering(t *testing.T, includeConditions []string, template *Template, shouldError, shouldMatch bool) { // basic properties - advancedFilter, err := NewTagFilter(&Config{IncludeConditions: includeConditions}) + advancedFilter, err := NewTagFilter(&TagFilterConfig{IncludeConditions: includeConditions}) if shouldError { require.NotNil(t, err) return