merge

go.sum

@@ -890,8 +890,6 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.84-0.20240311212130-16ce15974a4a h1:njYY24OsTQJ80L8O+QxcYFljiVl83xp/BWz3dsIJF30=
-github.com/projectdiscovery/utils v0.0.84-0.20240311212130-16ce15974a4a/go.mod h1:bvcudEteeZ5MIZeBCXEfpcgj9h3tyB9qtnmc7zQR92w=
 github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42 h1:l22rSOP8i6HXu1QfAtIot8NvmJgUmBHEn6Mih7s8Gak=
 github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42/go.mod h1:VsoXXTuNAAziuodKWakLyurVXaV4tNTJU4Eo8umyr3Q=
 github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA=
@@ -1383,7 +1381,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/catalog/loader/loader.go

@@ -16,7 +16,6 @@ import (
 	cfg "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
-	"github.com/projectdiscovery/nuclei/v3/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -114,7 +113,7 @@ func NewConfig(options *types.Options, catalog catalog.Catalog, executerOpts pro
 
 // New creates a new template store based on provided configuration
 func New(cfg *Config) (*Store, error) {
-	tagFilter, err := templates.New(&filter.Config{
+	tagFilter, err := templates.NewTagFilter(&templates.TagFilterConfig{
 		Tags:              cfg.Tags,
 		ExcludeTags:       cfg.ExcludeTags,
 		Authors:           cfg.Authors,
@@ -408,7 +407,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 				}
 				if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
 					// donot include headless template in final list if headless flag is not set
-					stats.Increment(parsers.HeadlessFlagWarningStats)
+					stats.Increment(templates.HeadlessFlagWarningStats)
 					if config.DefaultConfig.LogAllEvents {
 						gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
 					}

pkg/loader/workflow/workflow_loader.go

@@ -17,7 +17,7 @@ type workflowLoader struct {
 
 // NewLoader returns a new workflow loader structure
 func NewLoader(options *protocols.ExecutorOptions) (model.WorkflowLoader, error) {
-	tagFilter, err := templates.NewTagFilter(&templates.Config{
+	tagFilter, err := templates.NewTagFilter(&templates.TagFilterConfig{
 		Authors:           options.Options.Authors,
 		Tags:              options.Options.Tags,
 		ExcludeTags:       options.Options.ExcludeTags,

pkg/parsers/parser.go

@@ -1,199 +0,0 @@
-package parsers
-
-import (
-	"encoding/json"
-	"fmt"
-	"regexp"
-	"strings"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates/cache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
-	"gopkg.in/yaml.v2"
-)
-
-const (
-	errMandatoryFieldMissingFmt = "mandatory '%s' field is missing"
-	errInvalidFieldFmt          = "invalid field format for '%s' (allowed format is %s)"
-	warningFieldMissingFmt      = "field '%s' is missing"
-	CouldNotLoadTemplate        = "Could not load template %s: %s"
-	LoadedWithWarnings          = "Loaded template %s: with syntax warning : %s"
-)
-
-// LoadTemplate returns true if the template is valid and matches the filtering criteria.
-func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string, catalog catalog.Catalog) (bool, error) {
-	template, templateParseError := ParseTemplate(templatePath, catalog)
-	if templateParseError != nil {
-		return false, fmt.Errorf(CouldNotLoadTemplate, templatePath, templateParseError)
-	}
-
-	if len(template.Workflows) > 0 {
-		return false, nil
-	}
-
-	validationError := validateTemplateMandatoryFields(template)
-	if validationError != nil {
-		stats.Increment(SyntaxErrorStats)
-		return false, fmt.Errorf(CouldNotLoadTemplate, templatePath, validationError)
-	}
-
-	ret, err := isTemplateInfoMetadataMatch(tagFilter, template, extraTags)
-	if err != nil {
-		return ret, fmt.Errorf(CouldNotLoadTemplate, templatePath, err)
-	}
-	// if template loaded then check the template for optional fields to add warnings
-	if ret {
-		validationWarning := validateTemplateOptionalFields(template)
-		if validationWarning != nil {
-			stats.Increment(SyntaxWarningStats)
-			return ret, fmt.Errorf(LoadedWithWarnings, templatePath, validationWarning)
-		}
-	}
-	return ret, nil
-}
-
-// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
-func LoadWorkflow(templatePath string, catalog catalog.Catalog) (bool, error) {
-	template, templateParseError := ParseTemplate(templatePath, catalog)
-	if templateParseError != nil {
-		return false, templateParseError
-	}
-
-	if len(template.Workflows) > 0 {
-		if validationError := validateTemplateMandatoryFields(template); validationError != nil {
-			stats.Increment(SyntaxErrorStats)
-			return false, validationError
-		}
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, template *templates.Template, extraTags []string) (bool, error) {
-	match, err := tagFilter.Match(template, extraTags)
-
-	if err == filter.ErrExcluded {
-		return false, filter.ErrExcluded
-	}
-
-	return match, err
-}
-
-// validateTemplateMandatoryFields validates the mandatory fields of a template
-// return error from this function will cause hard fail and not proceed further
-func validateTemplateMandatoryFields(template *templates.Template) error {
-	info := template.Info
-
-	var errors []string
-
-	if utils.IsBlank(info.Name) {
-		errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "name"))
-	}
-
-	if info.Authors.IsEmpty() {
-		errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "author"))
-	}
-
-	if template.ID == "" {
-		errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "id"))
-	} else if !templateIDRegexp.MatchString(template.ID) {
-		errors = append(errors, fmt.Sprintf(errInvalidFieldFmt, "id", templateIDRegexp.String()))
-	}
-
-	if len(errors) > 0 {
-		return fmt.Errorf(strings.Join(errors, ", "))
-	}
-
-	return nil
-}
-
-// validateTemplateOptionalFields validates the optional fields of a template
-// return error from this function will throw a warning and proceed further
-func validateTemplateOptionalFields(template *templates.Template) error {
-	info := template.Info
-
-	var warnings []string
-
-	if template.Type() != types.WorkflowProtocol && utils.IsBlank(info.SeverityHolder.Severity.String()) {
-		warnings = append(warnings, fmt.Sprintf(warningFieldMissingFmt, "severity"))
-	}
-
-	if len(warnings) > 0 {
-		return fmt.Errorf(strings.Join(warnings, ", "))
-	}
-
-	return nil
-}
-
-var (
-	parsedTemplatesCache *cache.Templates
-	ShouldValidate       bool
-	NoStrictSyntax       bool
-	templateIDRegexp     = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`)
-)
-
-const (
-	SyntaxWarningStats       = "syntax-warnings"
-	SyntaxErrorStats         = "syntax-errors"
-	RuntimeWarningsStats     = "runtime-warnings"
-	UnsignedCodeWarning      = "unsigned-warnings"
-	HeadlessFlagWarningStats = "headless-flag-missing-warnings"
-	TemplatesExecutedStats   = "templates-executed"
-	CodeFlagWarningStats     = "code-flag-missing-warnings"
-	FuzzFlagWarningStats     = "fuzz-flag-missing-warnings"
-	// Note: this is redefined in workflows.go to avoid circular dependency, so make sure to keep it in sync
-	SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates
-)
-
-func init() {
-	parsedTemplatesCache = cache.New()
-	config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache)
-
-	stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)")
-	stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)")
-	stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)")
-	stats.NewEntry(UnsignedCodeWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)")
-	stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.")
-	stats.NewEntry(CodeFlagWarningStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.")
-	stats.NewEntry(TemplatesExecutedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore")
-	stats.NewEntry(FuzzFlagWarningStats, "Excluded %d fuzz template[s] (disabled as default), use -fuzz option to run fuzz templates.")
-	stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]")
-}
-
-// ParseTemplate parses a template and returns a *templates.Template structure
-func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Template, error) {
-	if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
-		return value.(*templates.Template), err
-	}
-	data, err := utils.ReadFromPathOrURL(templatePath, catalog)
-	if err != nil {
-		return nil, err
-	}
-
-	template := &templates.Template{}
-
-	switch config.GetTemplateFormatFromExt(templatePath) {
-	case config.JSON:
-		err = json.Unmarshal(data, template)
-	case config.YAML:
-		if NoStrictSyntax {
-			err = yaml.Unmarshal(data, template)
-		} else {
-			err = yaml.UnmarshalStrict(data, template)
-		}
-	default:
-		err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	parsedTemplatesCache.Store(templatePath, template, nil)
-	return template, nil
-}

pkg/templates/parser_stats.go

@@ -8,5 +8,6 @@ const (
 	HeadlessFlagWarningStats = "headless-flag-missing-warnings"
 	TemplatesExecutedStats   = "templates-executed"
 	CodeFlagWarningStats     = "code-flag-missing-warnings"
+	FuzzFlagWarningStats     = "fuzz-flag-missing-warnings"
 	SkippedUnsignedStats     = "skipped-unsigned-stats" // tracks loading of unsigned templates
 )

pkg/templates/parser_test.go

@@ -21,7 +21,7 @@ func TestLoadTemplate(t *testing.T) {
 		name        string
 		template    *Template
 		templateErr error
-		filter      Config
+		filter      TagFilterConfig
 
 		expectedErr error
 		isValid     bool
@@ -79,7 +79,7 @@ func TestLoadTemplate(t *testing.T) {
 			// should be error because the template is loaded
 			expectedErr: errors.New("field 'severity' is missing"),
 			isValid:     true,
-			filter:      Config{IncludeIds: []string{"CVE-2021-27330"}},
+			filter:      TagFilterConfig{IncludeIds: []string{"CVE-2021-27330"}},
 		},
 		{
 			name: "template-without-severity-with-diff-filter-id",
@@ -91,7 +91,7 @@ func TestLoadTemplate(t *testing.T) {
 				},
 			},
 			isValid: false,
-			filter:  Config{IncludeIds: []string{"another-id"}},
+			filter:  TagFilterConfig{IncludeIds: []string{"another-id"}},
 			// no error because the template is not loaded
 			expectedErr: nil,
 		},
@@ -143,7 +143,7 @@ func TestLoadTemplate(t *testing.T) {
 				}
 				p.parsedTemplatesCache.Store(name, template, nil, nil)
 
-				tagFilter, err := NewTagFilter(&Config{})
+				tagFilter, err := NewTagFilter(&TagFilterConfig{})
 				require.Nil(t, err)
 				success, err := p.LoadTemplate(name, tagFilter, nil, catalog)
 				if tc.success {

pkg/templates/stats.go

@@ -0,0 +1,15 @@
+package templates
+
+import "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
+
+func init() {
+	stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)")
+	stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)")
+	stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)")
+	stats.NewEntry(UnsignedCodeWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)")
+	stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.")
+	stats.NewEntry(CodeFlagWarningStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.")
+	stats.NewEntry(TemplatesExecutedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore")
+	stats.NewEntry(FuzzFlagWarningStats, "Excluded %d fuzz template[s] (disabled as default), use -fuzz option to run fuzz templates.")
+	stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]")
+}

pkg/templates/tag_filter.go

@@ -347,7 +347,7 @@ func isConditionMatch(tagFilter *TagFilter, template *Template) bool {
 	return true
 }
 
-type Config struct {
+type TagFilterConfig struct {
 	Tags              []string
 	ExcludeTags       []string
 	Authors           []string
@@ -364,7 +364,7 @@ type Config struct {
 // New returns a tag filter for nuclei tag based execution
 //
 // It takes into account Tags, Severities, ExcludeSeverities, Authors, IncludeTags, ExcludeTags, Conditions.
-func NewTagFilter(config *Config) (*TagFilter, error) {
+func NewTagFilter(config *TagFilterConfig) (*TagFilter, error) {
 	filter := &TagFilter{
 		allowedTags:       make(map[string]struct{}),
 		authors:           make(map[string]struct{}),

pkg/templates/tag_filter_test.go

@@ -34,7 +34,7 @@ func TestTagBasedFilter(t *testing.T) {
 		return dummyTemplate
 	}
 
-	filter, err := NewTagFilter(&Config{
+	filter, err := NewTagFilter(&TagFilterConfig{
 		Tags: []string{"cves", "2021", "jira"},
 	})
 	require.Nil(t, err)
@@ -61,7 +61,7 @@ func TestTagBasedFilter(t *testing.T) {
 	})
 
 	t.Run("not-match-excludes", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			ExcludeTags: []string{"dos"},
 		})
 		require.Nil(t, err)
@@ -71,7 +71,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.Equal(t, ErrExcluded, err, "could not get correct error")
 	})
 	t.Run("match-includes", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			Tags:        []string{"cves", "fuzz"},
 			ExcludeTags: []string{"dos", "fuzz"},
 			IncludeTags: []string{"fuzz"},
@@ -83,7 +83,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-includes", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			IncludeTags: []string{"fuzz"},
 			ExcludeTags: []string{"fuzz"},
 		})
@@ -94,7 +94,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-author", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			Authors: []string{"pdteam"},
 		})
 		require.Nil(t, err)
@@ -103,7 +103,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-severity", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			Severities: severity.Severities{severity.High},
 		})
 		require.Nil(t, err)
@@ -112,7 +112,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-id", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			IncludeIds: []string{"cve-test"},
 		})
 		require.Nil(t, err)
@@ -121,7 +121,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-severity", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			ExcludeSeverities: severity.Severities{severity.Low},
 		})
 		require.Nil(t, err)
@@ -133,7 +133,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-with-tags", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			Tags:        []string{"tag"},
 			ExcludeTags: []string{"another"},
 		})
@@ -143,7 +143,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-conditions", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			Authors:    []string{"pdteam"},
 			Tags:       []string{"jira"},
 			Severities: severity.Severities{severity.High},
@@ -164,7 +164,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-type", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			Protocols: []types.ProtocolType{types.HTTPProtocol},
 		})
 		require.Nil(t, err)
@@ -174,7 +174,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-id", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			ExcludeIds: []string{"cve-test"},
 		})
 		require.Nil(t, err)
@@ -186,7 +186,7 @@ func TestTagBasedFilter(t *testing.T) {
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-type", func(t *testing.T) {
-		filter, err := NewTagFilter(&Config{
+		filter, err := NewTagFilter(&TagFilterConfig{
 			ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol},
 		})
 		require.Nil(t, err)
@@ -268,7 +268,7 @@ func TestTagBasedFilter(t *testing.T) {
 
 func testAdvancedFiltering(t *testing.T, includeConditions []string, template *Template, shouldError, shouldMatch bool) {
 	// basic properties
-	advancedFilter, err := NewTagFilter(&Config{IncludeConditions: includeConditions})
+	advancedFilter, err := NewTagFilter(&TagFilterConfig{IncludeConditions: includeConditions})
 	if shouldError {
 		require.NotNil(t, err)
 		return