Adding support for template id based execution (#1448)

v2/cmd/nuclei/main.go

@@ -101,6 +101,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.VarP(&options.Protocols, "type", "pt", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),
 		flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "ept", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),
 		flagSet.NormalizedStringSliceVarP(&options.Authors, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"),
+		flagSet.NormalizedStringSliceVarP(&options.IncludeIds, "template-id", "id", []string{}, "List of template IDs to run (comma-separated, file)"),
+		flagSet.NormalizedStringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", []string{}, "List of template IDs to exclude (comma-separated, file)"),
 	)
 
 	createGroup(flagSet, "output", "Output",

v2/internal/runner/options.go

@@ -17,6 +17,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 )
 
 // ParseOptions parses the command line flags provided by a user
@@ -114,9 +115,37 @@ func validateOptions(options *types.Options) error {
 		validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile})
 	}
 
+	// expand include/exclude templates id filenames
+	if includeIds, err := processIdsFiltering(options.IncludeIds); err != nil {
+		return err
+	} else {
+		options.IncludeIds = includeIds
+	}
+	if excludeIds, err := processIdsFiltering(options.ExcludeIds); err != nil {
+		return err
+	} else {
+		options.ExcludeIds = excludeIds
+	}
+
 	return nil
 }
 
+func processIdsFiltering(ids []string) ([]string, error) {
+	var finalIds []string
+	for _, id := range ids {
+		if fileutil.FileExists(id) {
+			fileIds, err := utils.LoadFile(id)
+			if err != nil {
+				return nil, err
+			}
+			finalIds = append(finalIds, fileIds...)
+		} else {
+			finalIds = append(finalIds, id)
+		}
+	}
+	return finalIds, nil
+}
+
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output

v2/pkg/catalog/loader/filter/tag_filter.go

@@ -18,6 +18,8 @@ type TagFilter struct {
 	matchAllows       map[string]struct{}
 	types             map[types.ProtocolType]struct{}
 	excludeTypes      map[types.ProtocolType]struct{}
+	allowedIds        map[string]struct{}
+	excludeIds        map[string]struct{}
 }
 
 // ErrExcluded is returned for excluded templates
@@ -28,7 +30,7 @@ var ErrExcluded = errors.New("the template was excluded")
 // unless it is explicitly specified by user using the includeTags (matchAllows field).
 // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
 // Returns true if the template matches the filter criteria, false otherwise.
-func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType) (bool, error) {
+func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType, templateId string) (bool, error) {
 	for _, templateTag := range templateTags {
 		_, blocked := tagFilter.block[templateTag]
 		_, allowed := tagFilter.matchAllows[templateTag]
@@ -57,6 +59,11 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
 	if !isTemplateTypeMatch(tagFilter, templateType) {
 		return false, nil
 	}
+
+	if !isIdMatch(tagFilter, templateId) {
+		return false, nil
+	}
+
 	return true, nil
 }
 
@@ -143,6 +150,23 @@ func isTemplateTypeMatch(tagFilter *TagFilter, templateType types.ProtocolType)
 	return included && !excluded
 }
 
+func isIdMatch(tagFilter *TagFilter, templateId string) bool {
+	if len(tagFilter.excludeIds) == 0 && len(tagFilter.allowedIds) == 0 {
+		return true
+	}
+	included := true
+	if len(tagFilter.allowedIds) > 0 {
+		_, included = tagFilter.allowedIds[templateId]
+	}
+
+	excluded := false
+	if len(tagFilter.excludeIds) > 0 {
+		_, excluded = tagFilter.excludeIds[templateId]
+	}
+
+	return included && !excluded
+}
+
 type Config struct {
 	Tags              []string
 	ExcludeTags       []string
@@ -150,6 +174,8 @@ type Config struct {
 	Severities        severity.Severities
 	ExcludeSeverities severity.Severities
 	IncludeTags       []string
+	IncludeIds        []string
+	ExcludeIds        []string
 	Protocols         types.ProtocolTypes
 	ExcludeProtocols  types.ProtocolTypes
 }
@@ -167,6 +193,8 @@ func New(config *Config) *TagFilter {
 		matchAllows:       make(map[string]struct{}),
 		types:             make(map[types.ProtocolType]struct{}),
 		excludeTypes:      make(map[types.ProtocolType]struct{}),
+		allowedIds:        make(map[string]struct{}),
+		excludeIds:        make(map[string]struct{}),
 	}
 	for _, tag := range config.ExcludeTags {
 		for _, val := range splitCommaTrim(tag) {
@@ -218,6 +246,21 @@ func New(config *Config) *TagFilter {
 			filter.excludeTypes[tag] = struct{}{}
 		}
 	}
+	for _, id := range config.ExcludeIds {
+		for _, val := range splitCommaTrim(id) {
+			if _, ok := filter.block[val]; !ok {
+				filter.excludeIds[val] = struct{}{}
+			}
+		}
+	}
+	for _, id := range config.IncludeIds {
+		for _, val := range splitCommaTrim(id) {
+			if _, ok := filter.allowedIds[val]; !ok {
+				filter.allowedIds[val] = struct{}{}
+			}
+			delete(filter.excludeIds, val)
+		}
+	}
 	return filter
 }
 

v2/pkg/catalog/loader/filter/tag_filter_test.go

@@ -16,19 +16,19 @@ func TestTagBasedFilter(t *testing.T) {
 		})
 
 		t.Run("true", func(t *testing.T) {
-			matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+			matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 			require.True(t, matched, "could not get correct match")
 		})
 		t.Run("false", func(t *testing.T) {
-			matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+			matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 			require.False(t, matched, "could not get correct match")
 		})
 		t.Run("match-extra-tags-positive", func(t *testing.T) {
-			matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol)
+			matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol, "")
 			require.True(t, matched, "could not get correct match")
 		})
 		t.Run("match-extra-tags-negative", func(t *testing.T) {
-			matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol)
+			matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol, "")
 			require.False(t, matched, "could not get correct match")
 		})
 	}
@@ -37,7 +37,7 @@ func TestTagBasedFilter(t *testing.T) {
 		filter := New(&Config{
 			ExcludeTags: []string{"dos"},
 		})
-		matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 		require.Equal(t, ErrExcluded, err, "could not get correct error")
 	})
@@ -47,7 +47,7 @@ func TestTagBasedFilter(t *testing.T) {
 			ExcludeTags: []string{"dos", "fuzz"},
 			IncludeTags: []string{"fuzz"},
 		})
-		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.Nil(t, err, "could not get match")
 		require.True(t, matched, "could not get correct match")
 	})
@@ -56,7 +56,7 @@ func TestTagBasedFilter(t *testing.T) {
 			Tags:        []string{"fuzz"},
 			ExcludeTags: []string{"fuzz"},
 		})
-		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.Nil(t, err, "could not get match")
 		require.True(t, matched, "could not get correct match")
 	})
@@ -64,24 +64,31 @@ func TestTagBasedFilter(t *testing.T) {
 		filter := New(&Config{
 			Authors: []string{"pdteam"},
 		})
-		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-severity", func(t *testing.T) {
 		filter := New(&Config{
 			Severities: severity.Severities{severity.High},
 		})
-		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-id", func(t *testing.T) {
+		filter := New(&Config{
+			IncludeIds: []string{"cve-test"},
+		})
+		matched, _ := filter.Match([]string{""}, []string{""}, severity.Low, nil, types.HTTPProtocol, "cve-test")
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-severity", func(t *testing.T) {
 		filter := New(&Config{
 			ExcludeSeverities: severity.Severities{severity.Low},
 		})
-		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
 		require.True(t, matched, "could not get correct match")
 
-		matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-with-tags", func(t *testing.T) {
@@ -89,7 +96,7 @@ func TestTagBasedFilter(t *testing.T) {
 			Tags:        []string{"tag"},
 			ExcludeTags: []string{"another"},
 		})
-		matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
+		matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-conditions", func(t *testing.T) {
@@ -98,33 +105,43 @@ func TestTagBasedFilter(t *testing.T) {
 			Tags:       []string{"jira"},
 			Severities: severity.Severities{severity.High},
 		})
-		matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol)
+		matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol, "")
 		require.True(t, matched, "could not get correct match")
 
-		matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 
-		matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol)
+		matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 
-		matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol)
+		matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-type", func(t *testing.T) {
 		filter := New(&Config{
 			Protocols: []types.ProtocolType{types.HTTPProtocol},
 		})
-		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
 		require.True(t, matched, "could not get correct match")
 	})
+	t.Run("match-exclude-id", func(t *testing.T) {
+		filter := New(&Config{
+			ExcludeIds: []string{"cve-test"},
+		})
+		matched, _ := filter.Match([]string{""}, []string{""}, severity.High, nil, types.DNSProtocol, "cve-test1")
+		require.True(t, matched, "could not get correct match")
+
+		matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "cve-test")
+		require.False(t, matched, "could not get correct match")
+	})
 	t.Run("match-exclude-type", func(t *testing.T) {
 		filter := New(&Config{
 			ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol},
 		})
-		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol)
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol, "")
 		require.True(t, matched, "could not get correct match")
 
-		matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
+		matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
 		require.False(t, matched, "could not get correct match")
 	})
 }

v2/pkg/catalog/loader/loader.go

@@ -32,6 +32,8 @@ type Config struct {
 	Severities        severity.Severities
 	ExcludeSeverities severity.Severities
 	IncludeTags       []string
+	IncludeIds        []string
+	ExcludeIds        []string
 
 	Catalog            *catalog.Catalog
 	ExecutorOptions    protocols.ExecuterOptions
@@ -67,6 +69,8 @@ func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts pr
 		Severities:         options.Severities,
 		ExcludeSeverities:  options.ExcludeSeverities,
 		IncludeTags:        options.IncludeTags,
+		IncludeIds:         options.IncludeIds,
+		ExcludeIds:         options.ExcludeIds,
 		TemplatesDirectory: options.TemplatesDirectory,
 		Protocols:          options.Protocols,
 		ExcludeProtocols:   options.ExcludeProtocols,
@@ -88,6 +92,8 @@ func New(config *Config) (*Store, error) {
 			Severities:        config.Severities,
 			ExcludeSeverities: config.ExcludeSeverities,
 			IncludeTags:       config.IncludeTags,
+			IncludeIds:        config.IncludeIds,
+			ExcludeIds:        config.ExcludeIds,
 			Protocols:         config.Protocols,
 			ExcludeProtocols:  config.ExcludeProtocols,
 		}),

v2/pkg/parsers/parser.go

@@ -40,7 +40,9 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
 		return false, validationError
 	}
 
-	return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type())
+	templateId := strings.ToLower(template.ID)
+
+	return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type(), templateId)
 }
 
 // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
@@ -60,12 +62,12 @@ func LoadWorkflow(templatePath string) (bool, error) {
 	return false, nil
 }
 
-func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType) (bool, error) {
+func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType, templateId string) (bool, error) {
 	templateTags := templateInfo.Tags.ToSlice()
 	templateAuthors := templateInfo.Authors.ToSlice()
 	templateSeverity := templateInfo.SeverityHolder.Severity
 
-	match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType)
+	match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType, templateId)
 
 	if err == filter.ErrExcluded {
 		return false, filter.ErrExcluded

v2/pkg/parsers/workflow_loader.go

@@ -21,6 +21,8 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
 		Authors:     options.Options.Authors,
 		Severities:  options.Options.Severities,
 		IncludeTags: options.Options.IncludeTags,
+		IncludeIds:  options.Options.IncludeIds,
+		ExcludeIds:  options.Options.ExcludeIds,
 	})
 	pathFilter := filter.NewPathFilter(&filter.PathFilterConfig{
 		IncludedTemplates: options.Options.IncludeTemplates,

v2/pkg/types/types.go

@@ -45,6 +45,10 @@ type Options struct {
 	IncludeTags goflags.NormalizedStringSlice
 	// IncludeTemplates includes specified templates to be run even while being in denylist
 	IncludeTemplates goflags.StringSlice
+	// IncludeIds includes specified ids to be run even while being in denylist
+	IncludeIds goflags.NormalizedStringSlice
+	// ExcludeIds contains templates ids to not be executed
+	ExcludeIds goflags.NormalizedStringSlice
 
 	InternalResolversList []string // normalized from resolvers flag as well as file provided.
 	// ProjectPath allows nuclei to use a user defined project folder

v2/pkg/utils/utils.go

@@ -3,6 +3,8 @@ package utils
 import (
 	"errors"
 	"strings"
+
+	"github.com/projectdiscovery/fileutil"
 )
 
 func IsBlank(value string) bool {
@@ -23,3 +25,15 @@ func UnwrapError(err error) error {
 	}
 	return err
 }
+
+func LoadFile(filename string) ([]string, error) {
+	var items []string
+	readfileChan, err := fileutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	for includeIdLine := range readfileChan {
+		items = append(items, includeIdLine)
+	}
+	return items, nil
+}