External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:55:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding support for template id based execution (#1448)

Browse Source
This commit is contained in:
Mzack9999 2022-01-07 13:00:20 +01:00 committed by GitHub
parent ccef8afbe7
commit 39519c01a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 142 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/cmd/nuclei/main.go
View File

@ -101,6 +101,8 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.VarP(&options.Protocols, "type", "pt", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),
flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "ept", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),
flagSet.NormalizedStringSliceVarP(&options.Authors, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"),
flagSet.NormalizedStringSliceVarP(&options.IncludeIds, "template-id", "id", []string{}, "List of template IDs to run (comma-separated, file)"),
flagSet.NormalizedStringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", []string{}, "List of template IDs to exclude (comma-separated, file)"),
)
createGroup(flagSet, "output", "Output",

29
v2/internal/runner/options.go
View File

@ -17,6 +17,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
)
// ParseOptions parses the command line flags provided by a user
@ -114,9 +115,37 @@ func validateOptions(options *types.Options) error {
validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile})
}
// expand include/exclude templates id filenames
if includeIds, err := processIdsFiltering(options.IncludeIds); err != nil {
return err
} else {
options.IncludeIds = includeIds
}
if excludeIds, err := processIdsFiltering(options.ExcludeIds); err != nil {
return err
} else {
options.ExcludeIds = excludeIds
}
return nil
}
func processIdsFiltering(ids []string) ([]string, error) {
var finalIds []string
for _, id := range ids {
if fileutil.FileExists(id) {
fileIds, err := utils.LoadFile(id)
if err != nil {
return nil, err
}
finalIds = append(finalIds, fileIds...)
} else {
finalIds = append(finalIds, id)
}
}
return finalIds, nil
}
// configureOutput configures the output logging levels to be displayed on the screen
func configureOutput(options *types.Options) {
// If the user desires verbose output, show verbose output

45
v2/pkg/catalog/loader/filter/tag_filter.go
View File

@ -18,6 +18,8 @@ type TagFilter struct {
matchAllows map[string]struct{}
types map[types.ProtocolType]struct{}
excludeTypes map[types.ProtocolType]struct{}
allowedIds map[string]struct{}
excludeIds map[string]struct{}
}
// ErrExcluded is returned for excluded templates
@ -28,7 +30,7 @@ var ErrExcluded = errors.New("the template was excluded")
// unless it is explicitly specified by user using the includeTags (matchAllows field).
// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
// Returns true if the template matches the filter criteria, false otherwise.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType) (bool, error) {
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType, templateId string) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
@ -57,6 +59,11 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
if !isTemplateTypeMatch(tagFilter, templateType) {
return false, nil
}
if !isIdMatch(tagFilter, templateId) {
return false, nil
}
return true, nil
}
@ -143,6 +150,23 @@ func isTemplateTypeMatch(tagFilter *TagFilter, templateType types.ProtocolType)
return included && !excluded
}
func isIdMatch(tagFilter *TagFilter, templateId string) bool {
if len(tagFilter.excludeIds) == 0 && len(tagFilter.allowedIds) == 0 {
return true
}
included := true
if len(tagFilter.allowedIds) > 0 {
_, included = tagFilter.allowedIds[templateId]
}
excluded := false
if len(tagFilter.excludeIds) > 0 {
_, excluded = tagFilter.excludeIds[templateId]
}
return included && !excluded
}
type Config struct {
Tags []string
ExcludeTags []string
@ -150,6 +174,8 @@ type Config struct {
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
IncludeIds []string
ExcludeIds []string
Protocols types.ProtocolTypes
ExcludeProtocols types.ProtocolTypes
}
@ -167,6 +193,8 @@ func New(config *Config) *TagFilter {
matchAllows: make(map[string]struct{}),
types: make(map[types.ProtocolType]struct{}),
excludeTypes: make(map[types.ProtocolType]struct{}),
allowedIds: make(map[string]struct{}),
excludeIds: make(map[string]struct{}),
}
for _, tag := range config.ExcludeTags {
for _, val := range splitCommaTrim(tag) {
@ -218,6 +246,21 @@ func New(config *Config) *TagFilter {
filter.excludeTypes[tag] = struct{}{}
}
}
for _, id := range config.ExcludeIds {
for _, val := range splitCommaTrim(id) {
if _, ok := filter.block[val]; !ok {
filter.excludeIds[val] = struct{}{}
}
}
}
for _, id := range config.IncludeIds {
for _, val := range splitCommaTrim(id) {
if _, ok := filter.allowedIds[val]; !ok {
filter.allowedIds[val] = struct{}{}
}
delete(filter.excludeIds, val)
}
}
return filter
}

55
v2/pkg/catalog/loader/filter/tag_filter_test.go
View File

@ -16,19 +16,19 @@ func TestTagBasedFilter(t *testing.T) {
})
t.Run("true", func(t *testing.T) {
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
})
t.Run("false", func(t *testing.T) {
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
})
t.Run("match-extra-tags-positive", func(t *testing.T) {
matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol)
matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
})
t.Run("match-extra-tags-negative", func(t *testing.T) {
matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol)
matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
})
}
@ -37,7 +37,7 @@ func TestTagBasedFilter(t *testing.T) {
filter := New(&Config{
ExcludeTags: []string{"dos"},
})
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
require.Equal(t, ErrExcluded, err, "could not get correct error")
})
@ -47,7 +47,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"dos", "fuzz"},
IncludeTags: []string{"fuzz"},
})
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -56,7 +56,7 @@ func TestTagBasedFilter(t *testing.T) {
Tags: []string{"fuzz"},
ExcludeTags: []string{"fuzz"},
})
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -64,24 +64,31 @@ func TestTagBasedFilter(t *testing.T) {
filter := New(&Config{
Authors: []string{"pdteam"},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
})
t.Run("match-severity", func(t *testing.T) {
filter := New(&Config{
Severities: severity.Severities{severity.High},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
})
t.Run("match-id", func(t *testing.T) {
filter := New(&Config{
IncludeIds: []string{"cve-test"},
})
matched, _ := filter.Match([]string{""}, []string{""}, severity.Low, nil, types.HTTPProtocol, "cve-test")
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-severity", func(t *testing.T) {
filter := New(&Config{
ExcludeSeverities: severity.Severities{severity.Low},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
@ -89,7 +96,7 @@ func TestTagBasedFilter(t *testing.T) {
Tags: []string{"tag"},
ExcludeTags: []string{"another"},
})
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
})
t.Run("match-conditions", func(t *testing.T) {
@ -98,33 +105,43 @@ func TestTagBasedFilter(t *testing.T) {
Tags: []string{"jira"},
Severities: severity.Severities{severity.High},
})
matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol)
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol)
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
})
t.Run("match-type", func(t *testing.T) {
filter := New(&Config{
Protocols: []types.ProtocolType{types.HTTPProtocol},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol, "")
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-id", func(t *testing.T) {
filter := New(&Config{
ExcludeIds: []string{"cve-test"},
})
matched, _ := filter.Match([]string{""}, []string{""}, severity.High, nil, types.DNSProtocol, "cve-test1")
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "cve-test")
require.False(t, matched, "could not get correct match")
})
t.Run("match-exclude-type", func(t *testing.T) {
filter := New(&Config{
ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol, "")
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol, "")
require.False(t, matched, "could not get correct match")
})
}

6
v2/pkg/catalog/loader/loader.go
View File

@ -32,6 +32,8 @@ type Config struct {
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
IncludeIds []string
ExcludeIds []string
Catalog *catalog.Catalog
ExecutorOptions protocols.ExecuterOptions
@ -67,6 +69,8 @@ func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts pr
Severities: options.Severities,
ExcludeSeverities: options.ExcludeSeverities,
IncludeTags: options.IncludeTags,
IncludeIds: options.IncludeIds,
ExcludeIds: options.ExcludeIds,
TemplatesDirectory: options.TemplatesDirectory,
Protocols: options.Protocols,
ExcludeProtocols: options.ExcludeProtocols,
@ -88,6 +92,8 @@ func New(config *Config) (*Store, error) {
Severities: config.Severities,
ExcludeSeverities: config.ExcludeSeverities,
IncludeTags: config.IncludeTags,
IncludeIds: config.IncludeIds,
ExcludeIds: config.ExcludeIds,
Protocols: config.Protocols,
ExcludeProtocols: config.ExcludeProtocols,
}),

8
v2/pkg/parsers/parser.go
View File

@ -40,7 +40,9 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
return false, validationError
}
return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type())
templateId := strings.ToLower(template.ID)
return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type(), templateId)
}
// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
@ -60,12 +62,12 @@ func LoadWorkflow(templatePath string) (bool, error) {
return false, nil
}
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType) (bool, error) {
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType, templateId string) (bool, error) {
templateTags := templateInfo.Tags.ToSlice()
templateAuthors := templateInfo.Authors.ToSlice()
templateSeverity := templateInfo.SeverityHolder.Severity
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType)
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType, templateId)
if err == filter.ErrExcluded {
return false, filter.ErrExcluded

2
v2/pkg/parsers/workflow_loader.go
View File

@ -21,6 +21,8 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
Authors: options.Options.Authors,
Severities: options.Options.Severities,
IncludeTags: options.Options.IncludeTags,
IncludeIds: options.Options.IncludeIds,
ExcludeIds: options.Options.ExcludeIds,
})
pathFilter := filter.NewPathFilter(&filter.PathFilterConfig{
IncludedTemplates: options.Options.IncludeTemplates,

4
v2/pkg/types/types.go
View File

@ -45,6 +45,10 @@ type Options struct {
IncludeTags goflags.NormalizedStringSlice
// IncludeTemplates includes specified templates to be run even while being in denylist
IncludeTemplates goflags.StringSlice
// IncludeIds includes specified ids to be run even while being in denylist
IncludeIds goflags.NormalizedStringSlice
// ExcludeIds contains templates ids to not be executed
ExcludeIds goflags.NormalizedStringSlice
InternalResolversList []string // normalized from resolvers flag as well as file provided.
// ProjectPath allows nuclei to use a user defined project folder

14
v2/pkg/utils/utils.go
View File

@ -3,6 +3,8 @@ package utils
import (
"errors"
"strings"
"github.com/projectdiscovery/fileutil"
)
func IsBlank(value string) bool {
@ -23,3 +25,15 @@ func UnwrapError(err error) error {
}
return err
}
func LoadFile(filename string) ([]string, error) {
var items []string
readfileChan, err := fileutil.ReadFile(filename)
if err != nil {
return nil, err
}
for includeIdLine := range readfileChan {
items = append(items, includeIdLine)
}
return items, nil
}