Added exclude-matchers support for template & matchers (#2218)

* Added exclude-matchers support for template & matchers

* Fixed panics due to typo

* Added support for only template ID + misc cleanup

v2/cmd/nuclei/main.go

@@ -117,6 +117,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.FileNormalizedStringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", []string{}, "templates to exclude based on template ids (comma-separated, file)"),
 		flagSet.FileNormalizedOriginalStringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
 		flagSet.FileNormalizedOriginalStringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory to exclude (comma-separated, file)"),
+		flagSet.FileCommaSeparatedStringSliceVarP(&options.ExcludeMatchers, "exclude-matchers", "em", []string{}, "template matchers to exclude in result"),
 		flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
 		flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
 		flagSet.VarP(&options.Protocols, "type", "pt", fmt.Sprintf("templates to run based on protocol type. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),

v2/internal/runner/runner.go

@@ -32,6 +32,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
@@ -333,6 +334,7 @@ func (r *Runner) RunEnumeration() error {
 		HostErrorsCache: cache,
 		Colorizer:       r.colorizer,
 		ResumeCfg:       r.resumeCfg,
+		ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
 	}
 	engine := core.New(r.options)
 	engine.SetExecuterOptions(executerOpts)

v2/pkg/operators/operators.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/sliceutil"
 )
 
@@ -34,6 +35,11 @@ type Operators struct {
 	MatchersCondition string `yaml:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"`
 	// cached variables that may be used along with request.
 	matchersCondition matchers.ConditionType
+
+	// TemplateID is the ID of the template for matcher
+	TemplateID string
+	// ExcludeMatchers is a list of excludeMatchers items
+	ExcludeMatchers *excludematchers.ExcludeMatchers
 }
 
 // Compile compiles the operators as well as their corresponding matchers and extractors
@@ -238,6 +244,12 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 	}
 
 	for matcherIndex, matcher := range operators.Matchers {
+		// Skip matchers that are in the blocklist
+		if operators.ExcludeMatchers != nil {
+			if operators.ExcludeMatchers.Match(operators.TemplateID, matcher.Name) {
+				continue
+			}
+		}
 		if isMatch, matched := match(data, matcher); isMatch {
 			if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled
 				matcherName := getMatcherName(matcher, matcherIndex)

v2/pkg/protocols/common/utils/excludematchers/excludematchers.go

@@ -0,0 +1,67 @@
+package excludematchers
+
+import (
+	"strings"
+)
+
+// ExcludeMatchers is an instance for excluding matchers with template IDs
+type ExcludeMatchers struct {
+	values       map[string]struct{}
+	templateIDs  map[string]struct{}
+	matcherNames map[string]struct{}
+}
+
+// New returns a new exclude matchers instance
+//
+// Wildcard and non-wildcard values are supported.
+// <template-id>:<matcher-name> is the syntax. Wildcards can be specified
+// using * character for either value.
+//
+//  Ex- http-missing-security-headers:* skips all http-missing-security-header templates
+func New(values []string) *ExcludeMatchers {
+	excludeMatchers := &ExcludeMatchers{
+		values:       make(map[string]struct{}),
+		templateIDs:  make(map[string]struct{}),
+		matcherNames: make(map[string]struct{}),
+	}
+	for _, value := range values {
+		partValues := strings.SplitN(value, ":", 2)
+		if len(partValues) < 2 {
+			// If there is no matcher name, consider it as template ID
+			if _, ok := excludeMatchers.templateIDs[value]; !ok {
+				excludeMatchers.templateIDs[value] = struct{}{}
+			}
+			continue
+		}
+		templateID, matcherName := partValues[0], partValues[1]
+
+		// Handle wildcards
+		if templateID == "*" {
+			if _, ok := excludeMatchers.matcherNames[matcherName]; !ok {
+				excludeMatchers.matcherNames[matcherName] = struct{}{}
+			}
+		} else if matcherName == "*" {
+			if _, ok := excludeMatchers.templateIDs[templateID]; !ok {
+				excludeMatchers.templateIDs[templateID] = struct{}{}
+			}
+		} else {
+			if _, ok := excludeMatchers.values[value]; !ok {
+				excludeMatchers.values[value] = struct{}{}
+			}
+		}
+	}
+	return excludeMatchers
+}
+
+// Match returns true if templateID and matcherName matches the blocklist
+func (e *ExcludeMatchers) Match(templateID, matcherName string) bool {
+	if _, ok := e.templateIDs[templateID]; ok {
+		return true
+	}
+	if _, ok := e.matcherNames[matcherName]; ok {
+		return true
+	}
+	matchName := strings.Join([]string{templateID, matcherName}, ":")
+	_, found := e.values[matchName]
+	return found
+}

v2/pkg/protocols/common/utils/excludematchers/excludematchers_test.go

@@ -0,0 +1,19 @@
+package excludematchers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExcludeMatchers(t *testing.T) {
+	em := New([]string{"test-template:test-matcher", "new-template:*", "*:new-matcher", "only-template-id"})
+
+	require.True(t, em.Match("test-template", "test-matcher"), "could not get template-matcher value")
+	require.False(t, em.Match("test-template", "random-matcher"), "could get template-matcher value")
+
+	require.True(t, em.Match("new-template", "random-matcher"), "could not get template-matcher value wildcard")
+	require.True(t, em.Match("random-template", "new-matcher"), "could not get template-matcher value wildcard")
+
+	require.True(t, em.Match("only-template-id", "test"), "could not get only template id match value")
+}

v2/pkg/protocols/dns/dns.go

@@ -130,6 +130,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/dns/operators_test.go

@@ -226,6 +226,10 @@ func TestDNSMakeResult(t *testing.T) {
 	recursion := false
 	testutils.Init(options)
 	templateID := "testing-dns"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		RequestType: DNSRequestTypeHolder{DNSRequestType: A},
 		Class:       "INET",
@@ -246,11 +250,8 @@ func TestDNSMakeResult(t *testing.T) {
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
 

v2/pkg/protocols/dns/request_test.go

@@ -20,6 +20,10 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	recursion := false
 	testutils.Init(options)
 	templateID := "testing-dns"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		RequestType: DNSRequestTypeHolder{DNSRequestType: A},
 		Class:       "INET",
@@ -40,11 +44,8 @@ func TestDNSExecuteWithResults(t *testing.T) {
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
 

v2/pkg/protocols/file/file.go

@@ -101,6 +101,8 @@ func (request *Request) GetID() string {
 func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/file/operators_test.go

@@ -239,6 +239,10 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 
 	testutils.Init(options)
 	templateID := "testing-file"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		ID:          templateID,
 		MaxSize:     "1Gb",
@@ -254,11 +258,8 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 

v2/pkg/protocols/file/request_test.go

@@ -22,6 +22,10 @@ func TestFileExecuteWithResults(t *testing.T) {
 
 	testutils.Init(options)
 	templateID := "testing-file"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		ID:          templateID,
 		MaxSize:     "1Gb",
@@ -41,11 +45,8 @@ func TestFileExecuteWithResults(t *testing.T) {
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 

v2/pkg/protocols/headless/headless.go

@@ -118,6 +118,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/http/http.go

@@ -270,6 +270,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	}
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if compileErr := compiled.Compile(); compileErr != nil {
 			return errors.Wrap(compileErr, "could not compile operators")
 		}

v2/pkg/protocols/network/network.go

@@ -199,6 +199,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/protocols.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
@@ -66,6 +67,8 @@ type ExecuterOptions struct {
 	StopAtFirstMatch bool
 	// Variables is a list of variables from template
 	Variables variables.Variable
+	// ExcludeMatchers is the list of matchers to exclude
+	ExcludeMatchers *excludematchers.ExcludeMatchers
 
 	Operators []*operators.Operators // only used by offlinehttp module
 

v2/pkg/protocols/ssl/ssl.go

@@ -86,6 +86,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/websocket/websocket.go

@@ -110,6 +110,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/whois/whois.go

@@ -64,6 +64,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/templates/cluster.go

@@ -112,7 +112,7 @@ func ClusterTemplates(templatesList []*Template, options protocols.ExecuterOptio
 			finalTemplatesList = append(finalTemplatesList, &Template{
 				ID:            clusterID,
 				RequestsHTTP:  cluster[0].RequestsHTTP,
-				Executer:      NewExecuter(cluster, &executerOpts),
+				Executer:      NewClusterExecuter(cluster, &executerOpts),
 				TotalRequests: len(cluster[0].RequestsHTTP),
 			})
 			clusterCount += len(cluster)
@@ -123,12 +123,12 @@ func ClusterTemplates(templatesList []*Template, options protocols.ExecuterOptio
 	return finalTemplatesList, clusterCount
 }
 
-// Executer executes a group of requests for a protocol for a clustered
+// ClusterExecuter executes a group of requests for a protocol for a clustered
 // request. It is different from normal executers since the original
 // operators are all combined and post processed after making the request.
 //
 // TODO: We only cluster http requests as of now.
-type Executer struct {
+type ClusterExecuter struct {
 	requests  *http.Request
 	operators []*clusteredOperator
 	options   *protocols.ExecuterOptions
@@ -141,21 +141,25 @@ type clusteredOperator struct {
 	operator     *operators.Operators
 }
 
-var _ protocols.Executer = &Executer{}
+var _ protocols.Executer = &ClusterExecuter{}
 
-// NewExecuter creates a new request executer for list of requests
+// NewClusterExecuter creates a new request executer for list of requests
-func NewExecuter(requests []*Template, options *protocols.ExecuterOptions) *Executer {
+func NewClusterExecuter(requests []*Template, options *protocols.ExecuterOptions) *ClusterExecuter {
-	executer := &Executer{
+	executer := &ClusterExecuter{
 		options:  options,
 		requests: requests[0].RequestsHTTP[0],
 	}
 	for _, req := range requests {
 		if req.RequestsHTTP[0].CompiledOperators != nil {
+			operator := req.RequestsHTTP[0].CompiledOperators
+			operator.TemplateID = req.ID
+			operator.ExcludeMatchers = options.ExcludeMatchers
+
 			executer.operators = append(executer.operators, &clusteredOperator{
+				operator:     operator,
 				templateID:   req.ID,
 				templateInfo: req.Info,
 				templatePath: req.Path,
-				operator:     req.RequestsHTTP[0].CompiledOperators,
 			})
 		}
 	}
@@ -163,19 +167,19 @@ func NewExecuter(requests []*Template, options *protocols.ExecuterOptions) *Exec
 }
 
 // Compile compiles the execution generators preparing any requests possible.
-func (e *Executer) Compile() error {
+func (e *ClusterExecuter) Compile() error {
 	return e.requests.Compile(e.options)
 }
 
 // Requests returns the total number of requests the rule will perform
-func (e *Executer) Requests() int {
+func (e *ClusterExecuter) Requests() int {
 	var count int
 	count += e.requests.Requests()
 	return count
 }
 
 // Execute executes the protocol group and returns true or false if results were found.
-func (e *Executer) Execute(input string) (bool, error) {
+func (e *ClusterExecuter) Execute(input string) (bool, error) {
 	var results bool
 
 	previous := make(map[string]interface{})
@@ -209,7 +213,7 @@ func (e *Executer) Execute(input string) (bool, error) {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error {
+func (e *ClusterExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error {
 	dynamicValues := make(map[string]interface{})
 	err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
 		for _, operator := range e.operators {

v2/pkg/types/types.go

@@ -27,6 +27,8 @@ type Options struct {
 	RemoteTemplateDomainList goflags.StringSlice
 	// 	ExcludedTemplates  specifies the template/templates to exclude
 	ExcludedTemplates goflags.FileOriginalNormalizedStringSlice
+	// ExcludeMatchers is a list of matchers to exclude processing
+	ExcludeMatchers goflags.FileCommaSeparatedStringSlice
 	// CustomHeaders is the list of custom global headers to send with each request.
 	CustomHeaders goflags.FileStringSlice
 	// Vars is the list of custom global vars