More work on other protocol additions

v2/go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/owenrumney/go-sarif v1.0.11
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e
+	github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb
 	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa

v2/go.sum

@@ -341,6 +341,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod
 github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb h1:h+HvVw51KUvcO4Tww1QCd95D6MWV/6wpXuSbmFpPQSI=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII=
 github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
 github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=

v2/pkg/protocols/others/ssl/ssl.go

@@ -3,7 +3,6 @@ package ssl
 import (
 	"context"
 	"crypto/tls"
-	"fmt"
 	"net"
 	"net/url"
 	"strings"
@@ -12,11 +11,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -61,53 +59,12 @@ func (r *Request) GetID() string {
 	return ""
 }
 
-// Match performs matching operation for a matcher on model and returns true or false.
-func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
-	partItem, ok := data[matcher.Part]
-	if !ok {
-		return false
-	}
-	item := types.ToString(partItem)
-
-	switch matcher.GetType() {
-	case matchers.SizeMatcher:
-		return matcher.Result(matcher.MatchSize(len(item)))
-	case matchers.WordsMatcher:
-		return matcher.Result(matcher.MatchWords(item))
-	case matchers.RegexMatcher:
-		return matcher.Result(matcher.MatchRegex(item))
-	case matchers.BinaryMatcher:
-		return matcher.Result(matcher.MatchBinary(item))
-	case matchers.DSLMatcher:
-		return matcher.Result(matcher.MatchDSL(data))
-	}
-	return false
-}
-
-// Extract performs extracting operation for an extractor on model and returns true or false.
-func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
-	item, ok := data[extractor.Part]
-	if !ok {
-		return nil
-	}
-	itemStr := types.ToString(item)
-
-	switch extractor.GetType() {
-	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(itemStr)
-	case extractors.KValExtractor:
-		return extractor.ExtractKval(data)
-	}
-	return nil
-}
-
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	address, err := getAddress(input)
 	if err != nil {
 		return nil
 	}
-	fmt.Printf("address: %v\n", address)
 	hostname, _, _ := net.SplitHostPort(input)
 
 	config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname}
@@ -136,9 +93,9 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu
 	event := &output.InternalWrappedEvent{InternalEvent: data}
 	if r.CompiledOperators != nil {
 		var ok bool
-		event.OperatorsResult, ok = r.CompiledOperators.Execute(data, r.Match, r.Extract)
+		event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc)
 		if ok && event.OperatorsResult != nil {
-			event.Results = r.makeResultEvent(event)
+			event.Results = utils.MakeResultEvent(event, r.makeResultEventItem)
 		}
 		callback(event)
 	}
@@ -163,34 +120,6 @@ func getAddress(toTest string) (string, error) {
 	return toTest, nil
 }
 
-// makeResultEvent creates a result event from internal wrapped event
-func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
-	if len(wrapped.OperatorsResult.DynamicValues) > 0 {
-		return nil
-	}
-	results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
-
-	// If we have multiple matchers with names, write each of them separately.
-	if len(wrapped.OperatorsResult.Matches) > 0 {
-		for k := range wrapped.OperatorsResult.Matches {
-			data := r.makeResultEventItem(wrapped)
-			data.MatcherName = k
-			results = append(results, data)
-		}
-	} else if len(wrapped.OperatorsResult.Extracts) > 0 {
-		for k, v := range wrapped.OperatorsResult.Extracts {
-			data := r.makeResultEventItem(wrapped)
-			data.ExtractedResults = v
-			data.ExtractorName = k
-			results = append(results, data)
-		}
-	} else {
-		data := r.makeResultEventItem(wrapped)
-		results = append(results, data)
-	}
-	return results
-}
-
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(r.options.TemplateID),

v2/pkg/protocols/others/utils/utils.go

@@ -0,0 +1,83 @@
+package utils
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// MakeResultEventItemFunc returns a result event for an internal wrapped event item
+type MakeResultEventItemFunc func(wrapped *output.InternalWrappedEvent) *output.ResultEvent
+
+// MakeResultEvent creates a result event from internal wrapped event
+func MakeResultEvent(wrapped *output.InternalWrappedEvent, makeEventItemFunc MakeResultEventItemFunc) []*output.ResultEvent {
+	if len(wrapped.OperatorsResult.DynamicValues) > 0 {
+		return nil
+	}
+	results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
+
+	// If we have multiple matchers with names, write each of them separately.
+	if len(wrapped.OperatorsResult.Matches) > 0 {
+		for k := range wrapped.OperatorsResult.Matches {
+			data := makeEventItemFunc(wrapped)
+			data.MatcherName = k
+			results = append(results, data)
+		}
+	} else if len(wrapped.OperatorsResult.Extracts) > 0 {
+		for k, v := range wrapped.OperatorsResult.Extracts {
+			data := makeEventItemFunc(wrapped)
+			data.ExtractedResults = v
+			data.ExtractorName = k
+			results = append(results, data)
+		}
+	} else {
+		data := makeEventItemFunc(wrapped)
+		results = append(results, data)
+	}
+	return results
+}
+
+// ExtractFunc performs extracting operation for an extractor on model and returns true or false.
+func ExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
+	item, ok := data[extractor.Part]
+	if !ok {
+		return nil
+	}
+	itemStr := types.ToString(item)
+
+	switch extractor.GetType() {
+	case extractors.RegexExtractor:
+		return extractor.ExtractRegex(itemStr)
+	case extractors.KValExtractor:
+		return extractor.ExtractKval(data)
+	case extractors.JSONExtractor:
+		return extractor.ExtractJSON(itemStr)
+	case extractors.XPathExtractor:
+		return extractor.ExtractHTML(itemStr)
+	}
+	return nil
+}
+
+// MatchFunc performs matching operation for a matcher on model and returns true or false.
+func MatchFunc(data map[string]interface{}, matcher *matchers.Matcher) bool {
+	partItem, ok := data[matcher.Part]
+	if !ok {
+		return false
+	}
+	item := types.ToString(partItem)
+
+	switch matcher.GetType() {
+	case matchers.SizeMatcher:
+		return matcher.Result(matcher.MatchSize(len(item)))
+	case matchers.WordsMatcher:
+		return matcher.Result(matcher.MatchWords(item))
+	case matchers.RegexMatcher:
+		return matcher.Result(matcher.MatchRegex(item))
+	case matchers.BinaryMatcher:
+		return matcher.Result(matcher.MatchBinary(item))
+	case matchers.DSLMatcher:
+		return matcher.Result(matcher.MatchDSL(data))
+	}
+	return false
+}

v2/pkg/protocols/others/wss/wss.go

@@ -0,0 +1 @@
+package wss

v2/pkg/protocols/protocols.go

@@ -6,8 +6,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
@@ -74,10 +72,6 @@ type Request interface {
 	// condition matching. So, two requests can be sent and their match can
 	// be evaluated from the third request by using the IDs for both requests.
 	GetID() string
-	// Match performs matching operation for a matcher on model and returns true or false.
-	Match(data map[string]interface{}, matcher *matchers.Matcher) bool
-	// Extract performs extracting operation for an extractor on model and returns true or false.
-	Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
 	// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 	ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
 }

v2/pkg/templates/compile.go

@@ -70,7 +70,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 	options.TemplatePath = filePath
 
 	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows)+len(template.RequestsSSL) == 0 {
+	if template.Requests() == 0 {
 		return nil, fmt.Errorf("no requests defined for %s", template.ID)
 	}
 
@@ -129,6 +129,19 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 	return template, nil
 }
 
+// Requests returns the total number of requests for the template.
+func (t *Template) Requests() int {
+	sum := len(t.RequestsDNS) +
+		len(t.RequestsHTTP) +
+		len(t.RequestsFile) +
+		len(t.RequestsNetwork) +
+		len(t.RequestsHeadless) +
+		len(t.Workflows) +
+		len(t.RequestsSSL)
+	return sum
+}
+
+// makeRequestsForTemplate compiles all the requests for the template.
 func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptions) {
 	requests := []protocols.Request{}