Added clustering support for TLS templates (#3209)

* Added clustering support for DNS protocol templates

* Added clustering support for TLS templates

* Fixed randomly populated info block in ssl templates

* Moved to a switch-case + added tests for clustering

v2/pkg/protocols/dns/cluster.go

@@ -12,9 +12,16 @@ func (request *Request) CanCluster(other *Request) bool {
 	if request.Name != other.Name ||
 		request.class != other.class ||
 		request.Retries != other.Retries ||
-		request.question != other.question ||
+		request.question != other.question {
-		*request.Recursion != *other.Recursion {
 		return false
 	}
+	if request.Recursion != nil {
+		if other.Recursion == nil {
+			return false
+		}
+		if *request.Recursion != *other.Recursion {
+			return false
+		}
+	}
 	return true
 }

v2/pkg/protocols/ssl/ssl.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
@@ -77,6 +78,17 @@ type Request struct {
 	options *protocols.ExecuterOptions
 }
 
+// CanCluster returns true if the request can be clustered.
+func (request *Request) CanCluster(other *Request) bool {
+	if len(request.CiperSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" {
+		return false
+	}
+	if request.Address != other.Address || request.ScanMode != other.ScanMode {
+		return false
+	}
+	return true
+}
+
 // Compile compiles the request generators preparing any requests possible.
 func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	request.options = options
@@ -126,6 +138,11 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	return nil
 }
 
+// Options returns executer options for http request
+func (r *Request) Options() *protocols.ExecuterOptions {
+	return r.options
+}
+
 // Requests returns the total number of requests the rule will perform
 func (request *Request) Requests() int {
 	return 1
@@ -315,9 +332,9 @@ func (request *Request) Type() templateTypes.ProtocolType {
 
 func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
-		TemplateID:       types.ToString(request.options.TemplateID),
+		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
-		TemplatePath:     types.ToString(request.options.TemplatePath),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
-		Info:             request.options.TemplateInfo,
+		Info:             wrapped.InternalEvent["template-info"].(model.Info),
 		Type:             types.ToString(wrapped.InternalEvent["type"]),
 		Host:             types.ToString(wrapped.InternalEvent["host"]),
 		Matched:          types.ToString(wrapped.InternalEvent["host"]),

v2/pkg/templates/cluster.go

@@ -23,7 +23,7 @@ import (
 // request which saves time and network resources during execution.
 //
 // The clusterer goes through all the templates, looking for templates with a single
-// HTTP/DNS request to an endpoint (multiple requests aren't clustered as of now).
+// HTTP/DNS/TLS request to an endpoint (multiple requests aren't clustered as of now).
 //
 // All the templates are iterated and any templates with request that is identical
 // to the first individual request is compared for equality.
@@ -35,6 +35,7 @@ import (
 //   - If request paths aren't identical.
 //   - If request headers aren't identical
 //   - Similarly for DNS, only identical DNS requests are clustered to a target.
+//   - Similarly for TLS, only identical TLS requests are clustered to a target.
 //
 // If multiple requests are identified as identical, they are appended to a slice.
 // Finally, the engine creates a single executer with a clusteredexecuter for all templates
@@ -46,33 +47,46 @@ func Cluster(list map[string]*Template) [][]*Template {
 	for key, template := range list {
 		// We only cluster http and dns requests as of now.
 		// Take care of requests that can't be clustered first.
-		if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 {
+		if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 && len(template.RequestsSSL) == 0 {
 			delete(list, key)
 			final = append(final, []*Template{template})
 			continue
 		}
 		delete(list, key) // delete element first so it's not found later.
 
+		var templateType types.ProtocolType
+		switch {
+		case len(template.RequestsDNS) == 1:
+			templateType = types.DNSProtocol
+		case len(template.RequestsHTTP) == 1:
+			templateType = types.HTTPProtocol
+		case len(template.RequestsSSL) == 1:
+			templateType = types.SSLProtocol
+		}
+
 		// Find any/all similar matching request that is identical to
 		// this one and cluster them together for http protocol only.
 		cluster := []*Template{}
-		if len(template.RequestsDNS) == 1 {
+		for otherKey, other := range list {
-			for otherKey, other := range list {
+			switch templateType {
+			case types.DNSProtocol:
 				if len(other.RequestsDNS) == 0 || len(other.RequestsDNS) > 1 {
 					continue
-				}
+				} else if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) {
-				if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) {
 					delete(list, otherKey)
 					cluster = append(cluster, other)
 				}
-			}
+			case types.HTTPProtocol:
-		}
-		if len(template.RequestsHTTP) == 1 {
-			for otherKey, other := range list {
 				if len(other.RequestsHTTP) == 0 || len(other.RequestsHTTP) > 1 {
 					continue
+				} else if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) {
+					delete(list, otherKey)
+					cluster = append(cluster, other)
 				}
-				if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) {
+			case types.SSLProtocol:
+				if len(other.RequestsSSL) == 0 || len(other.RequestsSSL) > 1 {
+					continue
+				} else if template.RequestsSSL[0].CanCluster(other.RequestsSSL[0]) {
 					delete(list, otherKey)
 					cluster = append(cluster, other)
 				}
@@ -123,11 +137,15 @@ func ClusterTemplates(templatesList []*Template, options protocols.ExecuterOptio
 			for _, req := range cluster[0].RequestsHTTP {
 				req.Options().TemplateID = clusterID
 			}
+			for _, req := range cluster[0].RequestsSSL {
+				req.Options().TemplateID = clusterID
+			}
 			executerOpts.TemplateID = clusterID
 			finalTemplatesList = append(finalTemplatesList, &Template{
 				ID:            clusterID,
 				RequestsDNS:   cluster[0].RequestsDNS,
 				RequestsHTTP:  cluster[0].RequestsHTTP,
+				RequestsSSL:   cluster[0].RequestsSSL,
 				Executer:      NewClusterExecuter(cluster, &executerOpts),
 				TotalRequests: len(cluster[0].RequestsHTTP) + len(cluster[0].RequestsDNS),
 			})
@@ -167,6 +185,9 @@ func NewClusterExecuter(requests []*Template, options *protocols.ExecuterOptions
 	} else if len(requests[0].RequestsHTTP) == 1 {
 		executer.templateType = types.HTTPProtocol
 		executer.requests = requests[0].RequestsHTTP[0]
+	} else if len(requests[0].RequestsSSL) == 1 {
+		executer.templateType = types.SSLProtocol
+		executer.requests = requests[0].RequestsSSL[0]
 	}
 	appendOperator := func(req *Template, operator *operators.Operators) {
 		operator.TemplateID = req.ID
@@ -188,6 +209,10 @@ func NewClusterExecuter(requests []*Template, options *protocols.ExecuterOptions
 			if req.RequestsHTTP[0].CompiledOperators != nil {
 				appendOperator(req, req.RequestsHTTP[0].CompiledOperators)
 			}
+		} else if executer.templateType == types.SSLProtocol {
+			if req.RequestsSSL[0].CompiledOperators != nil {
+				appendOperator(req, req.RequestsSSL[0].CompiledOperators)
+			}
 		}
 	}
 	return executer

v2/pkg/templates/cluster_test.go

@@ -1 +1,58 @@
 package templates
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClusterTemplates(t *testing.T) {
+	tests := []struct {
+		name      string
+		templates map[string]*Template
+		expected  [][]*Template
+	}{
+		{
+			name: "http-cluster-get",
+			templates: map[string]*Template{
+				"first.yaml":  {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}},
+				"second.yaml": {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}},
+			},
+			expected: [][]*Template{{
+				{RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}},
+				{RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}},
+			}},
+		},
+		{
+			name: "no-http-cluster",
+			templates: map[string]*Template{
+				"first.yaml":  {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/random"}}}},
+				"second.yaml": {RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/another"}}}},
+			},
+			expected: [][]*Template{
+				{{RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/random"}}}}},
+				{{RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/another"}}}}},
+			},
+		},
+		{
+			name: "dns-cluster",
+			templates: map[string]*Template{
+				"first.yaml":  {RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}},
+				"second.yaml": {RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}},
+			},
+			expected: [][]*Template{{
+				{RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}},
+				{RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}},
+			}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			returned := Cluster(test.templates)
+			require.ElementsMatch(t, returned, test.expected, "could not get cluster results")
+		})
+	}
+}