Misc work on extractor and replacer

v2/pkg/executer/executer_http.go

@@ -549,61 +549,6 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 		result.Unlock()
 	}
 
-	matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
-	for _, matcher := range e.bulkHTTPRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.Match(resp, body, headers, duration, matchData) {
-			// If the condition is AND we haven't matched, try next request.
-			if matcherCondition == matchers.ANDCondition {
-				return nil
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition {
-				result.Lock()
-				result.Matches[matcher.Name] = nil
-				// probably redundant but ensures we snapshot current payload values when matchers are valid
-				result.Meta = request.Meta
-				result.GotResults = true
-				result.Unlock()
-				e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL)
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults, outputExtractorResults []string
-
-	for _, extractor := range e.bulkHTTPRequest.Extractors {
-		for match := range extractor.Extract(resp, body, headers) {
-			if _, ok := dynamicvalues[extractor.Name]; !ok {
-				dynamicvalues[extractor.Name] = match
-			}
-
-			extractorResults = append(extractorResults, match)
-
-			if !extractor.Internal {
-				outputExtractorResults = append(outputExtractorResults, match)
-			}
-		}
-		// probably redundant but ensures we snapshot current payload values when extractors are valid
-		result.Lock()
-		result.Meta = request.Meta
-		result.Extractions[extractor.Name] = extractorResults
-		result.Unlock()
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL)
-		result.Lock()
-		result.GotResults = true
-		result.Unlock()
-	}
-
 	return nil
 }
 

v2/pkg/operators/operators.go

@@ -24,3 +24,64 @@ type Operators struct {
 func (r *Operators) GetMatchersCondition() matchers.ConditionType {
 	return r.matchersCondition
 }
+
+// Result is a result structure created from operators running on data.
+type Result struct {
+	// Matches is a map of matcher names that we matched
+	Matches map[string]struct{}
+	// Extracts contains all the data extracted from inputs
+	Extracts map[string][]string
+	// DynamicValues contains any dynamic values to be templated
+	DynamicValues map[string]string
+}
+
+// Execute executes the operators on data and returns a result structure
+func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
+	matcherCondition := r.GetMatchersCondition()
+
+	result := &Result{
+		Matches:       make(map[string]struct{}),
+		Extracts:      make(map[string][]string),
+		DynamicValues: make(map[string]string),
+	}
+	for _, matcher := range r.Matchers {
+		// Check if the matcher matched
+		if !matcher.Match(data) {
+			// If the condition is AND we haven't matched, try next request.
+			if matcherCondition == matchers.ANDCondition {
+				return nil, false
+			}
+		} else {
+			// If the matcher has matched, and its an OR
+			// write the first output then move to next matcher.
+			if matcherCondition == matchers.ORCondition {
+				result.Matches[matcher.Name] = struct{}{}
+			}
+		}
+	}
+
+	// All matchers have successfully completed so now start with the
+	// next task which is extraction of input from matchers.
+	var extractorResults, outputExtractorResults []string
+	for _, extractor := range r.Extractors {
+		for match := range extractor.Extract(data) {
+			extractorResults = append(extractorResults, match)
+
+			if extractor.Internal {
+				if _, ok := result.DynamicValues[extractor.Name]; !ok {
+					result.DynamicValues[extractor.Name] = match
+				}
+			} else {
+				outputExtractorResults = append(outputExtractorResults, match)
+			}
+		}
+		result.Extracts[extractor.Name] = extractorResults
+	}
+
+	// Write a final string of output if matcher type is
+	// AND or if we have extractors for the mechanism too.
+	if len(result.Extracts) > 0 || len(result.Matches) > 0 || matcherCondition == matchers.ANDCondition {
+		return result, true
+	}
+	return nil, false
+}

v2/pkg/protocols/common/generators/generators.go

@@ -110,7 +110,7 @@ func (i *Iterator) sniperValue() map[string]interface{} {
 		if !p.Next() {
 			p.ResetPosition()
 		}
-		values[p.Keyword()] = p.Value()
+		values[p.name] = p.Value()
 		p.IncrementPosition()
 	}
 	return values
@@ -124,7 +124,7 @@ func (i *Iterator) pitchforkValue() map[string]interface{} {
 		if !p.Next() {
 			p.ResetPosition()
 		}
-		values[p.Keyword()] = p.Value()
+		values[p.name] = p.Value()
 		p.IncrementPosition()
 	}
 	return values
@@ -154,7 +154,7 @@ func (i *Iterator) clusterbombValue() map[string]interface{} {
 			p.ResetPosition()
 			signalNext = true
 		}
-		values[p.Keyword()] = p.Value()
+		values[p.name] = p.Value()
 		if first {
 			p.IncrementPosition()
 			first = false
@@ -189,11 +189,6 @@ func (i *payloadIterator) Next() bool {
 	return true
 }
 
-// Position returns the position of reader in payload iterator
-func (i *payloadIterator) Position() int {
-	return i.index
-}
-
 func (i *payloadIterator) ResetPosition() {
 	i.index = 0
 }
@@ -210,7 +205,3 @@ func (i *payloadIterator) Value() string {
 func (i *payloadIterator) Total() int {
 	return len(i.values)
 }
-
-func (i *payloadIterator) Keyword() string {
-	return i.name
-}

v2/pkg/protocols/common/replacer/replacer.go

@@ -0,0 +1,31 @@
+package replacer
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	markerGeneral          = "§"
+	markerParenthesisOpen  = "{{"
+	markerParenthesisClose = "}}"
+)
+
+// New creates a new replacer structure for values replacement on the fly.
+func New(values map[string]interface{}) *strings.Replacer {
+	replacerItems := make([]string, 0, len(values)*4)
+
+	for key, val := range values {
+		valueStr := fmt.Sprintf("%s", val)
+
+		replacerItems = append(replacerItems,
+			fmt.Sprintf("%s%s%s", markerParenthesisOpen, key, markerParenthesisClose),
+			valueStr,
+		)
+		replacerItems = append(replacerItems,
+			fmt.Sprintf("%s%s%s", markerGeneral, key, markerGeneral),
+			valueStr,
+		)
+	}
+	return strings.NewReplacer(replacerItems...)
+}

v2/pkg/protocols/dns/dns.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/miekg/dns"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 )
 
 // Request contains a DNS protocol request to be made from a template
@@ -49,7 +50,7 @@ func (r *Request) Make(domain string) (*dns.Msg, error) {
 
 	var q dns.Question
 
-	replacer := newReplacer(map[string]interface{}{"FQDN": domain})
+	replacer := replacer.New(map[string]interface{}{"FQDN": domain})
 
 	q.Name = dns.Fqdn(replacer.Replace(r.Name))
 	q.Qclass = classToInt(r.Class)

v2/pkg/requests/dns-request.go

@@ -1,121 +0,0 @@
-package requests
-
-import (
-	"strings"
-
-	"github.com/miekg/dns"
-	"github.com/projectdiscovery/nuclei/v2/pkg/extractors"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-)
-
-// DNSRequest contains a request to be made from a template
-type DNSRequest struct {
-	Recursion bool `yaml:"recursion"`
-	// Path contains the path/s for the request
-	Name    string `yaml:"name"`
-	Type    string `yaml:"type"`
-	Class   string `yaml:"class"`
-	Retries int    `yaml:"retries"`
-	// Raw contains a raw request
-	Raw string `yaml:"raw,omitempty"`
-
-	// Matchers contains the detection mechanism for the request to identify
-	// whether the request was successful
-	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
-	// matchersCondition is internal condition for the matchers.
-	matchersCondition matchers.ConditionType
-	// MatchersCondition is the condition of the matchers
-	// whether to use AND or OR. Default is OR.
-	MatchersCondition string `yaml:"matchers-condition,omitempty"`
-	// Extractors contains the extraction mechanism for the request to identify
-	// and extract parts of the response.
-	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
-}
-
-// GetMatchersCondition returns the condition for the matcher
-func (r *DNSRequest) GetMatchersCondition() matchers.ConditionType {
-	return r.matchersCondition
-}
-
-// SetMatchersCondition sets the condition for the matcher
-func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) {
-	r.matchersCondition = condition
-}
-
-// Returns the total number of requests the YAML rule will perform
-func (r *DNSRequest) GetRequestCount() int64 {
-	return 1
-}
-
-// MakeDNSRequest creates a *dns.Request from a request template
-func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
-	domain = dns.Fqdn(domain)
-
-	// Build a request on the specified URL
-	req := new(dns.Msg)
-	req.Id = dns.Id()
-	req.RecursionDesired = r.Recursion
-
-	var q dns.Question
-
-	replacer := newReplacer(map[string]interface{}{"FQDN": domain})
-
-	q.Name = dns.Fqdn(replacer.Replace(r.Name))
-	q.Qclass = toQClass(r.Class)
-	q.Qtype = toQType(r.Type)
-
-	req.Question = append(req.Question, q)
-
-	return req, nil
-}
-
-func toQType(ttype string) (rtype uint16) {
-	ttype = strings.TrimSpace(strings.ToUpper(ttype))
-
-	switch ttype {
-	case "A":
-		rtype = dns.TypeA
-	case "NS":
-		rtype = dns.TypeNS
-	case "CNAME":
-		rtype = dns.TypeCNAME
-	case "SOA":
-		rtype = dns.TypeSOA
-	case "PTR":
-		rtype = dns.TypePTR
-	case "MX":
-		rtype = dns.TypeMX
-	case "TXT":
-		rtype = dns.TypeTXT
-	case "AAAA":
-		rtype = dns.TypeAAAA
-	default:
-		rtype = dns.TypeA
-	}
-
-	return
-}
-
-func toQClass(tclass string) (rclass uint16) {
-	tclass = strings.TrimSpace(strings.ToUpper(tclass))
-
-	switch tclass {
-	case "INET":
-		rclass = dns.ClassINET
-	case "CSNET":
-		rclass = dns.ClassCSNET
-	case "CHAOS":
-		rclass = dns.ClassCHAOS
-	case "HESIOD":
-		rclass = dns.ClassHESIOD
-	case "NONE":
-		rclass = dns.ClassNONE
-	case "ANY":
-		rclass = dns.ClassANY
-	default:
-		// Use INET by default.
-		rclass = dns.ClassINET
-	}
-
-	return
-}