Added executor + refactor

internal/runner/options.go

@@ -7,15 +7,6 @@ import (
 	"github.com/projectdiscovery/gologger"
 )
 
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
-	"9.9.9.9:53", // Quad9
-}
-
 // Options contains the configuration options for tuning
 // the template requesting process.
 type Options struct {

internal/runner/runner.go

@@ -165,59 +165,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 func (r *Runner) sendRequest(template *templates.Template, request interface{}, URL string, writer *bufio.Writer, httpclient *retryablehttp.Client, dnsclient *retryabledns.Client) {
 	switch request.(type) {
 	case *requests.HTTPRequest:
-		if !isURL(URL) {
-			break
-		}
-		httpRequest := request.(*requests.HTTPRequest)
-
-		// Compile each request for the template based on the URL
-		compiledRequest, err := httpRequest.MakeHTTPRequest(URL)
-		if err != nil {
-			gologger.Warningf("[%s] Could not make request %s: %s\n", template.ID, URL, err)
-			return
-		}
-
-		// Send the request to the target servers
-		for _, req := range compiledRequest {
-			resp, err := httpclient.Do(req)
-			if err != nil {
-				if resp != nil {
-					resp.Body.Close()
-				}
-				gologger.Warningf("[%s] Could not send request %s: %s\n", template.ID, URL, err)
-				return
-			}
-
-			data, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				io.Copy(ioutil.Discard, resp.Body)
-				resp.Body.Close()
-				gologger.Warningf("[%s] Could not read body %s: %s\n", template.ID, URL, err)
-				continue
-			}
-			resp.Body.Close()
-
-			body := unsafeToString(data)
-
-			var headers string
-			for _, matcher := range httpRequest.Matchers {
-				// Only build the headers string if the matcher asks for it
-				part := matcher.GetPart()
-				if part == matchers.AllPart || part == matchers.HeaderPart && headers == "" {
-					headers = headersToString(resp.Header)
-				}
-
-				// Check if the matcher matched
-				if matcher.Match(resp, body, headers) {
-					// If there is an extractor, run it.
-					var extractorResults []string
-					for _, extractor := range httpRequest.Extractors {
-						part := extractor.GetPart()
-						if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" {
-							headers = headersToString(resp.Header)
-						}
-						extractorResults = append(extractorResults, extractor.Extract(body, headers)...)
-					}
+	
 
 					// All the matchers matched, print the output on the screen
 					output := buildOutputHTTP(template, req, extractorResults, matcher)
@@ -232,40 +180,6 @@ func (r *Runner) sendRequest(template *templates.Template, request interface{},
 			}
 		}
 	case *requests.DNSRequest:
-		// eventually extracts dns from url
-		var domain string = URL
-		if isURL(URL) {
-			domain = extractDomain(URL)
-		}
-
-		dnsRequest := request.(*requests.DNSRequest)
-
-		// Compile each request for the template based on the URL
-		compiledRequest, err := dnsRequest.MakeDNSRequest(domain)
-		if err != nil {
-			gologger.Warningf("[%s] Could not make request %s: %s\n", template.ID, domain, err)
-			return
-		}
-
-		// Send the request to the target servers
-		resp, err := dnsclient.Do(compiledRequest)
-		if err != nil {
-			gologger.Warningf("[%s] Could not send request %s: %s\n", template.ID, domain, err)
-			return
-		}
-
-		for _, matcher := range dnsRequest.Matchers {
-			// Check if the matcher matched
-			if !matcher.MatchDNS(resp) {
-				return
-			}
-		}
-
-		// If there is an extractor, run it.
-		var extractorResults []string
-		for _, extractor := range dnsRequest.Extractors {
-			extractorResults = append(extractorResults, extractor.ExtractDNS(resp.String())...)
-		}
 
 		// All the matchers matched, print the output on the screen
 		output := buildOutputDNS(template, domain, extractorResults)
@@ -278,117 +192,3 @@ func (r *Runner) sendRequest(template *templates.Template, request interface{},
 		}
 	}
 }
-
-// buildOutputHTTP builds an output text for writing results
-func buildOutputHTTP(template *templates.Template, req *retryablehttp.Request, extractorResults []string, matcher *matchers.Matcher) string {
-	builder := &strings.Builder{}
-	builder.WriteRune('[')
-	builder.WriteString(template.ID)
-	if len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] ")
-
-	// Escape the URL by replacing all % with %%
-	URL := req.URL.String()
-	escapedURL := strings.Replace(URL, "%", "%%", -1)
-	builder.WriteString(escapedURL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	return builder.String()
-}
-
-// buildOutput builds an output text for writing results
-func buildOutputDNS(template *templates.Template, domain string, extractorResults []string) string {
-	builder := &strings.Builder{}
-	builder.WriteRune('[')
-	builder.WriteString(template.ID)
-	builder.WriteString("] [dns] ")
-
-	builder.WriteString(domain)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	return builder.String()
-}
-
-// makeHTTPClient creates a HTTP client with configurable redirect field
-func (r *Runner) makeHTTPClientByRequest(request interface{}) *retryablehttp.Client {
-
-	redirects := false
-	maxRedirects := 0
-	// Request is HTTP
-	if httpRequest, ok := request.(requests.HTTPRequest); ok {
-		redirects = httpRequest.Redirects
-		maxRedirects = httpRequest.MaxRedirects
-	}
-
-	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
-	retryablehttpOptions.RetryWaitMax = 10 * time.Second
-	retryablehttpOptions.RetryMax = r.options.Retries
-
-	// Create the HTTP Client
-	client := retryablehttp.NewWithHTTPClient(&http.Client{
-		Transport: &http.Transport{
-			MaxIdleConnsPerHost: -1,
-			TLSClientConfig: &tls.Config{
-				Renegotiation:      tls.RenegotiateOnceAsClient,
-				InsecureSkipVerify: true,
-			},
-			DisableKeepAlives: true,
-		},
-		Timeout: time.Duration(r.options.Timeout) * time.Second,
-		CheckRedirect: func(_ *http.Request, requests []*http.Request) error {
-			if !redirects {
-				return http.ErrUseLastResponse
-			}
-			if maxRedirects == 0 {
-				if len(requests) > 10 {
-					return http.ErrUseLastResponse
-				}
-				return nil
-			}
-			if len(requests) > maxRedirects {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		},
-	}, retryablehttpOptions)
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
-	return client
-}
-
-// makeHTTPClient creates a HTTP client with configurable redirect field
-func (r *Runner) makeDNSClientByRequest(request interface{}) *retryabledns.Client {
-	retries := r.options.Retries
-	if dnsRequest, ok := request.(*requests.DNSRequest); ok {
-		retries = dnsRequest.Retries
-	}
-
-	dnsClient, _ := retryabledns.New(DefaultResolvers, retries)
-	return dnsClient
-}

pkg/executor/dns_utils.go

@@ -1,38 +1,11 @@
-package runner
+package executor
 
 import (
-	"net/http"
 	"net/url"
-	"strings"
-	"unsafe"
 
 	"github.com/asaskevich/govalidator"
 )
 
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-			if i != len(values)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}
-
 // isURL tests a string to determine if it is a well-structured url or not.
 func isURL(toTest string) bool {
 	_, err := url.ParseRequestURI(toTest)
@@ -44,17 +17,17 @@ func isURL(toTest string) bool {
 	if err != nil || u.Scheme == "" || u.Host == "" {
 		return false
 	}
-
 	return true
 }
 
+// extractDomain extracts the domain name of a URL
 func extractDomain(URL string) string {
 	u, err := url.Parse(URL)
 	if err != nil {
 		return ""
 	}
-
-	return u.Hostname()
+	hostname := u.Hostname()
+	return hostname
 }
 
 // isDNS tests a string to determine if it is a well-structured dns or not

pkg/executor/executer_http.go

@@ -0,0 +1,127 @@
+package executor
+
+import (
+	"crypto/tls"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/pkg/extractors"
+	"github.com/projectdiscovery/nuclei/pkg/matchers"
+	"github.com/projectdiscovery/nuclei/pkg/requests"
+	"github.com/projectdiscovery/nuclei/pkg/templates"
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+// HTTPExecutor is client for performing HTTP requests
+// for a template.
+type HTTPExecutor struct {
+	httpClient  *retryablehttp.Client
+	template    *templates.Template
+	httpRequest *requests.HTTPRequest
+}
+
+// NewHTTPExecutor creates a new HTTP executor from a template
+// and a HTTP request query.
+func NewHTTPExecutor(template *templates.Template, httpRequest *requests.HTTPRequest) *HTTPExecutor {
+	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
+	retryablehttpOptions.RetryWaitMax = 10 * time.Second
+	retryablehttpOptions.RetryMax = r.options.Retries
+
+	// Create the HTTP Client
+	client := retryablehttp.NewWithHTTPClient(&http.Client{
+		Transport: &http.Transport{
+			MaxIdleConnsPerHost: -1,
+			TLSClientConfig: &tls.Config{
+				Renegotiation:      tls.RenegotiateOnceAsClient,
+				InsecureSkipVerify: true,
+			},
+			DisableKeepAlives: true,
+		},
+		Timeout: time.Duration(r.options.Timeout) * time.Second,
+		CheckRedirect: func(_ *http.Request, requests []*http.Request) error {
+			if !httpRequest.Redirects {
+				return http.ErrUseLastResponse
+			}
+			if httpRequest.MaxRedirects == 0 {
+				if len(requests) > 10 {
+					return http.ErrUseLastResponse
+				}
+				return nil
+			}
+			if len(requests) > httpRequest.MaxRedirects {
+				return http.ErrUseLastResponse
+			}
+			return nil
+		},
+	}, retryablehttpOptions)
+	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
+
+	executer := &HTTPExecutor{
+		httpClient:  client,
+		template:    template,
+		httpRequest: httpRequest,
+	}
+	return executer
+}
+
+// ExecuteHTTP executes the HTTP request on a URL
+func (e *HTTPExecutor) ExecuteHTTP(URL string) {
+	if !isURL(URL) {
+		return
+	}
+
+	// Compile each request for the template based on the URL
+	compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
+	if err != nil {
+		gologger.Warningf("[%s] Could not make request %s: %s\n", e.template.ID, URL, err)
+		return
+	}
+
+	// Send the request to the target servers
+	for _, req := range compiledRequest {
+		resp, err := e.httpClient.Do(req)
+		if err != nil {
+			if resp != nil {
+				resp.Body.Close()
+			}
+			gologger.Warningf("[%s] Could not send request %s: %s\n", e.template.ID, URL, err)
+			return
+		}
+
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+			gologger.Warningf("[%s] Could not read body %s: %s\n", e.template.ID, URL, err)
+			continue
+		}
+		resp.Body.Close()
+
+		body := unsafeToString(data)
+
+		var headers string
+		for _, matcher := range e.httpRequest.Matchers {
+			// Only build the headers string if the matcher asks for it
+			part := matcher.GetPart()
+			if part == matchers.AllPart || part == matchers.HeaderPart && headers == "" {
+				headers = headersToString(resp.Header)
+			}
+
+			// Check if the matcher matched
+			if matcher.Match(resp, body, headers) {
+				// If there is an extractor, run it.
+				var extractorResults []string
+				for _, extractor := range e.httpRequest.Extractors {
+					part := extractor.GetPart()
+					if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" {
+						headers = headersToString(resp.Header)
+					}
+					extractorResults = append(extractorResults, extractor.Extract(body, headers)...)
+				}
+			}
+		}
+	}
+}

pkg/executor/executor_dns.go

@@ -0,0 +1,75 @@
+package executor
+
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/pkg/requests"
+	"github.com/projectdiscovery/nuclei/pkg/templates"
+	retryabledns "github.com/projectdiscovery/retryabledns"
+)
+
+// DNSExecutor is a client for performing a DNS request
+// for a template.
+type DNSExecutor struct {
+	dnsClient  *retryabledns.Client
+	template   *templates.Template
+	dnsRequest *requests.DNSRequest
+}
+
+// DefaultResolvers contains the list of resolvers known to be trusted.
+var DefaultResolvers = []string{
+	"1.1.1.1:53", // Cloudflare
+	"1.0.0.1:53", // Cloudflare
+	"8.8.8.8:53", // Google
+	"8.8.4.4:53", // Google
+}
+
+// NewDNSExecutor creates a new DNS executor from a template
+// and a DNS request query.
+func NewDNSExecutor(template *templates.Template, dnsRequest *requests.DNSRequest) *DNSExecutor {
+	dnsClient := retryabledns.New(DefaultResolvers, dnsRequest.Retries)
+
+	executer := &DNSExecutor{
+		dnsClient:  dnsClient,
+		template:   template,
+		dnsRequest: dnsRequest,
+	}
+	return executer
+}
+
+// ExecuteDNS executes the DNS request on a URL
+func (e *DNSExecutor) ExecuteDNS(URL string) {
+	// Parse the URL and return domain if URL.
+	var domain string
+	if isURL(URL) {
+		domain = extractDomain(URL)
+	} else {
+		domain = URL
+	}
+
+	// Compile each request for the template based on the URL
+	compiledRequest, err := e.dnsRequest.MakeDNSRequest(URL)
+	if err != nil {
+		gologger.Warningf("[%s] Could not make request %s: %s\n", e.template.ID, domain, err)
+		return
+	}
+
+	// Send the request to the target servers
+	resp, err := e.dnsClient.Do(compiledRequest)
+	if err != nil {
+		gologger.Warningf("[%s] Could not send request %s: %s\n", e.template.ID, domain, err)
+		return
+	}
+
+	for _, matcher := range e.dnsRequest.Matchers {
+		// Check if the matcher matched
+		if !matcher.MatchDNS(resp) {
+			return
+		}
+	}
+
+	// If there is an extractor, run it.
+	var extractorResults []string
+	for _, extractor := range e.dnsRequest.Extractors {
+		extractorResults = append(extractorResults, extractor.ExtractDNS(resp.String())...)
+	}
+}

pkg/executor/http_utils.go

@@ -0,0 +1,31 @@
+package executor
+
+import (
+	"net/http"
+	"strings"
+	"unsafe"
+)
+
+// unsafeToString converts byte slice to string with zero allocations
+func unsafeToString(bs []byte) string {
+	return *(*string)(unsafe.Pointer(&bs))
+}
+
+// headersToString converts http headers to string
+func headersToString(headers http.Header) string {
+	builder := &strings.Builder{}
+
+	for header, values := range headers {
+		builder.WriteString(header)
+		builder.WriteString(": ")
+
+		for i, value := range values {
+			builder.WriteString(value)
+			if i != len(values)-1 {
+				builder.WriteRune(',')
+			}
+		}
+		builder.WriteRune('\n')
+	}
+	return builder.String()
+}

pkg/executor/output_dns.go

@@ -0,0 +1,30 @@
+package executor
+
+import (
+	"strings"
+)
+
+// buildOutput builds an output text for writing results
+func (e *DNSExecutor) buildOutputDNS(domain string, extractorResults []string) string {
+	builder := &strings.Builder{}
+	builder.WriteRune('[')
+	builder.WriteString(e.template.ID)
+	builder.WriteString("] [dns] ")
+
+	builder.WriteString(domain)
+
+	// If any extractors, write the results
+	if len(extractorResults) > 0 {
+		builder.WriteString(" [")
+		for i, result := range extractorResults {
+			builder.WriteString(result)
+			if i != len(extractorResults)-1 {
+				builder.WriteRune(',')
+			}
+		}
+		builder.WriteString("]")
+	}
+	builder.WriteRune('\n')
+
+	return builder.String()
+}

pkg/executor/output_http.go

@@ -0,0 +1,41 @@
+package executor
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/pkg/matchers"
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+// buildOutputHTTP builds an output text for writing results
+func (e *HTTPExecutor) buildOutputHTTP(req *retryablehttp.Request, extractorResults []string, matcher *matchers.Matcher) string {
+	builder := &strings.Builder{}
+
+	builder.WriteRune('[')
+	builder.WriteString(e.template.ID)
+	if len(matcher.Name) > 0 {
+		builder.WriteString(":")
+		builder.WriteString(matcher.Name)
+	}
+	builder.WriteString("] [http] ")
+
+	// Escape the URL by replacing all % with %%
+	URL := req.URL.String()
+	escapedURL := strings.Replace(URL, "%", "%%", -1)
+	builder.WriteString(escapedURL)
+
+	// If any extractors, write the results
+	if len(extractorResults) > 0 {
+		builder.WriteString(" [")
+		for i, result := range extractorResults {
+			builder.WriteString(result)
+			if i != len(extractorResults)-1 {
+				builder.WriteRune(',')
+			}
+		}
+		builder.WriteString("]")
+	}
+	builder.WriteRune('\n')
+
+	return builder.String()
+}

pkg/requests/dns-request.go

@@ -1,7 +1,6 @@
 package requests
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/miekg/dns"
@@ -29,7 +28,6 @@ type DNSRequest struct {
 
 // MakeDNSRequest creates a *dns.Request from a request template
 func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
-
 	domain = dns.Fqdn(domain)
 
 	// Build a request on the specified URL
@@ -40,28 +38,16 @@ func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
 	var q dns.Question
 
 	t := fasttemplate.New(r.Name, "{{", "}}")
-	q.Name = dns.Fqdn(t.ExecuteString(map[string]interface{}{
-		"FQDN": domain,
-	}))
-
-	qclass, err := toQClass(r.Class)
-	if err != nil {
-		return nil, err
-	}
-	q.Qclass = qclass
-
-	qtype, err := toQType(r.Type)
-	if err != nil {
-		return nil, err
-	}
-	q.Qtype = qtype
+	q.Name = dns.Fqdn(t.ExecuteString(map[string]interface{}{"FQDN": domain}))
+	q.Qclass = toQClass(r.Class)
+	q.Qtype = toQType(r.Type)
 
 	req.Question = append(req.Question, q)
 
 	return req, nil
 }
 
-func toQType(ttype string) (rtype uint16, err error) {
+func toQType(ttype string) (rtype uint16) {
 	ttype = strings.TrimSpace(strings.ToUpper(ttype))
 
 	switch ttype {
@@ -82,14 +68,12 @@ func toQType(ttype string) (rtype uint16, err error) {
 	case "AAAA":
 		rtype = dns.TypeAAAA
 	default:
-		rtype = dns.TypeNone
-		err = fmt.Errorf("incorrect type")
+		rtype = dns.TypeA
 	}
-
 	return
 }
 
-func toQClass(tclass string) (rclass uint16, err error) {
+func toQClass(tclass string) (rclass uint16) {
 	tclass = strings.TrimSpace(strings.ToUpper(tclass))
 
 	switch tclass {
@@ -106,8 +90,8 @@ func toQClass(tclass string) (rclass uint16, err error) {
 	case "ANY":
 		rclass = dns.ClassANY
 	default:
-		err = fmt.Errorf("incorrect class")
+		// Use INET by default.
+		rclass = dns.ClassINET
 	}
-
 	return
 }

pkg/requests/http-request.go

@@ -25,6 +25,9 @@ type HTTPRequest struct {
 	// Matchers contains the detection mechanism for the request to identify
 	// whether the request was successful
 	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
+	// MatchersCondition is the condition of the matchers
+	// whether to use AND or OR. Default is OR.
+	MatchersCondition string `yaml:"matchers-condition,omitempty"`
 	// Extractors contains the extraction mechanism for the request to identify
 	// and extract parts of the response.
 	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`

pkg/templates/templates.go

@@ -10,9 +10,9 @@ type Template struct {
 	ID string `yaml:"id"`
 	// Info contains information about the template
 	Info Info `yaml:"info"`
-	// Request contains the http request to make in the template
+	// RequestHTTP contains the http request to make in the template
 	RequestsHTTP []*requests.HTTPRequest `yaml:"requests"`
-	// Request contains the dns request to make in the template
+	// RequestDNS contains the dns request to make in the template
 	RequestsDNS []*requests.DNSRequest `yaml:"dns"`
 }