Added payload validation + misc

v2/pkg/executer/executer_http.go

@@ -1,577 +0,0 @@
-package executer
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/corpix/uarand"
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/rawhttp"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"github.com/remeh/sizedwaitgroup"
-	"go.uber.org/ratelimit"
-)
-
-const (
-	two                   = 2
-	ten                   = 10
-	defaultMaxWorkers     = 150
-	defaultMaxHistorydata = 150
-)
-
-// HTTPExecuter is client for performing HTTP requests
-// for a template.
-type HTTPExecuter struct {
-	pf               *projetctfile.ProjectFile
-	customHeaders    requests.CustomHeaders
-	colorizer        colorizer.NucleiColorizer
-	httpClient       *retryablehttp.Client
-	rawHTTPClient    *rawhttp.Client
-	template         *templates.Template
-	bulkHTTPRequest  *requests.BulkHTTPRequest
-	writer           *bufwriter.Writer
-	CookieJar        *cookiejar.Jar
-	traceLog         tracelog.Log
-	decolorizer      *regexp.Regexp
-	randomAgent      bool
-	vhost            bool
-	coloredOutput    bool
-	debug            bool
-	Results          bool
-	jsonOutput       bool
-	jsonRequest      bool
-	noMeta           bool
-	stopAtFirstMatch bool
-	ratelimiter      ratelimit.Limiter
-}
-
-// HTTPOptions contains configuration options for the HTTP executer.
-type HTTPOptions struct {
-	Template        *templates.Template
-	BulkHTTPRequest *requests.BulkHTTPRequest
-	CookieJar       *cookiejar.Jar
-	PF              *projetctfile.ProjectFile
-}
-
-// NewHTTPExecuter creates a new HTTP executer from a template
-// and a HTTP request query.
-func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
-	var (
-		proxyURL *url.URL
-		err      error
-	)
-
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the HTTP Client
-	client := makeHTTPClient(proxyURL, options)
-	// nolint:bodyclose // false positive there is no body to close yet
-
-	if options.CookieJar != nil {
-		client.HTTPClient.Jar = options.CookieJar
-	} else if options.CookieReuse {
-		jar, err := cookiejar.New(nil)
-		if err != nil {
-			return nil, err
-		}
-		client.HTTPClient.Jar = jar
-	}
-
-	executer := &HTTPExecuter{
-		debug:            options.Debug,
-		jsonOutput:       options.JSON,
-		jsonRequest:      options.JSONRequests,
-		noMeta:           options.NoMeta,
-		httpClient:       client,
-		rawHTTPClient:    rawClient,
-		traceLog:         options.TraceLog,
-		template:         options.Template,
-		bulkHTTPRequest:  options.BulkHTTPRequest,
-		writer:           options.Writer,
-		randomAgent:      options.RandomAgent,
-		customHeaders:    options.CustomHeaders,
-		CookieJar:        options.CookieJar,
-		coloredOutput:    options.ColoredOutput,
-		colorizer:        *options.Colorizer,
-		decolorizer:      options.Decolorizer,
-		stopAtFirstMatch: options.StopAtFirstMatch,
-		pf:               options.PF,
-		vhost:            options.Vhost,
-		ratelimiter:      options.RateLimiter,
-	}
-	return executer, nil
-}
-
-func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// Workers that keeps enqueuing new requests
-	maxWorkers := e.bulkHTTPRequest.RaceNumberRequests
-	swg := sizedwaitgroup.New(maxWorkers)
-	for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
-		swg.Add()
-		// base request
-		result.Lock()
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
-		result.Unlock()
-		// ignore the error due to the base request having null paylods
-		if err == requests.ErrNoPayload {
-			// pass through
-		} else if err != nil {
-			result.Error = err
-		}
-		go func(httpRequest *requests.HTTPRequest) {
-			defer swg.Done()
-
-			// If the request was built correctly then execute it
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not handle http request")
-			}
-		}(request)
-	}
-
-	swg.Wait()
-
-	return result
-}
-
-func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	remaining := e.bulkHTTPRequest.GetRequestCount()
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// Workers that keeps enqueuing new requests
-	maxWorkers := e.bulkHTTPRequest.Threads
-	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) {
-		result.Lock()
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
-		result.Unlock()
-		// ignore the error due to the base request having null paylods
-		if err == requests.ErrNoPayload {
-			// pass through
-		} else if err != nil {
-			result.Error = err
-			p.Drop(remaining)
-		} else {
-			swg.Add()
-			go func(httpRequest *requests.HTTPRequest) {
-				defer swg.Done()
-
-				e.ratelimiter.Take()
-
-				// If the request was built correctly then execute it
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
-				if err != nil {
-					e.traceLog.Request(e.template.ID, reqURL, "http", err)
-					result.Error = errors.Wrap(err, "could not handle http request")
-					p.Drop(remaining)
-				} else {
-					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-				}
-			}(request)
-		}
-		p.Update()
-		e.bulkHTTPRequest.Increment(reqURL)
-	}
-	swg.Wait()
-
-	return result
-}
-
-func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// need to extract the target from the url
-	URL, err := url.Parse(reqURL)
-	if err != nil {
-		return result
-	}
-
-	// defaultMaxWorkers should be a sufficient value to keep queues always full
-	maxWorkers := defaultMaxWorkers
-	// in case the queue is bigger increase the workers
-	if pipeOptions.MaxPendingRequests > maxWorkers {
-		maxWorkers = pipeOptions.MaxPendingRequests
-	}
-	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) {
-		result.Lock()
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
-		result.Unlock()
-		// ignore the error due to the base request having null paylods
-		if err == requests.ErrNoPayload {
-			// pass through
-		} else if err != nil {
-			result.Error = err
-		} else {
-			swg.Add()
-			go func(httpRequest *requests.HTTPRequest) {
-				defer swg.Done()
-
-				// HTTP pipelining ignores rate limit
-				// If the request was built correctly then execute it
-				request.Pipeline = true
-				request.PipelineClient = pipeclient
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
-				if err != nil {
-					e.traceLog.Request(e.template.ID, reqURL, "http", err)
-					result.Error = errors.Wrap(err, "could not handle http request")
-				} else {
-					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-				}
-				request.PipelineClient = nil
-			}(request)
-		}
-
-		e.bulkHTTPRequest.Increment(reqURL)
-	}
-	swg.Wait()
-	return result
-}
-
-// ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result {
-	var customHost string
-	if e.vhost {
-		parts := strings.Split(reqURL, ",")
-		reqURL = parts[0]
-		customHost = parts[1]
-	}
-
-	// verify if pipeline was requested
-	if e.bulkHTTPRequest.Pipeline {
-		return e.ExecuteTurboHTTP(reqURL)
-	}
-
-	// verify if a basic race condition was requested
-	if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 {
-		return e.ExecuteRaceRequest(reqURL)
-	}
-
-	// verify if parallel elaboration was requested
-	if e.bulkHTTPRequest.Threads > 0 {
-		return e.ExecuteParallelHTTP(p, reqURL)
-	}
-
-	var requestNumber int
-
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-		historyData: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	remaining := e.bulkHTTPRequest.GetRequestCount()
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	for e.bulkHTTPRequest.Next(reqURL) {
-		requestNumber++
-		result.Lock()
-		httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
-		result.Unlock()
-		// ignore the error due to the base request having null paylods
-		if err == requests.ErrNoPayload {
-			// pass through
-		} else if err != nil {
-			result.Error = err
-			p.Drop(remaining)
-		} else {
-			if e.vhost {
-				if httpRequest.Request != nil {
-					httpRequest.Request.Host = customHost
-				}
-				if httpRequest.RawRequest != nil && httpRequest.RawRequest.Headers != nil {
-					httpRequest.RawRequest.Headers["Host"] = customHost
-				}
-			}
-
-			e.ratelimiter.Take()
-			// If the request was built correctly then execute it
-			format := "%s_" + strconv.Itoa(requestNumber)
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format)
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not handle http request")
-				p.Drop(remaining)
-				e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			} else {
-				e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-			}
-		}
-		p.Update()
-
-		// Check if has to stop processing at first valid result
-		if e.stopAtFirstMatch && result.GotResults {
-			p.Drop(remaining)
-			break
-		}
-
-		// move always forward with requests
-		e.bulkHTTPRequest.Increment(reqURL)
-		remaining--
-	}
-	gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL)
-	return result
-}
-
-func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error {
-	// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
-	if e.randomAgent {
-		// nolint:errcheck // ignoring error
-		e.customHeaders.Set("User-Agent: " + uarand.GetRandom())
-	}
-
-	e.setCustomHeaders(request)
-
-	var (
-		resp          *http.Response
-		err           error
-		dumpedRequest []byte
-		fromcache     bool
-	)
-
-	if e.debug || e.pf != nil {
-		dumpedRequest, err = requests.Dump(request, reqURL)
-		if err != nil {
-			return err
-		}
-	}
-
-	if e.debug {
-		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-	}
-
-	timeStart := time.Now()
-
-	if request.Pipeline {
-		resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			return err
-		}
-		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-	} else if request.Unsafe {
-		// rawhttp
-		// burp uses "\r\n" as new line character
-		request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n")
-		options := e.rawHTTPClient.Options
-		options.AutomaticContentLength = request.AutomaticContentLengthHeader
-		options.AutomaticHostHeader = request.AutomaticHostHeader
-		options.FollowRedirects = request.FollowRedirects
-		resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options)
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			return err
-		}
-		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-	} else {
-		// if nuclei-project is available check if the request was already sent previously
-		if e.pf != nil {
-			// if unavailable fail silently
-			fromcache = true
-			// nolint:bodyclose // false positive the response is generated at runtime
-			resp, err = e.pf.Get(dumpedRequest)
-			if err != nil {
-				fromcache = false
-			}
-		}
-
-		// retryablehttp
-		if resp == nil {
-			resp, err = e.httpClient.Do(request.Request)
-			if err != nil {
-				if resp != nil {
-					resp.Body.Close()
-				}
-				e.traceLog.Request(e.template.ID, reqURL, "http", err)
-				return err
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-		}
-	}
-
-	duration := time.Since(timeStart)
-
-	// Dump response - Step 1 - Decompression not yet handled
-	var dumpedResponse []byte
-	if e.debug {
-		var dumpErr error
-		dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
-		if dumpErr != nil {
-			return errors.Wrap(dumpErr, "could not dump http response")
-		}
-	}
-
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		_, copyErr := io.Copy(ioutil.Discard, resp.Body)
-		if copyErr != nil {
-			resp.Body.Close()
-			return copyErr
-		}
-
-		resp.Body.Close()
-
-		return errors.Wrap(err, "could not read http body")
-	}
-
-	resp.Body.Close()
-
-	// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-	// so in case we have to manually do it
-	dataOrig := data
-	data, err = requests.HandleDecompression(request, data)
-	if err != nil {
-		return errors.Wrap(err, "could not decompress http body")
-	}
-
-	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
-	if e.debug {
-		dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
-		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-	}
-
-	// if nuclei-project is enabled store the response if not previously done
-	if e.pf != nil && !fromcache {
-		err := e.pf.Set(dumpedRequest, resp, data)
-		if err != nil {
-			return errors.Wrap(err, "could not store in project file")
-		}
-	}
-
-	// Convert response body from []byte to string with zero copy
-	body := unsafeToString(data)
-
-	headers := headersToString(resp.Header)
-
-	var matchData map[string]interface{}
-	if payloads != nil {
-		matchData = generators.MergeMaps(result.historyData, payloads)
-	}
-
-	// store for internal purposes the DSL matcher data
-	// hardcode stopping storing data after defaultMaxHistorydata items
-	if len(result.historyData) < defaultMaxHistorydata {
-		result.Lock()
-		// update history data with current reqURL and hostname
-		result.historyData["reqURL"] = reqURL
-		if parsed, err := url.Parse(reqURL); err == nil {
-			result.historyData["Hostname"] = parsed.Host
-		}
-		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
-		if payloads == nil {
-			// merge them to history data
-			result.historyData = generators.MergeMaps(result.historyData, payloads)
-		}
-		result.historyData = generators.MergeMaps(result.historyData, dynamicvalues)
-
-		// complement match data with new one if necessary
-		matchData = generators.MergeMaps(matchData, result.historyData)
-		result.Unlock()
-	}
-
-	return nil
-}
-
-// Close closes the http executer for a template.
-func (e *HTTPExecuter) Close() {}
-
-func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
-	for _, customHeader := range e.customHeaders {
-		// This should be pre-computed somewhere and done only once
-		tokens := strings.SplitN(customHeader, ":", two)
-		// if it's an invalid header skip it
-		if len(tokens) < two {
-			continue
-		}
-
-		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
-		if r.RawRequest != nil {
-			// rawhttp
-			r.RawRequest.Headers[headerName] = headerValue
-		} else {
-			// retryablehttp
-			headerName = strings.TrimSpace(headerName)
-			headerValue = strings.TrimSpace(headerValue)
-			r.Request.Header[headerName] = []string{headerValue}
-		}
-	}
-}

v2/pkg/executer/output_http.go

@@ -1,147 +0,0 @@
-package executer
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httputil"
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-)
-
-// writeOutputHTTP writes http output to streams
-func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) {
-	var URL string
-	if req.RawRequest != nil {
-		URL = req.RawRequest.FullURL
-	}
-	if req.Request != nil {
-		URL = req.Request.URL.String()
-	}
-
-	if e.jsonOutput {
-		output := make(jsonOutput)
-
-		output["matched"] = URL
-		if !e.noMeta {
-			output["template"] = e.template.ID
-			output["type"] = "http"
-			output["host"] = reqURL
-			if len(meta) > 0 {
-				output["meta"] = meta
-			}
-			for k, v := range e.template.Info {
-				output[k] = v
-			}
-			if matcher != nil && len(matcher.Name) > 0 {
-				output["matcher_name"] = matcher.Name
-			}
-			if len(extractorResults) > 0 {
-				output["extracted_results"] = extractorResults
-			}
-
-			// TODO: URL should be an argument
-			if e.jsonRequest {
-				dumpedRequest, err := requests.Dump(req, URL)
-				if err != nil {
-					gologger.Warningf("could not dump request: %s\n", err)
-				} else {
-					output["request"] = string(dumpedRequest)
-				}
-
-				dumpedResponse, err := httputil.DumpResponse(resp, false)
-				if err != nil {
-					gologger.Warningf("could not dump response: %s\n", err)
-				} else {
-					output["response"] = string(dumpedResponse) + body
-				}
-			}
-		}
-
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			if err := e.writer.Write(data); err != nil {
-				gologger.Errorf("Could not write output data: %s\n", err)
-				return
-			}
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	colorizer := e.colorizer
-
-	if !e.noMeta {
-		builder.WriteRune('[')
-		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
-
-		if matcher != nil && len(matcher.Name) > 0 {
-			builder.WriteString(":")
-			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-		}
-
-		builder.WriteString("] [")
-		builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
-		builder.WriteString("] ")
-
-		if e.template.Info["severity"] != "" {
-			builder.WriteString("[")
-			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
-			builder.WriteString("] ")
-		}
-	}
-	builder.WriteString(URL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		for i, result := range extractorResults {
-			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
-
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-
-		builder.WriteString("]")
-	}
-
-	// write meta if any
-	if len(req.Meta) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		var metas []string
-		for name, value := range req.Meta {
-			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String())
-		}
-
-		builder.WriteString(strings.Join(metas, ","))
-		builder.WriteString("]")
-	}
-
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		if e.coloredOutput {
-			message = e.decolorizer.ReplaceAllString(message, "")
-		}
-
-		if err := e.writer.WriteString(message); err != nil {
-			gologger.Errorf("Could not write output data: %s\n", err)
-			return
-		}
-	}
-}

v2/pkg/executer/utils.go

@@ -1,59 +0,0 @@
-package executer
-
-import (
-	"net/http"
-	"net/url"
-	"strings"
-	"unsafe"
-)
-
-type jsonOutput map[string]interface{}
-
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-
-			if i != len(values)-1 {
-				builder.WriteRune('\n')
-				builder.WriteString(header)
-				builder.WriteString(": ")
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}
-
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
-		return false
-	}
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(theURL string) string {
-	u, err := url.Parse(theURL)
-	if err != nil {
-		return ""
-	}
-	return u.Hostname()
-}

v2/pkg/protocols/common/generators/generators.go

@@ -32,11 +32,18 @@ var StringToType = map[string]Type{
 }
 
 // New creates a new generator structure for payload generation
-func New(payloads map[string]interface{}, Type Type) (*Generator, error) {
+func New(payloads map[string]interface{}, Type Type, templatePath string) (*Generator, error) {
+	generator := &Generator{}
+	if err := generator.validate(payloads, templatePath); err != nil {
+		return nil, err
+	}
+
 	compiled, err := loadPayloads(payloads)
 	if err != nil {
 		return nil, err
 	}
+	generator.Type = Type
+	generator.payloads = compiled
 
 	// Validate the payload types
 	if Type == Sniper && len(compiled) > 1 {
@@ -51,7 +58,6 @@ func New(payloads map[string]interface{}, Type Type) (*Generator, error) {
 			totalLength = len(v)
 		}
 	}
-	generator := &Generator{Type: Type, payloads: compiled}
 	return generator, nil
 }
 

v2/pkg/protocols/common/generators/generators_test.go

@@ -9,7 +9,7 @@ import (
 func TestSniperGenerator(t *testing.T) {
 	usernames := []string{"admin", "password", "login", "test"}
 
-	generator, err := New(map[string]interface{}{"username": usernames}, Sniper)
+	generator, err := New(map[string]interface{}{"username": usernames}, Sniper, "")
 	require.Nil(t, err, "could not create generator")
 
 	iterator := generator.NewIterator()
@@ -29,7 +29,7 @@ func TestPitchforkGenerator(t *testing.T) {
 	usernames := []string{"admin", "token"}
 	passwords := []string{"admin", "password"}
 
-	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork)
+	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "")
 	require.Nil(t, err, "could not create generator")
 
 	iterator := generator.NewIterator()
@@ -50,7 +50,7 @@ func TestClusterbombGenerator(t *testing.T) {
 	usernames := []string{"admin"}
 	passwords := []string{"admin", "password", "token"}
 
-	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb)
+	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb, "")
 	require.Nil(t, err, "could not create generator")
 
 	iterator := generator.NewIterator()

v2/pkg/protocols/common/generators/validate.go

@@ -0,0 +1,61 @@
+package generators
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/spf13/cast"
+)
+
+// validate validates the payloads if any.
+func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error {
+	for name, payload := range payloads {
+		switch pt := payload.(type) {
+		case string:
+			// check if it's a multiline string list
+			if len(strings.Split(pt, "\n")) != 1 {
+				return errors.New("invalid number of lines in payload")
+			}
+
+			// check if it's a worldlist file and try to load it
+			if fileExists(pt) {
+				continue
+			}
+
+			changed := false
+			pathTokens := strings.Split(templatePath, "/")
+
+			for i := range pathTokens {
+				tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
+				if fileExists(tpath) {
+					payloads[name] = tpath
+					changed = true
+					break
+				}
+			}
+			if !changed {
+				return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name)
+			}
+		case interface{}:
+			loadedPayloads := cast.ToStringSlice(pt)
+			if len(loadedPayloads) == 0 {
+				return fmt.Errorf("the payload %s does not contain enough elements", name)
+			}
+		default:
+			return fmt.Errorf("the payload %s has invalid type", name)
+		}
+	}
+	return nil
+}
+
+// fileExists checks if a file exists and is not a directory
+func fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}

v2/pkg/protocols/dns/executer_test.go

@@ -1,73 +0,0 @@
-package dns
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRequest(t *testing.T) {
-	err := dnsclientpool.Init(&types.Options{})
-	require.Nil(t, err, "could not initialize dns client pool")
-
-	writer, err := output.NewStandardWriter(true, false, false, "", "")
-	require.Nil(t, err, "could not create standard output writer")
-
-	progress, err := progress.NewProgress(false, false, 0)
-	require.Nil(t, err, "could not create standard progress writer")
-
-	protocolOpts := &protocols.ExecuterOptions{
-		TemplateID:   "testing-dns",
-		TemplateInfo: map[string]string{"author": "test"},
-		Output:       writer,
-		Options:      &types.Options{},
-		Progress:     progress,
-	}
-	req := &Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{
-		Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}},
-	}}
-	err = req.Compile(protocolOpts)
-	require.Nil(t, err, "could not compile request")
-
-	output, err := req.ExecuteWithResults("docs.hackerone.com.", nil)
-	require.Nil(t, err, "could not execute request")
-
-	for _, result := range output {
-		fmt.Printf("%+v\n", result)
-	}
-}
-
-func TestExecuter(t *testing.T) {
-	err := dnsclientpool.Init(&types.Options{})
-	require.Nil(t, err, "could not initialize dns client pool")
-
-	writer, err := output.NewStandardWriter(true, false, false, "", "")
-	require.Nil(t, err, "could not create standard output writer")
-
-	progress, err := progress.NewProgress(false, false, 0)
-	require.Nil(t, err, "could not create standard progress writer")
-
-	protocolOpts := &protocols.ExecuterOptions{
-		TemplateID:   "testing-dns",
-		TemplateInfo: map[string]string{"author": "test"},
-		Output:       writer,
-		Options:      &types.Options{},
-		Progress:     progress,
-	}
-	executer := NewExecuter([]*Request{&Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{
-		Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}},
-	}}}, protocolOpts)
-	err = executer.Compile()
-	require.Nil(t, err, "could not compile request")
-
-	_, err = executer.Execute("docs.hackerone.com")
-	require.Nil(t, err, "could not execute request")
-}

v2/pkg/protocols/http/build_request_test.go

@@ -31,7 +31,7 @@ func TestRequestGeneratorClusterSingle(t *testing.T) {
 		attackType: generators.ClusterBomb,
 		Raw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`},
 	}
-	req.generator, err = generators.New(req.Payloads, req.attackType)
+	req.generator, err = generators.New(req.Payloads, req.attackType, "")
 	require.Nil(t, err, "could not create generator")
 
 	generator := req.newGenerator()
@@ -54,7 +54,7 @@ func TestRequestGeneratorClusterMultipleRaw(t *testing.T) {
 		attackType: generators.ClusterBomb,
 		Raw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
 	}
-	req.generator, err = generators.New(req.Payloads, req.attackType)
+	req.generator, err = generators.New(req.Payloads, req.attackType, "")
 	require.Nil(t, err, "could not create generator")
 
 	generator := req.newGenerator()

v2/pkg/protocols/http/executer_test.go

@@ -1,47 +0,0 @@
-package http
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/ratelimit"
-)
-
-func TestRequest(t *testing.T) {
-	err := httpclientpool.Init(&types.Options{})
-	require.Nil(t, err, "could not initialize dns client pool")
-
-	writer, err := output.NewStandardWriter(true, false, false, "", "")
-	require.Nil(t, err, "could not create standard output writer")
-
-	progress, err := progress.NewProgress(false, false, 0)
-	require.Nil(t, err, "could not create standard progress writer")
-
-	protocolOpts := &protocols.ExecuterOptions{
-		TemplateID:   "testing-dns",
-		TemplateInfo: map[string]string{"author": "test"},
-		Output:       writer,
-		Options:      &types.Options{},
-		Progress:     progress,
-		RateLimiter:  ratelimit.New(100),
-	}
-	executer := NewExecuter([]*Request{&Request{Path: []string{"{{BaseURL}}"}, Method: "GET", Operators: &operators.Operators{
-		Matchers: []*matchers.Matcher{{Type: "dsl", DSL: []string{"!contains(tolower(all_headers), 'x-frame-options')"}, Part: "body"}},
-	}}}, protocolOpts)
-	err = executer.Compile()
-	require.Nil(t, err, "could not compile request")
-
-	_, err = executer.Execute("https://example.com")
-	require.Nil(t, err, "could not execute request")
-
-	//	for _, result := range output {
-	//		fmt.Printf("%+v\n", result)
-	//	}
-}

v2/pkg/protocols/http/http.go

@@ -91,8 +91,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	}
 
 	if len(r.Payloads) > 0 {
-		r.attackType = generators.StringToType[r.AttackType]
+		attackType := r.AttackType
-		r.generator, err = generators.New(r.Payloads, r.attackType)
+		if attackType == "" {
+			attackType = "sniper"
+		}
+		r.attackType = generators.StringToType[attackType]
+
+		r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath)
 		if err != nil {
 			return errors.Wrap(err, "could not parse payloads")
 		}

v2/pkg/protocols/protocols.go

@@ -26,6 +26,8 @@ type Executer interface {
 type ExecuterOptions struct {
 	// TemplateID is the ID of the template for the request
 	TemplateID string
+	// TemplatePath is the path of the template for the request
+	TemplatePath string
 	// TemplateInfo contains information block of the template request
 	TemplateInfo map[string]string
 	// Output is a writer interface for writing output events from executer.

v2/pkg/templates/compile.go

@@ -3,11 +3,7 @@ package templates
 import (
 	"fmt"
 	"os"
-	"path"
-	"strings"
 
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
 	"gopkg.in/yaml.v2"
 )
 
@@ -35,99 +31,9 @@ func Parse(file string) (*Template, error) {
 
 	// Compile the matchers and the extractors for http requests
 	for _, request := range template.BulkRequestsHTTP {
-		// Get the condition between the matchers
-		condition, ok := matchers.ConditionTypes[request.MatchersCondition]
-		if !ok {
-			request.SetMatchersCondition(matchers.ORCondition)
-		} else {
-			request.SetMatchersCondition(condition)
-		}
-
-		// Set the attack type - used only in raw requests
-		attack, ok := generators.AttackTypes[request.AttackType]
-		if !ok {
-			request.SetAttackType(generators.Sniper)
-		} else {
-			request.SetAttackType(attack)
-		}
-
-		// Validate the payloads if any
-		for name, payload := range request.Payloads {
-			switch pt := payload.(type) {
-			case string:
-				// check if it's a multiline string list
-				if len(strings.Split(pt, "\n")) <= 1 {
-					// check if it's a worldlist file
-					if !generators.FileExists(pt) {
-						// attempt to load the file by taking the full path, tokezining it and searching the template in such paths
-						changed := false
-						pathTokens := strings.Split(template.path, "/")
-
-						for i := range pathTokens {
-							tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
-							if generators.FileExists(tpath) {
-								request.Payloads[name] = tpath
-								changed = true
-
-								break
-							}
-						}
-
-						if !changed {
-							return nil, fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name)
-						}
-					}
-				}
-			case []string, []interface{}:
-				if len(payload.([]interface{})) == 0 {
-					return nil, fmt.Errorf("the payload %s does not contain enough elements", name)
-				}
-			default:
-				return nil, fmt.Errorf("the payload %s has invalid type", name)
-			}
-		}
-
-		for _, matcher := range request.Matchers {
-			matchErr := matcher.CompileMatchers()
-			if matchErr != nil {
-				return nil, matchErr
-			}
-		}
-
-		for _, extractor := range request.Extractors {
-			extractErr := extractor.CompileExtractors()
-			if extractErr != nil {
-				return nil, extractErr
-			}
-		}
 
 		request.InitGenerator()
 	}
 
-	// Compile the matchers and the extractors for dns requests
-	for _, request := range template.RequestsDNS {
-		// Get the condition between the matchers
-		condition, ok := matchers.ConditionTypes[request.MatchersCondition]
-		if !ok {
-			request.SetMatchersCondition(matchers.ORCondition)
-		} else {
-			request.SetMatchersCondition(condition)
-		}
-
-		for _, matcher := range request.Matchers {
-			err = matcher.CompileMatchers()
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		for _, extractor := range request.Extractors {
-			err := extractor.CompileExtractors()
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
 	return template, nil
 }