code refactor

v2/internal/runner/runner.go

@@ -112,7 +112,7 @@ func (r *Runner) RunEnumeration() {
 			var results bool
 			template := t.(*templates.Template)
 			// process http requests
-			for _, request := range template.RequestsHTTP {
+			for _, request := range template.BulkRequestsHTTP {
 				results = r.processTemplateRequest(template, request)
 			}
 			// process dns requests
@@ -176,7 +176,7 @@ func (r *Runner) RunEnumeration() {
 					results = dnsResults
 				}
 			}
-			for _, request := range template.RequestsHTTP {
+			for _, request := range template.BulkRequestsHTTP {
 				httpResults := r.processTemplateRequest(template, request)
 				if httpResults {
 					results = httpResults
@@ -248,19 +248,19 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 			Writer:     writer,
 			JSON:       r.options.JSON,
 		})
-	case *requests.HTTPRequest:
+	case *requests.BulkHTTPRequest:
 		httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
-			Debug:         r.options.Debug,
+			Debug:           r.options.Debug,
-			Template:      template,
+			Template:        template,
-			HTTPRequest:   value,
+			BulkHttpRequest: value,
-			Writer:        writer,
+			Writer:          writer,
-			Timeout:       r.options.Timeout,
+			Timeout:         r.options.Timeout,
-			Retries:       r.options.Retries,
+			Retries:         r.options.Retries,
-			ProxyURL:      r.options.ProxyURL,
+			ProxyURL:        r.options.ProxyURL,
-			ProxySocksURL: r.options.ProxySocksURL,
+			ProxySocksURL:   r.options.ProxySocksURL,
-			CustomHeaders: r.options.CustomHeaders,
+			CustomHeaders:   r.options.CustomHeaders,
-			JSON:          r.options.JSON,
+			JSON:            r.options.JSON,
-			CookieReuse:   value.CookieReuse,
+			CookieReuse:     value.CookieReuse,
 		})
 	}
 	if err != nil {
@@ -302,11 +302,11 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 	// See if we got any results from the executers
 	var results bool
 	if httpExecuter != nil {
-		results = httpExecuter.GotResults()
+		results = httpExecuter.Results
 	}
 	if dnsExecuter != nil {
 		if !results {
-			results = dnsExecuter.GotResults()
+			results = dnsExecuter.Results
 		}
 	}
 	return results
@@ -376,7 +376,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 				return err
 			}
 			template := &workflows.Template{}
-			if len(t.RequestsHTTP) > 0 {
+			if len(t.BulkRequestsHTTP) > 0 {
 				template.HTTPOptions = &executer.HTTPOptions{
 					Debug:         r.options.Debug,
 					Writer:        writer,
@@ -427,7 +427,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 					return err
 				}
 				template := &workflows.Template{}
-				if len(t.RequestsHTTP) > 0 {
+				if len(t.BulkRequestsHTTP) > 0 {
 					template.HTTPOptions = &executer.HTTPOptions{
 						Debug:         r.options.Debug,
 						Writer:        writer,

v2/pkg/executer/executer_dns.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"sync"
-	"sync/atomic"
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
@@ -20,7 +19,7 @@ import (
 type DNSExecuter struct {
 	debug       bool
 	jsonOutput  bool
-	results     uint32
+	Results     bool
 	dnsClient   *retryabledns.Client
 	template    *templates.Template
 	dnsRequest  *requests.DNSRequest
@@ -53,7 +52,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
 	executer := &DNSExecuter{
 		debug:       options.Debug,
 		jsonOutput:  options.JSON,
-		results:     0,
 		dnsClient:   dnsClient,
 		template:    options.Template,
 		dnsRequest:  options.DNSRequest,
@@ -63,14 +61,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
 	return executer
 }
 
-// GotResults returns true if there were any results for the executer
-func (e *DNSExecuter) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
 // ExecuteDNS executes the DNS request on a URL
 func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 	// Parse the URL and return domain if URL.
@@ -120,7 +110,7 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 			// write the first output then move to next matcher.
 			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
 				e.writeOutputDNS(domain, matcher, nil)
-				atomic.CompareAndSwapUint32(&e.results, 0, 1)
+				e.Results = true
 			}
 		}
 	}
@@ -138,7 +128,7 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 	// AND or if we have extractors for the mechanism too.
 	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
 		e.writeOutputDNS(domain, nil, extractorResults)
-		atomic.CompareAndSwapUint32(&e.results, 0, 1)
+		e.Results = true
 	}
 
 	return
@@ -147,6 +137,6 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 // Close closes the dns executer for a template.
 func (e *DNSExecuter) Close() {
 	e.outputMutex.Lock()
+	defer e.outputMutex.Unlock()
 	e.writer.Flush()
-	e.outputMutex.Unlock()
 }

v2/pkg/executer/executer_http.go

@@ -13,7 +13,6 @@ import (
 	"os"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
@@ -28,32 +27,32 @@ import (
 // HTTPExecuter is client for performing HTTP requests
 // for a template.
 type HTTPExecuter struct {
-	debug         bool
+	debug           bool
-	results       uint32
+	Results         bool
-	jsonOutput    bool
+	jsonOutput      bool
-	httpClient    *retryablehttp.Client
+	httpClient      *retryablehttp.Client
-	template      *templates.Template
+	template        *templates.Template
-	httpRequest   *requests.HTTPRequest
+	bulkHttpRequest *requests.BulkHTTPRequest
-	writer        *bufio.Writer
+	writer          *bufio.Writer
-	outputMutex   *sync.Mutex
+	outputMutex     *sync.Mutex
-	customHeaders requests.CustomHeaders
+	customHeaders   requests.CustomHeaders
-	CookieJar     *cookiejar.Jar
+	CookieJar       *cookiejar.Jar
 }
 
 // HTTPOptions contains configuration options for the HTTP executer.
 type HTTPOptions struct {
-	Template      *templates.Template
+	Template        *templates.Template
-	HTTPRequest   *requests.HTTPRequest
+	BulkHttpRequest *requests.BulkHTTPRequest
-	Writer        *bufio.Writer
+	Writer          *bufio.Writer
-	Timeout       int
+	Timeout         int
-	Retries       int
+	Retries         int
-	ProxyURL      string
+	ProxyURL        string
-	ProxySocksURL string
+	ProxySocksURL   string
-	Debug         bool
+	Debug           bool
-	JSON          bool
+	JSON            bool
-	CustomHeaders requests.CustomHeaders
+	CustomHeaders   requests.CustomHeaders
-	CookieReuse   bool
+	CookieReuse     bool
-	CookieJar     *cookiejar.Jar
+	CookieJar       *cookiejar.Jar
 }
 
 // NewHTTPExecuter creates a new HTTP executer from a template
@@ -83,143 +82,36 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 	}
 
 	executer := &HTTPExecuter{
-		debug:         options.Debug,
+		debug:           options.Debug,
-		jsonOutput:    options.JSON,
+		jsonOutput:      options.JSON,
-		results:       0,
+		httpClient:      client,
-		httpClient:    client,
+		template:        options.Template,
-		template:      options.Template,
+		bulkHttpRequest: options.BulkHttpRequest,
-		httpRequest:   options.HTTPRequest,
+		outputMutex:     &sync.Mutex{},
-		outputMutex:   &sync.Mutex{},
+		writer:          options.Writer,
-		writer:        options.Writer,
+		customHeaders:   options.CustomHeaders,
-		customHeaders: options.CustomHeaders,
+		CookieJar:       options.CookieJar,
-		CookieJar:     options.CookieJar,
 	}
 	return executer, nil
 }
 
-// GotResults returns true if there were any results for the executer
-func (e *HTTPExecuter) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
 // ExecuteHTTP executes the HTTP request on a URL
 func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
 	result.Matches = make(map[string]interface{})
 	result.Extractions = make(map[string]interface{})
 	dynamicvalues := make(map[string]string)
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
-	if err != nil {
-		result.Error = errors.Wrap(err, "could not make http request")
-		return
-	}
 
-	// Send the request to the target servers
+	// Requests defined via Model
-mainLoop:
+	for e.bulkHttpRequest.Next() {
-	for compiledRequest := range compiledRequest {
+		httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, e.bulkHttpRequest.Current())
-		if compiledRequest.Error != nil {
+		if err != nil {
 			result.Error = errors.Wrap(err, "could not make http request")
 			return
 		}
-		e.setCustomHeaders(compiledRequest)
-		e.setDynamicValues(compiledRequest, dynamicvalues)
-		req := compiledRequest.Request
 
-		if e.debug {
+		e.handleHTTP(URL, httpRequest, dynamicvalues, &result)
-			gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
-			dumpedRequest, err := httputil.DumpRequest(req.Request, true)
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not make http request")
-				return
-			}
-			fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-		}
 
-		resp, err := e.httpClient.Do(req)
+		e.bulkHttpRequest.Increment()
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			gologger.Warningf("Could not do request: %s\n", err)
-			continue
-		}
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
-			dumpedResponse, err := httputil.DumpResponse(resp, true)
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not dump http response")
-				return
-			}
-			fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-		}
-
-		data, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			io.Copy(ioutil.Discard, resp.Body)
-			resp.Body.Close()
-			result.Error = errors.Wrap(err, "could not read http body")
-			return
-		}
-		resp.Body.Close()
-
-		// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-		// so in case we have to manually do it
-		data, err = requests.HandleDecompression(compiledRequest.Request, data)
-		if err != nil {
-			result.Error = errors.Wrap(err, "could not decompress http body")
-			return
-		}
-
-		// Convert response body from []byte to string with zero copy
-		body := unsafeToString(data)
-
-		headers := headersToString(resp.Header)
-		matcherCondition := e.httpRequest.GetMatchersCondition()
-		for _, matcher := range e.httpRequest.Matchers {
-			// Check if the matcher matched
-			if !matcher.Match(resp, body, headers) {
-				// If the condition is AND we haven't matched, try next request.
-				if matcherCondition == matchers.ANDCondition {
-					continue mainLoop
-				}
-			} else {
-				// If the matcher has matched, and its an OR
-				// write the first output then move to next matcher.
-				if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
-					result.Matches[matcher.Name] = nil
-					// probably redundant but ensures we snapshot current payload values when matchers are valid
-					result.Meta = compiledRequest.Meta
-					e.writeOutputHTTP(compiledRequest, matcher, nil)
-					atomic.CompareAndSwapUint32(&e.results, 0, 1)
-				}
-			}
-		}
-
-		// All matchers have successfully completed so now start with the
-		// next task which is extraction of input from matchers.
-		var extractorResults []string
-		for _, extractor := range e.httpRequest.Extractors {
-			for match := range extractor.Extract(resp, body, headers) {
-				if _, ok := dynamicvalues[extractor.Name]; !ok {
-					dynamicvalues[extractor.Name] = match
-				}
-				extractorResults = append(extractorResults, match)
-			}
-			// probably redundant but ensures we snapshot current payload values when extractors are valid
-			result.Meta = compiledRequest.Meta
-			result.Extractions[extractor.Name] = extractorResults
-		}
-
-		// Write a final string of output if matcher type is
-		// AND or if we have extractors for the mechanism too.
-		if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-			e.writeOutputHTTP(compiledRequest, nil, extractorResults)
-			atomic.CompareAndSwapUint32(&e.results, 0, 1)
-		}
 	}
 
 	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
@@ -227,11 +119,107 @@ mainLoop:
 	return
 }
 
+func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dynamicvalues map[string]string, result *Result) error {
+	e.setCustomHeaders(request)
+	e.setDynamicValues(request, dynamicvalues)
+	req := request.Request
+
+	if e.debug {
+		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
+		dumpedRequest, err := httputil.DumpRequest(req.Request, true)
+		if err != nil {
+			return errors.Wrap(err, "could not make http request")
+		}
+		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
+	}
+
+	resp, err := e.httpClient.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		gologger.Warningf("Could not do request: %s\n", err)
+	}
+
+	if e.debug {
+		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
+		dumpedResponse, err := httputil.DumpResponse(resp, true)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
+			return errors.Wrap(err, "could not dump http response")
+		}
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return errors.Wrap(err, "could not read http body")
+	}
+	resp.Body.Close()
+
+	// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
+	// so in case we have to manually do it
+	data, err = requests.HandleDecompression(req, data)
+	if err != nil {
+		return errors.Wrap(err, "could not decompress http body")
+	}
+
+	// Convert response body from []byte to string with zero copy
+	body := unsafeToString(data)
+
+	headers := headersToString(resp.Header)
+	matcherCondition := e.bulkHttpRequest.GetMatchersCondition()
+	for _, matcher := range e.bulkHttpRequest.Matchers {
+		// Check if the matcher matched
+		if !matcher.Match(resp, body, headers) {
+			// If the condition is AND we haven't matched, try next request.
+			if matcherCondition == matchers.ANDCondition {
+				return nil
+			}
+		} else {
+			// If the matcher has matched, and its an OR
+			// write the first output then move to next matcher.
+			if matcherCondition == matchers.ORCondition && len(e.bulkHttpRequest.Extractors) == 0 {
+				result.Matches[matcher.Name] = nil
+				// probably redundant but ensures we snapshot current payload values when matchers are valid
+				result.Meta = request.Meta
+				e.writeOutputHTTP(request, matcher, nil)
+				e.Results = true
+			}
+		}
+	}
+
+	// All matchers have successfully completed so now start with the
+	// next task which is extraction of input from matchers.
+	var extractorResults []string
+	for _, extractor := range e.bulkHttpRequest.Extractors {
+		for match := range extractor.Extract(resp, body, headers) {
+			if _, ok := dynamicvalues[extractor.Name]; !ok {
+				dynamicvalues[extractor.Name] = match
+			}
+			extractorResults = append(extractorResults, match)
+		}
+		// probably redundant but ensures we snapshot current payload values when extractors are valid
+		result.Meta = request.Meta
+		result.Extractions[extractor.Name] = extractorResults
+	}
+
+	// Write a final string of output if matcher type is
+	// AND or if we have extractors for the mechanism too.
+	if len(e.bulkHttpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
+		e.writeOutputHTTP(request, nil, extractorResults)
+		e.Results = true
+	}
+
+	return nil
+}
+
 // Close closes the http executer for a template.
 func (e *HTTPExecuter) Close() {
 	e.outputMutex.Lock()
+	defer e.outputMutex.Unlock()
 	e.writer.Flush()
-	e.outputMutex.Unlock()
 }
 
 // makeHTTPClient creates a http client
@@ -239,8 +227,8 @@ func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Clie
 	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
 	retryablehttpOptions.RetryWaitMax = 10 * time.Second
 	retryablehttpOptions.RetryMax = options.Retries
-	followRedirects := options.HTTPRequest.Redirects
+	followRedirects := options.BulkHttpRequest.Redirects
-	maxRedirects := options.HTTPRequest.MaxRedirects
+	maxRedirects := options.BulkHttpRequest.MaxRedirects
 
 	transport := &http.Transport{
 		MaxIdleConnsPerHost: -1,
@@ -296,7 +284,7 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect
 	}
 }
 
-func (e *HTTPExecuter) setCustomHeaders(r *requests.CompiledHTTP) {
+func (e *HTTPExecuter) setCustomHeaders(r *requests.HttpRequest) {
 	for _, customHeader := range e.customHeaders {
 		// This should be pre-computed somewhere and done only once
 		tokens := strings.Split(customHeader, ":")
@@ -313,7 +301,7 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.CompiledHTTP) {
 }
 
 // for now supports only headers
-func (e *HTTPExecuter) setDynamicValues(r *requests.CompiledHTTP, dynamicValues map[string]string) {
+func (e *HTTPExecuter) setDynamicValues(r *requests.HttpRequest, dynamicValues map[string]string) {
 	for dk, dv := range dynamicValues {
 		// replace within header values
 		for k, v := range r.Request.Header {

v2/pkg/executer/output_http.go

@@ -10,7 +10,7 @@ import (
 )
 
 // writeOutputHTTP writes http output to streams
-func (e *HTTPExecuter) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) {
+func (e *HTTPExecuter) writeOutputHTTP(req *requests.HttpRequest, matcher *matchers.Matcher, extractorResults []string) {
 	URL := req.Request.URL.String()
 
 	if e.jsonOutput {

v2/pkg/requests/bulk-http-request.go

@@ -16,8 +16,8 @@ import (
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 )
 
-// HTTPRequest contains a request to be made from a template
+// BulkHTTPRequest contains a request to be made from a template
-type HTTPRequest struct {
+type BulkHTTPRequest struct {
 	Name string `yaml:"Name,omitempty"`
 	// AttackType is the attack type
 	// Sniper, PitchFork and ClusterBomb. Default is Sniper
@@ -52,31 +52,37 @@ type HTTPRequest struct {
 	// MaxRedirects is the maximum number of redirects that should be followed.
 	MaxRedirects int `yaml:"max-redirects,omitempty"`
 	// Raw contains raw requests
-	Raw []string `yaml:"raw,omitempty"`
+	Raw                   []string `yaml:"raw,omitempty"`
+	positionPath          int
+	positionRaw           int
+	generator             func(payloads map[string][]string) (out chan map[string]interface{})
+	currentPayloads       map[string]interface{}
+	basePayloads          map[string][]string
+	generatorChan         chan map[string]interface{}
+	currentGeneratorValue map[string]interface{}
 }
 
 // GetMatchersCondition returns the condition for the matcher
-func (r *HTTPRequest) GetMatchersCondition() matchers.ConditionType {
+func (r *BulkHTTPRequest) GetMatchersCondition() matchers.ConditionType {
 	return r.matchersCondition
 }
 
 // SetMatchersCondition sets the condition for the matcher
-func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
+func (r *BulkHTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
 	r.matchersCondition = condition
 }
 
 // GetAttackType returns the attack
-func (r *HTTPRequest) GetAttackType() generators.Type {
+func (r *BulkHTTPRequest) GetAttackType() generators.Type {
 	return r.attackType
 }
 
 // SetAttackType sets the attack
-func (r *HTTPRequest) SetAttackType(attack generators.Type) {
+func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) {
 	r.attackType = attack
 }
 
-// MakeHTTPRequest creates a *http.Request from a request configuration
+func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, data string) (*HttpRequest, error) {
-func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) {
 	parsed, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
@@ -88,119 +94,102 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error
 		"Hostname": hostname,
 	}
 
-	if len(r.Raw) > 0 {
+	// if data contains \n it's a raw request
-		return r.makeHTTPRequestFromRaw(baseURL, values)
+	if strings.Contains(data, "\n") {
+		return r.makeHTTPRequestFromRaw(baseURL, data, values)
 	}
 
-	return r.makeHTTPRequestFromModel(baseURL, values)
+	return r.makeHTTPRequestFromModel(baseURL, data, values)
 }
 
 // MakeHTTPRequestFromModel creates a *http.Request from a request template
-func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
+func (r *BulkHTTPRequest) makeHTTPRequestFromModel(baseURL string, data string, values map[string]interface{}) (*HttpRequest, error) {
-	requests = make(chan *CompiledHTTP)
+	replacer := newReplacer(values)
+	URL := replacer.Replace(data)
 
-	// request generator
+	// Build a request on the specified URL
-	go func() {
+	req, err := http.NewRequest(r.Method, URL, nil)
-		defer close(requests)
+	if err != nil {
-		for _, path := range r.Path {
+		return nil, err
-			// process base request
+	}
-			replacer := newReplacer(values)
 
-			// Replace the dynamic variables in the URL if any
+	request, err := r.fillRequest(req, values)
-			URL := replacer.Replace(path)
+	if err != nil {
+		return nil, err
+	}
 
-			// Build a request on the specified URL
+	return &HttpRequest{Request: request}, nil
-			req, err := http.NewRequest(r.Method, URL, nil)
+}
-			if err != nil {
-				requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-				return
-			}
 
-			request, err := r.fillRequest(req, values)
+func (r *BulkHTTPRequest) StartGenerator() {
-			if err != nil {
+	r.generatorChan = r.generator(r.basePayloads)
-				requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+}
-				return
-			}
 
-			requests <- &CompiledHTTP{Request: request, Error: nil, Meta: nil}
+func (r *BulkHTTPRequest) PickOne() {
-		}
+	var ok bool
-	}()
+	r.currentGeneratorValue, ok = <-r.generatorChan
-
+	if !ok {
-	return
+		r.generator = nil
+	}
 }
 
 // makeHTTPRequestFromRaw creates a *http.Request from a raw request
-func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
+func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(baseURL string, data string, values map[string]interface{}) (*HttpRequest, error) {
-	requests = make(chan *CompiledHTTP)
+	// Add trailing line
-	// request generator
+	data += "\n"
-	go func() {
+	if len(r.Payloads) > 0 {
-		defer close(requests)
+		if r.generator == nil {
-
+			r.basePayloads = generators.LoadPayloads(r.Payloads)
-		for _, raw := range r.Raw {
+			generatorFunc := generators.SniperGenerator
-			// Add trailing line
+			switch r.attackType {
-			raw += "\n"
+			case generators.PitchFork:
-
+				generatorFunc = generators.PitchforkGenerator
-			if len(r.Payloads) > 0 {
+			case generators.ClusterBomb:
-				basePayloads := generators.LoadPayloads(r.Payloads)
+				generatorFunc = generators.ClusterbombGenerator
-				generatorFunc := generators.SniperGenerator
-				switch r.attackType {
-				case generators.PitchFork:
-					generatorFunc = generators.PitchforkGenerator
-				case generators.ClusterBomb:
-					generatorFunc = generators.ClusterbombGenerator
-				}
-
-				for genValues := range generatorFunc(basePayloads) {
-					compiledHTTP := r.handleRawWithPaylods(raw, baseURL, values, genValues)
-					requests <- compiledHTTP
-					if compiledHTTP.Error != nil {
-						return
-					}
-				}
-			} else {
-				// otherwise continue with normal flow
-				compiledHTTP := r.handleSimpleRaw(raw, baseURL, values)
-				requests <- compiledHTTP
-				if compiledHTTP.Error != nil {
-					return
-				}
 			}
+			r.generator = generatorFunc
+			r.StartGenerator()
 		}
-	}()
 
-	return requests, nil
+		r.PickOne()
+
+		return r.handleRawWithPaylods(data, baseURL, values, r.currentGeneratorValue)
+	}
+
+	// otherwise continue with normal flow
+	return r.handleSimpleRaw(data, baseURL, values)
 }
 
-func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) *CompiledHTTP {
+func (r *BulkHTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) (*HttpRequest, error) {
 	// base request
 	replacer := newReplacer(values)
 	// Replace the dynamic variables in the request if any
 	raw = replacer.Replace(raw)
 
-	compiledRequest, err := r.parseRawRequest(raw, baseURL)
+	rawRequest, err := r.parseRawRequest(raw, baseURL)
 	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+		return nil, err
 	}
 
-	req, err := http.NewRequest(compiledRequest.Method, compiledRequest.FullURL, strings.NewReader(compiledRequest.Data))
+	req, err := http.NewRequest(rawRequest.Method, rawRequest.FullURL, strings.NewReader(rawRequest.Data))
 	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+		return nil, err
 	}
 
 	// copy headers
-	for key, value := range compiledRequest.Headers {
+	for key, value := range rawRequest.Headers {
 		req.Header[key] = []string{value}
 	}
 
 	request, err := r.fillRequest(req, values)
 	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+		return nil, err
 	}
 
-	return &CompiledHTTP{Request: request, Error: nil, Meta: nil}
+	return &HttpRequest{Request: request}, nil
 }
 
-func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) *CompiledHTTP {
+func (r *BulkHTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) (*HttpRequest, error) {
 	baseValues := generators.CopyMap(values)
 	finValues := generators.MergeMaps(baseValues, genValues)
 
@@ -218,11 +207,11 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
 			expr := generators.TrimDelimiters(match)
 			compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
 			if err != nil {
-				return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+				return nil, err
 			}
 			result, err := compiled.Evaluate(finValues)
 			if err != nil {
-				return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+				return nil, err
 			}
 			dynamicValues[expr] = result
 		}
@@ -234,12 +223,12 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
 
 	compiledRequest, err := r.parseRawRequest(raw, baseURL)
 	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+		return nil, err
 	}
 
 	req, err := http.NewRequest(compiledRequest.Method, compiledRequest.FullURL, strings.NewReader(compiledRequest.Data))
 	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+		return nil, err
 	}
 
 	// copy headers
@@ -249,13 +238,13 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
 
 	request, err := r.fillRequest(req, values)
 	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
+		return nil, err
 	}
 
-	return &CompiledHTTP{Request: request, Error: nil, Meta: genValues}
+	return &HttpRequest{Request: request, Meta: genValues}, nil
 }
 
-func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
+func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
 	req.Header.Set("Connection", "close")
 	req.Close = true
 	replacer := newReplacer(values)
@@ -290,10 +279,8 @@ func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface
 	return retryablehttp.FromRequest(req)
 }
 
-// CompiledHTTP contains Generated HTTP Request or error
+type HttpRequest struct {
-type CompiledHTTP struct {
 	Request *retryablehttp.Request
-	Error   error
 	Meta    map[string]interface{}
 }
 
@@ -311,7 +298,7 @@ func (c *CustomHeaders) Set(value string) error {
 	return nil
 }
 
-type compiledRawRequest struct {
+type RawRequest struct {
 	FullURL string
 	Method  string
 	Path    string
@@ -320,10 +307,10 @@ type compiledRawRequest struct {
 }
 
 // parseRawRequest parses the raw request as supplied by the user
-func (r *HTTPRequest) parseRawRequest(request string, baseURL string) (*compiledRawRequest, error) {
+func (r *BulkHTTPRequest) parseRawRequest(request string, baseURL string) (*RawRequest, error) {
 	reader := bufio.NewReader(strings.NewReader(request))
 
-	rawRequest := compiledRawRequest{
+	rawRequest := RawRequest{
 		Headers: make(map[string]string),
 	}
 
@@ -406,3 +393,41 @@ func (r *HTTPRequest) parseRawRequest(request string, baseURL string) (*compiled
 	rawRequest.Data = string(b)
 	return &rawRequest, nil
 }
+
+func (r *BulkHTTPRequest) Next() bool {
+	if r.positionPath+r.positionRaw >= len(r.Path)+len(r.Raw) {
+		return false
+	}
+	return true
+}
+func (r *BulkHTTPRequest) Position() int {
+	return r.positionPath + r.positionRaw
+}
+func (r *BulkHTTPRequest) Reset() {
+	r.positionPath = 0
+	r.positionRaw = 0
+}
+func (r *BulkHTTPRequest) Current() string {
+	if r.positionPath <= len(r.Path) && len(r.Path) != 0 && r.positionRaw == 0 {
+		return r.Path[r.positionPath]
+	}
+
+	return r.Raw[r.positionRaw]
+}
+func (r *BulkHTTPRequest) Total() int {
+	return len(r.Path) + len(r.Raw)
+}
+
+func (r *BulkHTTPRequest) Increment() int {
+	if r.positionPath <= len(r.Path) && len(r.Path) != 0 && r.positionRaw == 0 {
+		r.positionPath++
+		return r.positionPath
+	}
+
+	// if we have payloads increment only when the generators are done
+	if r.generator == nil {
+		r.positionRaw++
+	}
+
+	return r.positionPath + r.positionRaw
+}

v2/pkg/templates/compile.go

@@ -27,12 +27,12 @@ func Parse(file string) (*Template, error) {
 	defer f.Close()
 
 	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsHTTP)+len(template.RequestsDNS) <= 0 {
+	if len(template.BulkRequestsHTTP)+len(template.RequestsDNS) <= 0 {
 		return nil, errors.New("No requests defined")
 	}
 
 	// Compile the matchers and the extractors for http requests
-	for _, request := range template.RequestsHTTP {
+	for _, request := range template.BulkRequestsHTTP {
 		// Get the condition between the matchers
 		condition, ok := matchers.ConditionTypes[request.MatchersCondition]
 		if !ok {

v2/pkg/templates/templates.go

@@ -10,9 +10,9 @@ type Template struct {
 	ID string `yaml:"id"`
 	// Info contains information about the template
 	Info Info `yaml:"info"`
-	// RequestHTTP contains the http request to make in the template
+	// BulkRequestsHTTP contains the http request to make in the template
-	RequestsHTTP []*requests.HTTPRequest `yaml:"requests,omitempty"`
+	BulkRequestsHTTP []*requests.BulkHTTPRequest `yaml:"requests,omitempty"`
-	// RequestDNS contains the dns request to make in the template
+	// RequestsDNS contains the dns request to make in the template
 	RequestsDNS []*requests.DNSRequest `yaml:"dns,omitempty"`
 }
 

v2/pkg/workflows/var.go

@@ -53,12 +53,12 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 	var gotResult bool
 	for _, template := range n.Templates {
 		if template.HTTPOptions != nil {
-			for _, request := range template.HTTPOptions.Template.RequestsHTTP {
+			for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
 				// apply externally supplied payloads if any
 				request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
 				// apply externally supplied payloads if any
 				request.Payloads = generators.MergeMaps(request.Payloads, externalVars)
-				template.HTTPOptions.HTTPRequest = request
+				template.HTTPOptions.BulkHttpRequest = request
 				httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions)
 				if err != nil {
 					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
@@ -70,7 +70,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 					continue
 				}
 
-				if httpExecuter.GotResults() {
+				if httpExecuter.Results {
 					gotResult = true
 					n.addResults(&result)
 				}
@@ -87,7 +87,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 					continue
 				}
 
-				if dnsExecuter.GotResults() {
+				if dnsExecuter.Results {
 					gotResult = true
 					n.addResults(&result)
 				}