more experimental parallelism

v2/internal/runner/options.go

@@ -50,7 +50,7 @@ func ParseOptions() *Options {
 	options := &Options{}
 
 	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
-	flag.Var(&options.Templates, "t","Template input file/files to run on host. Can be used multiple times.")
+	flag.Var(&options.Templates, "t", "Template input file/files to run on host. Can be used multiple times.")
 	flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on")
 	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
 	flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
@@ -59,7 +59,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
 	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
 	flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
-	flag.IntVar(&options.Threads, "c", 10, "Number of concurrent requests to make")
+	flag.IntVar(&options.Threads, "c", 50, "Number of concurrent requests to make")
 	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
 	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
 	flag.Var(&options.CustomHeaders, "H", "Custom Header.")

v2/internal/runner/runner.go

@@ -16,6 +16,7 @@ import (
 	"github.com/d5/tengo/v2/stdlib"
 	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
 	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -32,6 +33,7 @@ type Runner struct {
 	templatesConfig *nucleiConfig
 	// options contains configuration options for runner
 	options *Options
+	limiter chan struct{}
 }
 
 // New creates a new client for running enumeration process.
@@ -79,6 +81,9 @@ func New(options *Options) (*Runner, error) {
 		}
 		runner.output = output
 	}
+
+	runner.limiter = make(chan struct{}, options.Threads)
+
 	return runner, nil
 }
 
@@ -191,33 +196,37 @@ func (r *Runner) RunEnumeration() {
 		gologger.Fatalf("Error, no templates were found.\n")
 	}
 
-	// run with the specified templates
-	var results bool
+	var (
+		wgtemplates sync.WaitGroup
+		results     atomicboolean.AtomBool
+	)
+
 	for _, match := range allTemplates {
-		t, err := r.parse(match)
-		switch t.(type) {
-		case *templates.Template:
-			template := t.(*templates.Template)
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if dnsResults {
-					results = dnsResults
+		wgtemplates.Add(1)
+		go func(match string) {
+			defer wgtemplates.Done()
+			t, err := r.parse(match)
+			switch t.(type) {
+			case *templates.Template:
+				template := t.(*templates.Template)
+				for _, request := range template.RequestsDNS {
+					results.Or(r.processTemplateRequest(template, request))
 				}
-			}
-			for _, request := range template.BulkRequestsHTTP {
-				httpResults := r.processTemplateRequest(template, request)
-				if httpResults {
-					results = httpResults
+				for _, request := range template.BulkRequestsHTTP {
+					results.Or(r.processTemplateRequest(template, request))
 				}
+			case *workflows.Workflow:
+				workflow := t.(*workflows.Workflow)
+				r.ProcessWorkflowWithList(workflow)
+			default:
+				gologger.Errorf("Could not parse file '%s': %s\n", match, err)
 			}
-		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", match, err)
-		}
+		}(match)
 	}
-	if !results {
+
+	wgtemplates.Wait()
+
+	if !results.Get() {
 		if r.output != nil {
 			outputFile := r.output.Name()
 			r.output.Close()
@@ -242,9 +251,9 @@ func (r *Runner) processTemplateRequest(template *templates.Template, request in
 	if err != nil {
 		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
 	}
-	results := r.processTemplateWithList(template, request, file)
-	file.Close()
-	return results
+	defer file.Close()
+
+	return r.processTemplateWithList(template, request, file)
 }
 
 // processDomain processes the list with a template
@@ -297,19 +306,19 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 		return false
 	}
 
-	limiter := make(chan struct{}, r.options.Threads)
-	wg := &sync.WaitGroup{}
-
+	var wg sync.WaitGroup
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		text := scanner.Text()
 		if text == "" {
 			continue
 		}
-		limiter <- struct{}{}
+
+		r.limiter <- struct{}{}
 		wg.Add(1)
 
 		go func(URL string) {
+			defer wg.Done()
 			var result executer.Result
 
 			if httpExecuter != nil {
@@ -321,11 +330,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 			if result.Error != nil {
 				gologger.Warningf("Could not execute step: %s\n", result.Error)
 			}
-			<-limiter
-			wg.Done()
+			<-r.limiter
 		}(text)
 	}
-	close(limiter)
+
 	wg.Wait()
 
 	// See if we got any results from the executers

v2/pkg/atomicboolean/bool.go

@@ -0,0 +1,38 @@
+package atomicboolean
+
+import (
+	"sync"
+)
+
+type AtomBool struct {
+	sync.RWMutex
+	flag bool
+}
+
+func (b *AtomBool) Or(value bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.flag = b.flag || value
+}
+
+func (b *AtomBool) And(value bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.flag = b.flag && value
+}
+
+func (b *AtomBool) Set(value bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.flag = value
+}
+
+func (b *AtomBool) Get() bool {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.flag
+}

v2/pkg/requests/bulk-http-request.go

@@ -115,11 +115,11 @@ func (r *BulkHTTPRequest) makeHTTPRequestFromModel(baseURL string, data string,
 	return &HttpRequest{Request: request}, nil
 }
 
-func (r *BulkHTTPRequest) CreateGenerator(URL string) {
-	if r.gsfm == nil {
-		r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw)
-	}
+func (r *BulkHTTPRequest) InitGenerator() {
+	r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw)
+}
 
+func (r *BulkHTTPRequest) CreateGenerator(URL string) {
 	r.gsfm.Add(URL)
 }
 

v2/pkg/templates/compile.go

@@ -81,6 +81,8 @@ func Parse(file string) (*Template, error) {
 				return nil, err
 			}
 		}
+
+		request.InitGenerator()
 	}
 
 	// Compile the matchers and the extractors for dns requests