From a0318ffc8fce43a7524006a7bd1db7d08934b517 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Tue, 26 Oct 2021 20:36:44 +0530
Subject: [PATCH 1/9] Started refactor of template compilation + protocol
 building

---
 v2/pkg/engine/engine.go      |   9 +++
 v2/pkg/engine/engine_test.go |   1 +
 v2/pkg/templates/compile.go  | 145 +++++++++++++++++++++--------------
 3 files changed, 98 insertions(+), 57 deletions(-)
 create mode 100644 v2/pkg/engine/engine.go
 create mode 100644 v2/pkg/engine/engine_test.go

diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go
new file mode 100644
index 000000000..2bdb4a969
--- /dev/null
+++ b/v2/pkg/engine/engine.go
@@ -0,0 +1,9 @@
+package engine
+
+// Engine is an engine for running Nuclei Templates/Workflows.
+//
+// The engine contains multiple thread pools which allow using different
+// concurrency values per protocol executed. This was something which was
+// missing from the previous versions of nuclei.
+type Engine struct {
+}
diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go
new file mode 100644
index 000000000..00a22ef62
--- /dev/null
+++ b/v2/pkg/engine/engine_test.go
@@ -0,0 +1 @@
+package engine
diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go
index bf4e79b18..31446c9fc 100644
--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -69,11 +70,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 	options.TemplateInfo = template.Info
 	options.TemplatePath = filePath
 
-	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 {
-		return nil, fmt.Errorf("no requests defined for %s", template.ID)
-	}
-
 	// Compile the workflow request
 	if len(template.Workflows) > 0 {
 		compiled := &template.Workflow
@@ -83,56 +79,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 		template.CompiledWorkflow.Options = &options
 	}
 
-	// Compile the requests found
-	requests := []protocols.Request{}
-	if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP {
-		for _, req := range template.RequestsDNS {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
+	if err := template.compileProtocolRequests(options); err != nil {
+		return nil, err
 	}
-	if len(template.RequestsHTTP) > 0 {
-		if options.Options.OfflineHTTP {
-			operatorsList := []*operators.Operators{}
 
-		mainLoop:
-			for _, req := range template.RequestsHTTP {
-				for _, path := range req.Path {
-					if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) {
-						break mainLoop
-					}
-				}
-				operatorsList = append(operatorsList, &req.Operators)
-			}
-			if len(operatorsList) > 0 {
-				options.Operators = operatorsList
-				template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
-			}
-		} else {
-			for _, req := range template.RequestsHTTP {
-				requests = append(requests, req)
-			}
-			template.Executer = executer.NewExecuter(requests, &options)
-		}
-	}
-	if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP {
-		for _, req := range template.RequestsFile {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
-	}
-	if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP {
-		for _, req := range template.RequestsNetwork {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
-	}
-	if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless {
-		for _, req := range template.RequestsHeadless {
-			requests = append(requests, req)
-		}
-		template.Executer = executer.NewExecuter(requests, &options)
-	}
 	if template.Executer != nil {
 		if err := template.Executer.Compile(); err != nil {
 			return nil, errors.Wrap(err, "could not compile request")
@@ -151,14 +101,95 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 }
 
 // parseSelfContainedRequests parses the self contained template requests.
-func (t *Template) parseSelfContainedRequests() {
-	if !t.SelfContained {
+func (template *Template) parseSelfContainedRequests() {
+	if !template.SelfContained {
 		return
 	}
-	for _, request := range t.RequestsHTTP {
+	for _, request := range template.RequestsHTTP {
 		request.SelfContained = true
 	}
-	for _, request := range t.RequestsNetwork {
+	for _, request := range template.RequestsNetwork {
 		request.SelfContained = true
 	}
 }
+
+// Requests returns the total request count for the template
+func (template *Template) Requests() int {
+	return len(template.RequestsDNS) +
+		len(template.RequestsHTTP) +
+		len(template.RequestsFile) +
+		len(template.RequestsNetwork) +
+		len(template.RequestsHeadless) +
+		len(template.Workflows)
+}
+
+// compileProtocolRequests compiles all the protocol requests for the template
+func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error {
+	templateRequests := template.Requests()
+
+	if templateRequests == 0 {
+		return fmt.Errorf("no requests defined for %s", template.ID)
+	}
+
+	if template.Options.Options.OfflineHTTP {
+		template.compileOfflineHTTPRequest(options)
+		return nil
+	}
+
+	var requests []protocols.Request
+	switch {
+	case len(template.RequestsDNS) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
+
+	case len(template.RequestsFile) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
+
+	case len(template.RequestsNetwork) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
+
+	case len(template.RequestsHTTP) > 0:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
+
+	case len(template.RequestsHeadless) > 0 && options.Options.Headless:
+		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
+	}
+	template.Executer = executer.NewExecuter(requests, &options)
+	return nil
+}
+
+func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request {
+	switch reflect.TypeOf(requests).Kind() {
+	case reflect.Slice:
+		s := reflect.ValueOf(requests)
+
+		requestSlice := make([]protocols.Request, s.Len())
+		for i := 0; i < s.Len(); i++ {
+			value := s.Index(i)
+			valueInterface := value.Interface()
+			requestSlice[i] = valueInterface.(protocols.Request)
+		}
+		return requestSlice
+	}
+	return nil
+}
+
+// compileOfflineHTTPRequest iterates all requests if offline http mode is
+// specified and collects all matchers for all the base request templates
+// (those with URL {{BaseURL}} and it's slash variation.)
+func (template *Template) compileOfflineHTTPRequest(options protocols.ExecuterOptions) {
+	operatorsList := []*operators.Operators{}
+
+mainLoop:
+	for _, req := range template.RequestsHTTP {
+		for _, path := range req.Path {
+			if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) {
+				break mainLoop
+			}
+		}
+		operatorsList = append(operatorsList, &req.Operators)
+	}
+	if len(operatorsList) > 0 {
+		options.Operators = operatorsList
+		template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
+	}
+}

From 2a84b9eb4443b9a826ff4d692d254eed2e5ea52d Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Tue, 26 Oct 2021 20:40:02 +0530
Subject: [PATCH 2/9] misc

---
 v2/pkg/templates/compile.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go
index 31446c9fc..8c48308c5 100644
--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@@ -139,19 +139,19 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
 	var requests []protocols.Request
 	switch {
 	case len(template.RequestsDNS) > 0:
-		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
+		requests = template.convertRequestToProtocolsRequest(template.RequestsDNS)
 
 	case len(template.RequestsFile) > 0:
-		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
+		requests = template.convertRequestToProtocolsRequest(template.RequestsFile)
 
 	case len(template.RequestsNetwork) > 0:
-		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
+		requests = template.convertRequestToProtocolsRequest(template.RequestsNetwork)
 
 	case len(template.RequestsHTTP) > 0:
-		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
+		requests = template.convertRequestToProtocolsRequest(template.RequestsHTTP)
 
 	case len(template.RequestsHeadless) > 0 && options.Options.Headless:
-		requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
+		requests = template.convertRequestToProtocolsRequest(template.RequestsHeadless)
 	}
 	template.Executer = executer.NewExecuter(requests, &options)
 	return nil

From 97645dde520a0191cc87012e496720903a79bc43 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Wed, 27 Oct 2021 15:53:04 +0530
Subject: [PATCH 3/9] Added new workpool package + Misc refactor

---
 v2/cmd/nuclei/main.go             |   2 +
 v2/internal/runner/options.go     |   7 --
 v2/internal/runner/runner.go      | 145 +++++-------------------------
 v2/pkg/engine/engine.go           |   7 ++
 v2/pkg/engine/inputs/hmap/hmap.go | 120 +++++++++++++++++++++++++
 v2/pkg/engine/workpool.go         |  37 ++++++++
 v2/pkg/templates/compile.go       |   5 +-
 v2/pkg/templates/templates.go     |  30 +++++++
 v2/pkg/types/types.go             |   4 +
 9 files changed, 228 insertions(+), 129 deletions(-)
 create mode 100644 v2/pkg/engine/inputs/hmap/hmap.go
 create mode 100644 v2/pkg/engine/workpool.go

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index d2f8ebb9f..13602bfcd 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -110,6 +110,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
+		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
+		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "hc", 10, "maximum number of headless templates to be executed in parallel"),
 	)
 
 	createGroup(flagSet, "optimization", "Optimizations",
diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go
index 59791fe63..557821745 100644
--- a/v2/internal/runner/options.go
+++ b/v2/internal/runner/options.go
@@ -47,13 +47,6 @@ func ParseOptions(options *types.Options) {
 		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
 
-	// Auto adjust rate limits when using headless mode if the user
-	// hasn't specified any custom limits.
-	if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 {
-		options.BulkSize = 2
-		options.TemplateThreads = 2
-	}
-
 	// Load the resolvers if user asked for them
 	loadResolvers(options)
 
diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index b3f806952..d4d7f40c8 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -16,14 +16,12 @@ import (
 	"go.uber.org/ratelimit"
 	"gopkg.in/yaml.v2"
 
-	"github.com/projectdiscovery/filekv"
-	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
@@ -46,22 +44,20 @@ import (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	hostMap         *hybrid.HybridMap
-	hostMapStream   *filekv.FileDB
-	output          output.Writer
-	interactsh      *interactsh.Client
-	inputCount      int64
-	templatesConfig *config.Config
-	options         *types.Options
-	projectFile     *projectfile.ProjectFile
-	catalog         *catalog.Catalog
-	progress        progress.Progress
-	colorizer       aurora.Aurora
-	issuesClient    *reporting.Client
-	addColor        func(severity.Severity) string
-	browser         *engine.Browser
-	ratelimiter     ratelimit.Limiter
-	hostErrors      *hosterrorscache.Cache
+	output            output.Writer
+	interactsh        *interactsh.Client
+	templatesConfig   *config.Config
+	options           *types.Options
+	projectFile       *projectfile.ProjectFile
+	catalog           *catalog.Catalog
+	progress          progress.Progress
+	colorizer         aurora.Aurora
+	issuesClient      *reporting.Client
+	addColor          func(severity.Severity) string
+	hmapInputProvider *hmap.Input
+	browser           *engine.Browser
+	ratelimiter       ratelimit.Limiter
+	hostErrors        *hosterrorscache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -116,103 +112,13 @@ func New(options *types.Options) (*Runner, error) {
 	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+
+	// Initialize the input source
+	hmapInput, err := hmap.New(options)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not create temporary input file")
-	}
-	runner.hostMap = hm
-
-	if options.Stream {
-		fkvOptions := filekv.DefaultOptions
-		if tmpFileName, err := fileutil.GetTempFileName(); err != nil {
-			return nil, errors.Wrap(err, "could not create temporary input file")
-		} else {
-			fkvOptions.Path = tmpFileName
-		}
-		fkv, err := filekv.Open(fkvOptions)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not create temporary unsorted input file")
-		}
-		runner.hostMapStream = fkv
-	}
-
-	runner.inputCount = 0
-	dupeCount := 0
-
-	// Handle multiple targets
-	if len(options.Targets) != 0 {
-		for _, target := range options.Targets {
-			url := strings.TrimSpace(target)
-			if url == "" {
-				continue
-			}
-
-			if _, ok := runner.hostMap.Get(url); ok {
-				dupeCount++
-				continue
-			}
-
-			runner.inputCount++
-			// nolint:errcheck // ignoring error
-			runner.hostMap.Set(url, nil)
-			if options.Stream {
-				_ = runner.hostMapStream.Set([]byte(url), nil)
-			}
-		}
-	}
-
-	// Handle stdin
-	if options.Stdin {
-		scanner := bufio.NewScanner(os.Stdin)
-		for scanner.Scan() {
-			url := strings.TrimSpace(scanner.Text())
-			if url == "" {
-				continue
-			}
-
-			if _, ok := runner.hostMap.Get(url); ok {
-				dupeCount++
-				continue
-			}
-
-			runner.inputCount++
-			// nolint:errcheck // ignoring error
-			runner.hostMap.Set(url, nil)
-			if options.Stream {
-				_ = runner.hostMapStream.Set([]byte(url), nil)
-			}
-		}
-	}
-
-	// Handle target file
-	if options.TargetsFilePath != "" {
-		input, inputErr := os.Open(options.TargetsFilePath)
-		if inputErr != nil {
-			return nil, errors.Wrap(inputErr, "could not open targets file")
-		}
-		scanner := bufio.NewScanner(input)
-		for scanner.Scan() {
-			url := strings.TrimSpace(scanner.Text())
-			if url == "" {
-				continue
-			}
-			if _, ok := runner.hostMap.Get(url); ok {
-				dupeCount++
-				continue
-			}
-			runner.inputCount++
-			// nolint:errcheck // ignoring error
-			runner.hostMap.Set(url, nil)
-			if options.Stream {
-				_ = runner.hostMapStream.Set([]byte(url), nil)
-			}
-		}
-		input.Close()
-	}
-
-	if dupeCount > 0 {
-		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
+		return nil, errors.Wrap(err, "could not create input provider")
 	}
+	runner.hmapInputProvider = hmapInput
 
 	// Create the output file if asked
 	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile)
@@ -312,13 +218,10 @@ func (r *Runner) Close() {
 	if r.output != nil {
 		r.output.Close()
 	}
-	r.hostMap.Close()
 	if r.projectFile != nil {
 		r.projectFile.Close()
 	}
-	if r.options.Stream {
-		r.hostMapStream.Close()
-	}
+	r.hmapInputProvider.Close()
 	protocolinit.Close()
 }
 
@@ -456,7 +359,7 @@ func (r *Runner) RunEnumeration() error {
 		if len(template.Workflows) > 0 {
 			continue
 		}
-		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
+		unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count()
 	}
 
 	if r.options.VerboseVerbose {
@@ -509,7 +412,7 @@ func (r *Runner) RunEnumeration() error {
 		if len(t.Workflows) > 0 {
 			continue
 		}
-		totalRequests += int64(t.TotalRequests) * r.inputCount
+		totalRequests += int64(t.TotalRequests) * r.hmapInputProvider.Count()
 	}
 	if totalRequests < unclusteredRequests {
 		gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
@@ -531,7 +434,7 @@ func (r *Runner) RunEnumeration() error {
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
 
 	// tracks global progress and captures stdout/stderr until p.Wait finishes
-	r.progress.Init(r.inputCount, templateCount, totalRequests)
+	r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests)
 
 	for _, t := range finalTemplates {
 		wgtemplates.Add()
diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go
index 2bdb4a969..978e845ca 100644
--- a/v2/pkg/engine/engine.go
+++ b/v2/pkg/engine/engine.go
@@ -7,3 +7,10 @@ package engine
 // missing from the previous versions of nuclei.
 type Engine struct {
 }
+
+// InputProvider is an input provider interface for the nuclei execution
+// engine.
+//
+// An example InputProvider is provided in form of hmap input provider.
+type InputProvider interface {
+}
diff --git a/v2/pkg/engine/inputs/hmap/hmap.go b/v2/pkg/engine/inputs/hmap/hmap.go
new file mode 100644
index 000000000..2910d0451
--- /dev/null
+++ b/v2/pkg/engine/inputs/hmap/hmap.go
@@ -0,0 +1,120 @@
+// Package hmap implements a hybrid hmap/filekv backed input provider
+// for nuclei that can either stream or store results using different kv stores.
+package hmap
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/filekv"
+	"github.com/projectdiscovery/fileutil"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// Input is a hmap/filekv backed nuclei Input provider
+type Input struct {
+	inputCount    int64
+	dupeCount     int64
+	hostMap       *hybrid.HybridMap
+	hostMapStream *filekv.FileDB
+}
+
+// New creates a new hmap backed nuclei Input Provider
+// and initializes it based on the passed options Model.
+func New(options *types.Options) (*Input, error) {
+	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create temporary input file")
+	}
+
+	input := &Input{hostMap: hm}
+	if options.Stream {
+		fkvOptions := filekv.DefaultOptions
+		if tmpFileName, err := fileutil.GetTempFileName(); err != nil {
+			return nil, errors.Wrap(err, "could not create temporary input file")
+		} else {
+			fkvOptions.Path = tmpFileName
+		}
+		fkv, err := filekv.Open(fkvOptions)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create temporary unsorted input file")
+		}
+		input.hostMapStream = fkv
+	}
+	if initErr := input.initializeInputSources(options); initErr != nil {
+		return nil, initErr
+	}
+	if input.dupeCount > 0 {
+		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", input.dupeCount)
+	}
+	return input, nil
+}
+
+// Close closes the input provider
+func (i *Input) Close() {
+	i.hostMap.Close()
+	if i.hostMapStream != nil {
+		i.hostMapStream.Close()
+	}
+}
+
+// initializeInputSources initializes the input sources for hmap input
+func (i *Input) initializeInputSources(options *types.Options) error {
+	// Handle targets flags
+	for _, target := range options.Targets {
+		i.normalizeStoreInputValue(target)
+	}
+
+	// Handle stdin
+	if options.Stdin {
+		i.scanInputFromReader(os.Stdin)
+	}
+
+	// Handle target file
+	if options.TargetsFilePath != "" {
+		input, inputErr := os.Open(options.TargetsFilePath)
+		if inputErr != nil {
+			return errors.Wrap(inputErr, "could not open targets file")
+		}
+		i.scanInputFromReader(input)
+		input.Close()
+	}
+	return nil
+}
+
+// scanInputFromReader scans a line of input from reader and passes it for storage
+func (i *Input) scanInputFromReader(reader io.Reader) {
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		i.normalizeStoreInputValue(scanner.Text())
+	}
+}
+
+// normalizeStoreInputValue normalizes and stores passed input values
+func (i *Input) normalizeStoreInputValue(value string) {
+	url := strings.TrimSpace(value)
+	if url == "" {
+		return
+	}
+
+	if _, ok := i.hostMap.Get(url); ok {
+		i.dupeCount++
+		return
+	}
+
+	i.inputCount++
+	_ = i.hostMap.Set(url, nil)
+	if i.hostMapStream != nil {
+		_ = i.hostMapStream.Set([]byte(url), nil)
+	}
+}
+
+// Count returns the input count
+func (i *Input) Count() int64 {
+	return i.inputCount
+}
diff --git a/v2/pkg/engine/workpool.go b/v2/pkg/engine/workpool.go
new file mode 100644
index 000000000..04db44715
--- /dev/null
+++ b/v2/pkg/engine/workpool.go
@@ -0,0 +1,37 @@
+package engine
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+// WorkPool implements an execution pool for executing different
+// types of task with different concurreny requirements.
+//
+// It also allows Configuration of such requirements. This is used
+// for per-module like separate headless concurrency etc.
+type WorkPool struct {
+	config WorkPoolConfig
+}
+
+// WorkPoolConfig is the configuration for workpool
+type WorkPoolConfig struct {
+	// InputConcurrency is the concurrency for inputs values.
+	InputConcurrency int
+	// TypeConcurrency is the concurrency for the request type templates.
+	TypeConcurrency int
+	// HeadlessInputConcurrency is the concurrency for headless inputs values.
+	HeadlessInputConcurrency int
+	// TypeConcurrency is the concurrency for the headless request type templates.
+	HeadlessTypeConcurrency int
+}
+
+// New returns a new WorkPool instance
+func New(config WorkPoolConfig) *WorkPool {
+	return &WorkPool{config: config}
+}
+
+// TODO: port these invocations of waitgroups and input logic into a generic
+// workpool type functionality.
+func (w *WorkPool) Execute(templates []*templates.Template) {
+
+}
diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go
index 8c48308c5..22c2d764f 100644
--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@@ -131,7 +131,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
 		return fmt.Errorf("no requests defined for %s", template.ID)
 	}
 
-	if template.Options.Options.OfflineHTTP {
+	if options.Options.OfflineHTTP {
 		template.compileOfflineHTTPRequest(options)
 		return nil
 	}
@@ -157,6 +157,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
 	return nil
 }
 
+// convertRequestToProtocolsRequest is a convenience wrapper to convert
+// arbitrary interfaces which are slices of requests from the template to a
+// slice of protocols.Request interface items.
 func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request {
 	switch reflect.TypeOf(requests).Kind() {
 	case reflect.Slice:
diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go
index 61cb49ff1..7e0e42a37 100644
--- a/v2/pkg/templates/templates.go
+++ b/v2/pkg/templates/templates.go
@@ -73,3 +73,33 @@ type Template struct {
 
 	Path string `yaml:"-" json:"-"`
 }
+
+// TemplateTypes is a list of accepted template types
+var TemplateTypes = []string{
+	"dns",
+	"file",
+	"http",
+	"headless",
+	"network",
+	"workflow",
+}
+
+// Type returns the type of the template
+func (t *Template) Type() string {
+	switch {
+	case len(t.RequestsDNS) > 0:
+		return "dns"
+	case len(t.RequestsFile) > 0:
+		return "file"
+	case len(t.RequestsHTTP) > 0:
+		return "http"
+	case len(t.RequestsHeadless) > 0:
+		return "headless"
+	case len(t.RequestsNetwork) > 0:
+		return "network"
+	case len(t.Workflows) > 0:
+		return "workflow"
+	default:
+		return ""
+	}
+}
diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
index d258f4245..9f8f784b1 100644
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@@ -77,6 +77,10 @@ type Options struct {
 	BulkSize int
 	// TemplateThreads is the number of templates executed in parallel
 	TemplateThreads int
+	// HeadlessBulkSize is the of targets analyzed in parallel for each headless template
+	HeadlessBulkSize int
+	// HeadlessTemplateThreads is the number of headless templates executed in parallel
+	HeadlessTemplateThreads int
 	// Timeout is the seconds to wait for a response from the server.
 	Timeout int
 	// Retries is the number of times to retry the request

From c16c93fe7ca698d3086b85fd86e0c1e8c99a222e Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Wed, 27 Oct 2021 16:50:36 +0530
Subject: [PATCH 4/9] refactor the modules to core

---
 v2/internal/runner/runner.go                  | 46 +++-------------
 v2/pkg/core/engine.go                         | 53 +++++++++++++++++++
 v2/pkg/core/engine_test.go                    |  1 +
 .../processor.go => pkg/core/execute.go}      | 46 +++++++++++++++-
 v2/pkg/{engine => core}/inputs/hmap/hmap.go   |  0
 .../execute.go => core/workflow_execute.go}   | 12 ++---
 .../workflow_execute_test.go}                 | 39 +++++++-------
 v2/pkg/{engine => core}/workpool.go           |  6 +--
 v2/pkg/engine/engine.go                       | 16 ------
 v2/pkg/engine/engine_test.go                  |  1 -
 v2/pkg/protocols/protocols.go                 |  6 +++
 v2/pkg/templates/workflows.go                 | 14 +----
 12 files changed, 139 insertions(+), 101 deletions(-)
 create mode 100644 v2/pkg/core/engine.go
 create mode 100644 v2/pkg/core/engine_test.go
 rename v2/{internal/runner/processor.go => pkg/core/execute.go} (61%)
 rename v2/pkg/{engine => core}/inputs/hmap/hmap.go (100%)
 rename v2/pkg/{workflows/execute.go => core/workflow_execute.go} (95%)
 rename v2/pkg/{workflows/execute_test.go => core/workflow_execute_test.go} (77%)
 rename v2/pkg/{engine => core}/workpool.go (90%)
 delete mode 100644 v2/pkg/engine/engine.go
 delete mode 100644 v2/pkg/engine/engine_test.go

diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index d4d7f40c8..100e364f0 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -2,7 +2,6 @@ package runner
 
 import (
 	"bufio"
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -11,7 +10,6 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/remeh/sizedwaitgroup"
-	"github.com/rs/xid"
 	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
 	"gopkg.in/yaml.v2"
@@ -21,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core"
 	"github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -28,7 +27,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
@@ -247,6 +245,9 @@ func (r *Runner) RunEnumeration() error {
 		cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose)
 	}
 	r.hostErrors = cache
+
+	// Create the executer options which will be used throughout the execution
+	// stage by the nuclei engine modules.
 	executerOpts := protocols.ExecuterOptions{
 		Output:          r.output,
 		Options:         r.options,
@@ -259,12 +260,13 @@ func (r *Runner) RunEnumeration() error {
 		Browser:         r.browser,
 		HostErrorsCache: cache,
 	}
+	engine := core.New(r.options)
+	engine.SetExecuterOptions(executerOpts)
 
 	workflowLoader, err := parsers.NewLoader(&executerOpts)
 	if err != nil {
 		return errors.Wrap(err, "Could not create loader.")
 	}
-
 	executerOpts.WorkflowLoader = workflowLoader
 
 	loaderConfig := loader.Config{
@@ -370,42 +372,6 @@ func (r *Runner) RunEnumeration() error {
 			r.logAvailableTemplate(template.Path)
 		}
 	}
-	templatesMap := make(map[string]*templates.Template)
-	for _, v := range store.Templates() {
-		templatesMap[v.Path] = v
-	}
-	originalTemplatesCount := len(store.Templates())
-	clusterCount := 0
-	clusters := clusterer.Cluster(templatesMap)
-	for _, cluster := range clusters {
-		if len(cluster) > 1 && !r.options.OfflineHTTP {
-			executerOpts := protocols.ExecuterOptions{
-				Output:          r.output,
-				Options:         r.options,
-				Progress:        r.progress,
-				Catalog:         r.catalog,
-				RateLimiter:     r.ratelimiter,
-				IssuesClient:    r.issuesClient,
-				Browser:         r.browser,
-				ProjectFile:     r.projectFile,
-				Interactsh:      r.interactsh,
-				HostErrorsCache: cache,
-			}
-			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
-
-			finalTemplates = append(finalTemplates, &templates.Template{
-				ID:            clusterID,
-				RequestsHTTP:  cluster[0].RequestsHTTP,
-				Executer:      clusterer.NewExecuter(cluster, &executerOpts),
-				TotalRequests: len(cluster[0].RequestsHTTP),
-			})
-			clusterCount += len(cluster)
-		} else {
-			finalTemplates = append(finalTemplates, cluster...)
-		}
-	}
-
-	finalTemplates = append(finalTemplates, store.Workflows()...)
 
 	var totalRequests int64
 	for _, t := range finalTemplates {
diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go
new file mode 100644
index 000000000..d7fdc6bbc
--- /dev/null
+++ b/v2/pkg/core/engine.go
@@ -0,0 +1,53 @@
+package core
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// Engine is an engine for running Nuclei Templates/Workflows.
+//
+// The engine contains multiple thread pools which allow using different
+// concurrency values per protocol executed.
+//
+// The engine does most of the heavy lifting of execution, from clustering
+// templates to leading to the final execution by the workpool, it is
+// handled by the engine.
+type Engine struct {
+	workPool     *WorkPool
+	options      *types.Options
+	executerOpts protocols.ExecuterOptions
+}
+
+// InputProvider is an input provider interface for the nuclei execution
+// engine.
+//
+// An example InputProvider is provided in form of hmap input provider.
+type InputProvider interface {
+}
+
+// New returns a new Engine instance
+func New(options *types.Options) *Engine {
+	workPool := NewWorkPool(WorkPoolConfig{
+		InputConcurrency:         options.BulkSize,
+		TypeConcurrency:          options.TemplateThreads,
+		HeadlessInputConcurrency: options.HeadlessBulkSize,
+		HeadlessTypeConcurrency:  options.HeadlessTemplateThreads,
+	})
+	engine := &Engine{
+		options:  options,
+		workPool: workPool,
+	}
+	return engine
+}
+
+// SetExecuterOptions sets the executer options for the engine. This is required
+// before using the engine to perform any execution.
+func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) {
+	e.executerOpts = options
+}
+
+// ExecuterOptions returns protocols.ExecuterOptions for nuclei engine.
+func (e *Engine) ExecuterOptions() protocols.ExecuterOptions {
+	return e.executerOpts
+}
diff --git a/v2/pkg/core/engine_test.go b/v2/pkg/core/engine_test.go
new file mode 100644
index 000000000..9a8bc9592
--- /dev/null
+++ b/v2/pkg/core/engine_test.go
@@ -0,0 +1 @@
+package core
diff --git a/v2/internal/runner/processor.go b/v2/pkg/core/execute.go
similarity index 61%
rename from v2/internal/runner/processor.go
rename to v2/pkg/core/execute.go
index 6b14b5f91..65c31558d 100644
--- a/v2/internal/runner/processor.go
+++ b/v2/pkg/core/execute.go
@@ -1,5 +1,48 @@
-package runner
+package core
 
+import (
+	"fmt"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/rs/xid"
+)
+
+// clusterTemplates performs identical http requests clustering for a list of templates
+func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) {
+	if e.options.OfflineHTTP {
+		return templatesList, 0
+	}
+
+	templatesMap := make(map[string]*templates.Template)
+	for _, v := range templatesList {
+		templatesMap[v.Path] = v
+	}
+	clusterCount := 0
+
+	finalTemplatesList := make([]*templates.Template, 0, len(templatesList))
+	clusters := clusterer.Cluster(templatesMap)
+	for _, cluster := range clusters {
+		if len(cluster) > 1 {
+			executerOpts := e.ExecuterOptions()
+
+			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
+
+			finalTemplatesList = append(finalTemplatesList, &templates.Template{
+				ID:            clusterID,
+				RequestsHTTP:  cluster[0].RequestsHTTP,
+				Executer:      clusterer.NewExecuter(cluster, &executerOpts),
+				TotalRequests: len(cluster[0].RequestsHTTP),
+			})
+			clusterCount += len(cluster)
+		} else {
+			finalTemplatesList = append(finalTemplatesList, cluster...)
+		}
+	}
+	return finalTemplatesList, clusterCount
+}
+
+/*
 import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -79,3 +122,4 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
 	wg.Wait()
 	return results.Load()
 }
+*/
diff --git a/v2/pkg/engine/inputs/hmap/hmap.go b/v2/pkg/core/inputs/hmap/hmap.go
similarity index 100%
rename from v2/pkg/engine/inputs/hmap/hmap.go
rename to v2/pkg/core/inputs/hmap/hmap.go
diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/core/workflow_execute.go
similarity index 95%
rename from v2/pkg/workflows/execute.go
rename to v2/pkg/core/workflow_execute.go
index c0710c6ad..59a5ade55 100644
--- a/v2/pkg/workflows/execute.go
+++ b/v2/pkg/core/workflow_execute.go
@@ -1,12 +1,6 @@
-package workflows
-
-import (
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/remeh/sizedwaitgroup"
-	"go.uber.org/atomic"
-)
+package core
 
+/*
 // RunWorkflow runs a workflow on an input and returns true or false
 func (w *Workflow) RunWorkflow(input string) bool {
 	results := &atomic.Bool{}
@@ -123,4 +117,4 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 		}
 	}
 	return mainErr
-}
+}*/
diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/core/workflow_execute_test.go
similarity index 77%
rename from v2/pkg/workflows/execute_test.go
rename to v2/pkg/core/workflow_execute_test.go
index 6d9ab6a09..62cd40840 100644
--- a/v2/pkg/workflows/execute_test.go
+++ b/v2/pkg/core/workflow_execute_test.go
@@ -1,5 +1,6 @@
-package workflows
+package core
 
+/*
 import (
 	"testing"
 
@@ -10,13 +11,14 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 )
 
 func TestWorkflowsSimple(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
-	workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
-		{Executers: []*ProtocolExecuterPair{{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
 		}},
 	}}
@@ -29,13 +31,13 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
-		{Executers: []*ProtocolExecuterPair{{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
 		}},
-		{Executers: []*ProtocolExecuterPair{{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				secondInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
@@ -53,14 +55,14 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
-		{Executers: []*ProtocolExecuterPair{{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
 			}, outputs: []*output.InternalWrappedEvent{
 				{OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}},
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
-		}, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+		}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				secondInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
@@ -78,12 +80,12 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
-		{Executers: []*ProtocolExecuterPair{{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: false, executeHook: func(input string) {
 				firstInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
-		}, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+		}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				secondInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
@@ -101,8 +103,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
-		{Executers: []*ProtocolExecuterPair{{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
 			}, outputs: []*output.InternalWrappedEvent{
@@ -111,7 +113,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 					Extracts: map[string][]string{},
 				}},
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
-		}, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+		}, Matchers: []*workflows.Matcher{{Name: "tomcat", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				secondInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
@@ -129,8 +131,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
-		{Executers: []*ProtocolExecuterPair{{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
+		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
 			}, outputs: []*output.InternalWrappedEvent{
@@ -139,7 +141,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 					Extracts: map[string][]string{},
 				}},
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
-		}, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+		}, Matchers: []*workflows.Matcher{{Name: "apache", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				secondInput = input
 			}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
@@ -187,3 +189,4 @@ func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.Outpu
 	}
 	return nil
 }
+*/
diff --git a/v2/pkg/engine/workpool.go b/v2/pkg/core/workpool.go
similarity index 90%
rename from v2/pkg/engine/workpool.go
rename to v2/pkg/core/workpool.go
index 04db44715..f307b205d 100644
--- a/v2/pkg/engine/workpool.go
+++ b/v2/pkg/core/workpool.go
@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -25,8 +25,8 @@ type WorkPoolConfig struct {
 	HeadlessTypeConcurrency int
 }
 
-// New returns a new WorkPool instance
-func New(config WorkPoolConfig) *WorkPool {
+// NewWorkPool returns a new WorkPool instance
+func NewWorkPool(config WorkPoolConfig) *WorkPool {
 	return &WorkPool{config: config}
 }
 
diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go
deleted file mode 100644
index 978e845ca..000000000
--- a/v2/pkg/engine/engine.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package engine
-
-// Engine is an engine for running Nuclei Templates/Workflows.
-//
-// The engine contains multiple thread pools which allow using different
-// concurrency values per protocol executed. This was something which was
-// missing from the previous versions of nuclei.
-type Engine struct {
-}
-
-// InputProvider is an input provider interface for the nuclei execution
-// engine.
-//
-// An example InputProvider is provided in form of hmap input provider.
-type InputProvider interface {
-}
diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go
deleted file mode 100644
index 00a22ef62..000000000
--- a/v2/pkg/engine/engine_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package engine
diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go
index d77ba147e..83a76f7cf 100644
--- a/v2/pkg/protocols/protocols.go
+++ b/v2/pkg/protocols/protocols.go
@@ -64,6 +64,12 @@ type ExecuterOptions struct {
 	WorkflowLoader model.WorkflowLoader
 }
 
+// Copy returns a copy of the executeroptions structure
+func (e ExecuterOptions) Copy() ExecuterOptions {
+	copy := e
+	return copy
+}
+
 // Request is an interface implemented any protocol based request generator.
 type Request interface {
 	// Compile compiles the request generators preparing any requests possible.
diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go
index 9d510f2e2..246c4bf8e 100644
--- a/v2/pkg/templates/workflows.go
+++ b/v2/pkg/templates/workflows.go
@@ -62,19 +62,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr
 		return nil
 	}
 	for _, path := range paths {
-		opts := protocols.ExecuterOptions{
-			Output:          options.Output,
-			Options:         options.Options,
-			Progress:        options.Progress,
-			Catalog:         options.Catalog,
-			Browser:         options.Browser,
-			RateLimiter:     options.RateLimiter,
-			IssuesClient:    options.IssuesClient,
-			Interactsh:      options.Interactsh,
-			ProjectFile:     options.ProjectFile,
-			HostErrorsCache: options.HostErrorsCache,
-		}
-		template, err := Parse(path, preprocessor, opts)
+		template, err := Parse(path, preprocessor, options.Copy())
 		if err != nil {
 			gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err)
 			continue

From df78ea72c5c266fecc4c01e97a700025a921ed42 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Wed, 27 Oct 2021 18:41:39 +0530
Subject: [PATCH 5/9] misc

---
 v2/pkg/core/execute.go  | 24 ++++++++++++++++++++++++
 v2/pkg/core/workpool.go |  2 --
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go
index 65c31558d..112be4e2c 100644
--- a/v2/pkg/core/execute.go
+++ b/v2/pkg/core/execute.go
@@ -8,6 +8,30 @@ import (
 	"github.com/rs/xid"
 )
 
+// Execute takes a list of templates/workflows that have been compiled
+// and executes them based on provided concurrency options.
+//
+// All the execution logic for the templates/workflows happens in this part
+// of the engine.
+func (e *Engine) Execute(templates []*templates.Template) {
+	finalTemplates, clusterCount := e.clusterTemplates(templates)
+
+	for _, template := range finalTemplates {
+		templateType := template.Type()
+
+		switch {
+		case template.SelfContained:
+			// Self Contained requests are executed here
+
+		case templateType == "workflow":
+			// Workflows requests are executed here
+
+		default:
+			// All other request types are executed here
+		}
+	}
+}
+
 // clusterTemplates performs identical http requests clustering for a list of templates
 func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) {
 	if e.options.OfflineHTTP {
diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go
index f307b205d..ef50f70fc 100644
--- a/v2/pkg/core/workpool.go
+++ b/v2/pkg/core/workpool.go
@@ -30,8 +30,6 @@ func NewWorkPool(config WorkPoolConfig) *WorkPool {
 	return &WorkPool{config: config}
 }
 
-// TODO: port these invocations of waitgroups and input logic into a generic
-// workpool type functionality.
 func (w *WorkPool) Execute(templates []*templates.Template) {
 
 }

From d124dbacc7cf3b3ce4328c391a908cedd68703d6 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Thu, 28 Oct 2021 17:20:07 +0530
Subject: [PATCH 6/9] Moved all important execution stuff to engine

---
 v2/internal/runner/runner.go                |  43 ++----
 v2/pkg/core/engine.go                       |   4 +
 v2/pkg/core/execute.go                      | 162 +++++++++-----------
 v2/pkg/core/inputs/{hmap => hybrid}/hmap.go |  17 +-
 v2/pkg/core/workflow_execute.go             |  29 ++--
 v2/pkg/core/workflow_execute_test.go        |  30 ++--
 v2/pkg/core/workpool.go                     |  38 ++++-
 v2/pkg/protocols/protocols.go               |   2 +
 8 files changed, 171 insertions(+), 154 deletions(-)
 rename v2/pkg/core/inputs/{hmap => hybrid}/hmap.go (88%)

diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index 100e364f0..84f2837a8 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -9,8 +9,6 @@ import (
 
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
-	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
 	"gopkg.in/yaml.v2"
 
@@ -20,7 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v2/pkg/core"
-	"github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap"
+	"github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hybrid"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
@@ -34,7 +32,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
@@ -52,7 +49,7 @@ type Runner struct {
 	colorizer         aurora.Aurora
 	issuesClient      *reporting.Client
 	addColor          func(severity.Severity) string
-	hmapInputProvider *hmap.Input
+	hmapInputProvider *hybrid.Input
 	browser           *engine.Browser
 	ratelimiter       ratelimit.Limiter
 	hostErrors        *hosterrorscache.Cache
@@ -112,7 +109,7 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	// Initialize the input source
-	hmapInput, err := hmap.New(options)
+	hmapInput, err := hybrid.New(options)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create input provider")
 	}
@@ -259,6 +256,7 @@ func (r *Runner) RunEnumeration() error {
 		ProjectFile:     r.projectFile,
 		Browser:         r.browser,
 		HostErrorsCache: cache,
+		Colorizer:       r.colorizer,
 	}
 	engine := core.New(r.options)
 	engine.SetExecuterOptions(executerOpts)
@@ -351,9 +349,6 @@ func (r *Runner) RunEnumeration() error {
 		gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
 	}
 
-	// pre-parse all the templates, apply filters
-	finalTemplates := []*templates.Template{}
-
 	var unclusteredRequests int64
 	for _, template := range store.Templates() {
 		// workflows will dynamically adjust the totals while running, as
@@ -373,6 +368,12 @@ func (r *Runner) RunEnumeration() error {
 		}
 	}
 
+	// Cluster the templates first because we want info on how many
+	// templates did we cluster for showing to user in CLI
+	originalTemplatesCount := len(store.Templates())
+	finalTemplates, clusterCount := engine.ClusterTemplates(store.Templates())
+	finalTemplates = append(finalTemplates, store.Workflows()...)
+
 	var totalRequests int64
 	for _, t := range finalTemplates {
 		if len(t.Workflows) > 0 {
@@ -391,32 +392,10 @@ func (r *Runner) RunEnumeration() error {
 		return errors.New("no valid templates were found")
 	}
 
-	/*
-		TODO does it make sense to run the logic below if there are no targets specified?
-		Can we safely assume the user is just experimenting with the template/workflow filters before running them?
-	*/
-
-	results := &atomic.Bool{}
-	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
-
 	// tracks global progress and captures stdout/stderr until p.Wait finishes
 	r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests)
 
-	for _, t := range finalTemplates {
-		wgtemplates.Add()
-		go func(template *templates.Template) {
-			defer wgtemplates.Done()
-
-			if template.SelfContained {
-				results.CAS(false, r.processSelfContainedTemplates(template))
-			} else if len(template.Workflows) > 0 {
-				results.CAS(false, r.processWorkflowWithList(template))
-			} else {
-				results.CAS(false, r.processTemplateWithList(template))
-			}
-		}(t)
-	}
-	wgtemplates.Wait()
+	results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true)
 
 	if r.interactsh != nil {
 		matched := r.interactsh.Close()
diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go
index d7fdc6bbc..455546244 100644
--- a/v2/pkg/core/engine.go
+++ b/v2/pkg/core/engine.go
@@ -24,6 +24,10 @@ type Engine struct {
 //
 // An example InputProvider is provided in form of hmap input provider.
 type InputProvider interface {
+	// Count returns the number of items for input provider
+	Count() int64
+	// Scan calls a callback function till the input provider is exhausted
+	Scan(callback func(value string))
 }
 
 // New returns a new Engine instance
diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go
index 112be4e2c..3593123a2 100644
--- a/v2/pkg/core/execute.go
+++ b/v2/pkg/core/execute.go
@@ -3,9 +3,12 @@ package core
 import (
 	"fmt"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/remeh/sizedwaitgroup"
 	"github.com/rs/xid"
+	"go.uber.org/atomic"
 )
 
 // Execute takes a list of templates/workflows that have been compiled
@@ -13,27 +16,86 @@ import (
 //
 // All the execution logic for the templates/workflows happens in this part
 // of the engine.
-func (e *Engine) Execute(templates []*templates.Template) {
-	finalTemplates, clusterCount := e.clusterTemplates(templates)
+func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool {
+	return e.ExecuteWithOpts(templates, input, false)
+}
 
+// ExecuteWithOpts is execute with the full options
+func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool {
+	var finalTemplates []*templates.Template
+	if !noCluster {
+		finalTemplates, _ = e.ClusterTemplates(templatesList)
+	} else {
+		finalTemplates = templatesList
+	}
+
+	results := &atomic.Bool{}
 	for _, template := range finalTemplates {
 		templateType := template.Type()
 
+		var wg *sizedwaitgroup.SizedWaitGroup
+		if templateType == "headless" {
+			wg = e.workPool.Headless
+		} else {
+			wg = e.workPool.Default
+		}
+
+		wg.Add()
 		switch {
 		case template.SelfContained:
-			// Self Contained requests are executed here
-
-		case templateType == "workflow":
-			// Workflows requests are executed here
-
+			// Self Contained requests are executed here separately
+			e.executeSelfContainedTemplateWithInput(template, results)
 		default:
 			// All other request types are executed here
+			e.executeModelWithInput(templateType, template, input, results)
 		}
 	}
+	e.workPool.Wait()
+	return results
 }
 
-// clusterTemplates performs identical http requests clustering for a list of templates
-func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) {
+// processSelfContainedTemplates execute a self-contained template.
+func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) {
+	match, err := template.Executer.Execute("")
+	if err != nil {
+		gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
+	}
+	results.CAS(false, match)
+}
+
+// executeModelWithInput executes a type of template with input
+func (e *Engine) executeModelWithInput(templateType string, template *templates.Template, input InputProvider, results *atomic.Bool) {
+	wg := e.workPool.InputPool(templateType)
+
+	input.Scan(func(scannedValue string) {
+		// Skip if the host has had errors
+		if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) {
+			return
+		}
+
+		wg.Waitgroup.Add()
+		go func(value string) {
+			defer wg.Waitgroup.Done()
+
+			var match bool
+			var err error
+			switch templateType {
+			case "workflow":
+				match = e.executeWorkflow(value, template.CompiledWorkflow)
+			default:
+				match, err = template.Executer.Execute(value)
+			}
+			if err != nil {
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
+			}
+			results.CAS(false, match)
+		}(scannedValue)
+	})
+	wg.Waitgroup.Wait()
+}
+
+// ClusterTemplates performs identical http requests clustering for a list of templates
+func (e *Engine) ClusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) {
 	if e.options.OfflineHTTP {
 		return templatesList, 0
 	}
@@ -65,85 +127,3 @@ func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templ
 	}
 	return finalTemplatesList, clusterCount
 }
-
-/*
-import (
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/remeh/sizedwaitgroup"
-	"go.uber.org/atomic"
-)
-
-// processSelfContainedTemplates execute a self-contained template.
-func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool {
-	match, err := template.Executer.Execute("")
-	if err != nil {
-		gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
-	}
-	return match
-}
-
-// processTemplateWithList execute a template against the list of user provided targets
-func (r *Runner) processTemplateWithList(template *templates.Template) bool {
-	results := &atomic.Bool{}
-	wg := sizedwaitgroup.New(r.options.BulkSize)
-	processItem := func(k, _ []byte) error {
-		URL := string(k)
-
-		// Skip if the host has had errors
-		if r.hostErrors != nil && r.hostErrors.Check(URL) {
-			return nil
-		}
-		wg.Add()
-		go func(URL string) {
-			defer wg.Done()
-
-			match, err := template.Executer.Execute(URL)
-			if err != nil {
-				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
-			}
-			results.CAS(false, match)
-		}(URL)
-		return nil
-	}
-	if r.options.Stream {
-		_ = r.hostMapStream.Scan(processItem)
-	} else {
-		r.hostMap.Scan(processItem)
-	}
-
-	wg.Wait()
-	return results.Load()
-}
-
-// processTemplateWithList process a template on the URL list
-func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
-	results := &atomic.Bool{}
-	wg := sizedwaitgroup.New(r.options.BulkSize)
-
-	processItem := func(k, _ []byte) error {
-		URL := string(k)
-
-		// Skip if the host has had errors
-		if r.hostErrors != nil && r.hostErrors.Check(URL) {
-			return nil
-		}
-		wg.Add()
-		go func(URL string) {
-			defer wg.Done()
-			match := template.CompiledWorkflow.RunWorkflow(URL)
-			results.CAS(false, match)
-		}(URL)
-		return nil
-	}
-
-	if r.options.Stream {
-		_ = r.hostMapStream.Scan(processItem)
-	} else {
-		r.hostMap.Scan(processItem)
-	}
-
-	wg.Wait()
-	return results.Load()
-}
-*/
diff --git a/v2/pkg/core/inputs/hmap/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go
similarity index 88%
rename from v2/pkg/core/inputs/hmap/hmap.go
rename to v2/pkg/core/inputs/hybrid/hmap.go
index 2910d0451..1eb4f5e42 100644
--- a/v2/pkg/core/inputs/hmap/hmap.go
+++ b/v2/pkg/core/inputs/hybrid/hmap.go
@@ -1,6 +1,6 @@
-// Package hmap implements a hybrid hmap/filekv backed input provider
+// Package hybrid implements a hybrid hmap/filekv backed input provider
 // for nuclei that can either stream or store results using different kv stores.
-package hmap
+package hybrid
 
 import (
 	"bufio"
@@ -118,3 +118,16 @@ func (i *Input) normalizeStoreInputValue(value string) {
 func (i *Input) Count() int64 {
 	return i.inputCount
 }
+
+// Scan calls an input provider till the callback is exhausted
+func (i *Input) Scan(callback func(value string)) {
+	callbackFunc := func(k, _ []byte) error {
+		callback(string(k))
+		return nil
+	}
+	if i.hostMapStream != nil {
+		_ = i.hostMapStream.Scan(callbackFunc)
+	} else {
+		i.hostMap.Scan(callbackFunc)
+	}
+}
diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go
index 59a5ade55..8c255d7ad 100644
--- a/v2/pkg/core/workflow_execute.go
+++ b/v2/pkg/core/workflow_execute.go
@@ -1,15 +1,22 @@
 package core
 
-/*
-// RunWorkflow runs a workflow on an input and returns true or false
-func (w *Workflow) RunWorkflow(input string) bool {
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
+)
+
+// executeWorkflow runs a workflow on an input and returns true or false
+func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool {
 	results := &atomic.Bool{}
 
 	swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads)
 	for _, template := range w.Workflows {
 		swg.Add()
-		func(template *WorkflowTemplate) {
-			if err := w.runWorkflowStep(template, input, results, &swg); err != nil {
+		func(template *workflows.WorkflowTemplate) {
+			if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil {
 				gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
 			}
 			swg.Done()
@@ -21,7 +28,7 @@ func (w *Workflow) RunWorkflow(input string) bool {
 
 // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
 // in a recursive manner running all subtemplates and matchers.
-func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error {
+func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error {
 	var firstMatched bool
 	var err error
 	var mainErr error
@@ -84,8 +91,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 					for _, subtemplate := range matcher.Subtemplates {
 						swg.Add()
 
-						go func(subtemplate *WorkflowTemplate) {
-							if err := w.runWorkflowStep(subtemplate, input, results, swg); err != nil {
+						go func(subtemplate *workflows.WorkflowTemplate) {
+							if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil {
 								gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", subtemplate.Template, err)
 							}
 							swg.Done()
@@ -108,8 +115,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 		for _, subtemplate := range template.Subtemplates {
 			swg.Add()
 
-			go func(template *WorkflowTemplate) {
-				if err := w.runWorkflowStep(template, input, results, swg); err != nil {
+			go func(template *workflows.WorkflowTemplate) {
+				if err := e.runWorkflowStep(template, input, results, swg, w); err != nil {
 					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
 				}
 				swg.Done()
@@ -117,4 +124,4 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 		}
 	}
 	return mainErr
-}*/
+}
diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go
index 62cd40840..a00ce6043 100644
--- a/v2/pkg/core/workflow_execute_test.go
+++ b/v2/pkg/core/workflow_execute_test.go
@@ -1,6 +1,5 @@
 package core
 
-/*
 import (
 	"testing"
 
@@ -17,13 +16,14 @@ import (
 func TestWorkflowsSimple(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
-	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
 		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
 		}},
 	}}
 
-	matched := workflow.RunWorkflow("https://test.com")
+	engine := &Engine{}
+	matched := engine.executeWorkflow("https://test.com", workflow)
 	require.True(t, matched, "could not get correct match value")
 }
 
@@ -31,7 +31,7 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
 		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
@@ -44,7 +44,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 		}},
 	}}
 
-	matched := workflow.RunWorkflow("https://test.com")
+	engine := &Engine{}
+	matched := engine.executeWorkflow("https://test.com", workflow)
 	require.True(t, matched, "could not get correct match value")
 
 	require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@@ -55,7 +56,7 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
 		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
@@ -69,7 +70,8 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 		}}}},
 	}}
 
-	matched := workflow.RunWorkflow("https://test.com")
+	engine := &Engine{}
+	matched := engine.executeWorkflow("https://test.com", workflow)
 	require.True(t, matched, "could not get correct match value")
 
 	require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@@ -80,7 +82,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
 		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: false, executeHook: func(input string) {
 				firstInput = input
@@ -92,7 +94,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 		}}}},
 	}}
 
-	matched := workflow.RunWorkflow("https://test.com")
+	engine := &Engine{}
+	matched := engine.executeWorkflow("https://test.com", workflow)
 	require.False(t, matched, "could not get correct match value")
 
 	require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@@ -103,7 +106,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 	progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	var firstInput, secondInput string
-	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{
+	workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
 		{Executers: []*workflows.ProtocolExecuterPair{{
 			Executer: &mockExecuter{result: true, executeHook: func(input string) {
 				firstInput = input
@@ -120,7 +123,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 		}}}}}},
 	}}
 
-	matched := workflow.RunWorkflow("https://test.com")
+	engine := &Engine{}
+	matched := engine.executeWorkflow("https://test.com", workflow)
 	require.True(t, matched, "could not get correct match value")
 
 	require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@@ -148,7 +152,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 		}}}}}},
 	}}
 
-	matched := workflow.RunWorkflow("https://test.com")
+	engine := &Engine{}
+	matched := engine.executeWorkflow("https://test.com", workflow)
 	require.False(t, matched, "could not get correct match value")
 
 	require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@@ -189,4 +194,3 @@ func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.Outpu
 	}
 	return nil
 }
-*/
diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go
index ef50f70fc..23b50d682 100644
--- a/v2/pkg/core/workpool.go
+++ b/v2/pkg/core/workpool.go
@@ -1,7 +1,7 @@
 package core
 
 import (
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 // WorkPool implements an execution pool for executing different
@@ -10,7 +10,9 @@ import (
 // It also allows Configuration of such requirements. This is used
 // for per-module like separate headless concurrency etc.
 type WorkPool struct {
-	config WorkPoolConfig
+	Headless *sizedwaitgroup.SizedWaitGroup
+	Default  *sizedwaitgroup.SizedWaitGroup
+	config   WorkPoolConfig
 }
 
 // WorkPoolConfig is the configuration for workpool
@@ -27,9 +29,35 @@ type WorkPoolConfig struct {
 
 // NewWorkPool returns a new WorkPool instance
 func NewWorkPool(config WorkPoolConfig) *WorkPool {
-	return &WorkPool{config: config}
+	headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency)
+	defaultWg := sizedwaitgroup.New(config.TypeConcurrency)
+
+	return &WorkPool{
+		config:   config,
+		Headless: &headlessWg,
+		Default:  &defaultWg,
+	}
 }
 
-func (w *WorkPool) Execute(templates []*templates.Template) {
-
+// Wait waits for all the workpool waitgroups to finish
+func (w *WorkPool) Wait() {
+	w.Default.Wait()
+	w.Headless.Wait()
+}
+
+// InputWorkPool is a workpool per-input
+type InputWorkPool struct {
+	Waitgroup *sizedwaitgroup.SizedWaitGroup
+}
+
+// InputPool returns a workpool for an input type
+func (w *WorkPool) InputPool(templateType string) *InputWorkPool {
+	var count int
+	if templateType == "headless" {
+		count = w.config.HeadlessInputConcurrency
+	} else {
+		count = w.config.InputConcurrency
+	}
+	swg := sizedwaitgroup.New(count)
+	return &InputWorkPool{Waitgroup: &swg}
 }
diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go
index 83a76f7cf..d877328fe 100644
--- a/v2/pkg/protocols/protocols.go
+++ b/v2/pkg/protocols/protocols.go
@@ -3,6 +3,7 @@ package protocols
 import (
 	"go.uber.org/ratelimit"
 
+	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
@@ -61,6 +62,7 @@ type ExecuterOptions struct {
 
 	Operators []*operators.Operators // only used by offlinehttp module
 
+	Colorizer      aurora.Aurora
 	WorkflowLoader model.WorkflowLoader
 }
 

From 0abc7202b1c4cd12da364100a2309ef8a9ca2ae1 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Thu, 28 Oct 2021 17:45:38 +0530
Subject: [PATCH 7/9] Misc fixes with goflags

---
 v2/go.mod                    | 73 +++++++++++++++++-------------
 v2/go.sum                    | 86 +++++++++++++++++++++---------------
 v2/internal/runner/runner.go |  2 +-
 v2/pkg/core/execute.go       |  1 +
 4 files changed, 96 insertions(+), 66 deletions(-)

diff --git a/v2/go.mod b/v2/go.mod
index e5e74498f..bbce9b151 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -5,93 +5,99 @@ go 1.17
 require (
 	github.com/Ice3man543/nvd v1.0.8
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
-	github.com/akrylysov/pogreb v0.10.1 // indirect
-	github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c
+	github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
 	github.com/andygrunwald/go-jira v1.14.0
-	github.com/antchfx/htmlquery v1.2.3
+	github.com/antchfx/htmlquery v1.2.4
 	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/bluele/gcache v0.0.2
-	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/corpix/uarand v0.1.1
-	github.com/go-rod/rod v0.101.7
+	github.com/go-rod/rod v0.101.8
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/gosuri/uilive v0.0.4 // indirect
-	github.com/gosuri/uiprogress v0.0.1 // indirect
-	github.com/itchyny/gojq v0.12.4
+	github.com/itchyny/gojq v0.12.5
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/miekg/dns v1.1.43
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/owenrumney/go-sarif v1.0.11
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e
+	github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542
 	github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
 	github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5
-	github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240
+	github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a
 	github.com/projectdiscovery/gologger v1.1.4
-	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
+	github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f
 	github.com/projectdiscovery/interactsh v0.0.6
-	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77
+	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df
 	github.com/projectdiscovery/rawhttp v0.0.7
-	github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a
+	github.com/projectdiscovery/retryabledns v1.0.12
 	github.com/projectdiscovery/retryablehttp-go v1.0.2
-	github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d
+	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
 	github.com/projectdiscovery/yamldoc-go v1.0.2
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil/v3 v3.21.7
+	github.com/shirou/gopsutil/v3 v3.21.9
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.4.1
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
 	github.com/valyala/fasttemplate v1.2.1
-	github.com/xanzy/go-gitlab v0.50.3
-	github.com/ysmood/gson v0.6.4 // indirect
-	github.com/ysmood/leakless v0.7.0 // indirect
+	github.com/xanzy/go-gitlab v0.51.1
 	go.uber.org/atomic v1.9.0
 	go.uber.org/multierr v1.7.0
 	go.uber.org/ratelimit v0.2.0
-	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
-	golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab
-	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect
+	golang.org/x/net v0.0.0-20211020060615-d418f374d309
+	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
 	golang.org/x/text v0.3.7
-	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	moul.io/http2curl v1.0.0
 )
 
 require (
 	git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
+	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
+	github.com/DataDog/zstd v1.4.8 // indirect
 	github.com/PuerkitoBio/goquery v1.6.0 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
+	github.com/akrylysov/pogreb v0.10.1 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
-	github.com/antchfx/xpath v1.1.6 // indirect
-	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
 	github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect
+	github.com/c4milo/unpackit v0.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
+	github.com/cockroachdb/errors v1.8.6 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgraph-io/badger v1.6.2 // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/eggsampler/acme/v3 v3.2.1 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/gosuri/uilive v0.0.4 // indirect
+	github.com/gosuri/uiprogress v0.0.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
@@ -100,8 +106,10 @@ require (
 	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -111,17 +119,22 @@ require (
 	github.com/projectdiscovery/mapcidr v0.0.8 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
-	github.com/tklauser/go-sysconf v0.3.7 // indirect
-	github.com/tklauser/numcpus v0.2.3 // indirect
+	github.com/rogpeppe/go-internal v1.8.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.9 // indirect
+	github.com/tklauser/numcpus v0.3.0 // indirect
 	github.com/trivago/tgo v1.0.7 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
 	github.com/ysmood/goob v0.3.0 // indirect
+	github.com/ysmood/gson v0.6.4 // indirect
+	github.com/ysmood/leakless v0.7.0 // indirect
 	github.com/zclconf/go-cty v1.8.4 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/exp v0.0.0-20210826195003-46c773283d9d // indirect
+	golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
diff --git a/v2/go.sum b/v2/go.sum
index b95089c7e..6b184d9f4 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -35,12 +35,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7
 git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
 git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
 github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY=
 github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c=
 github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc=
@@ -67,8 +69,9 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY
 github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
 github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
-github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY=
 github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
+github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA=
+github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -79,10 +82,12 @@ github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5z
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI=
 github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE=
-github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
 github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
-github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
+github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
+github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
@@ -96,14 +101,11 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
-github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
-github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
-github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -124,8 +126,10 @@ github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -140,13 +144,18 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
 github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
 github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
+github.com/cockroachdb/errors v1.8.6 h1:Am9evxl/po3RzpokemQvq7S7Cd0mxv24xy0B/trlQF4=
 github.com/cockroachdb/errors v1.8.6/go.mod h1:hOm5fabihW+xEyY1kuypGwqT+Vt7rafg04ytBtIpeIQ=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs=
+github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb h1:ZkYzWJexCQZyR1ejLy1IHvgdXTMe3Hiyvwlkyk5xr5Y=
 github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs=
 github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.1/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
@@ -172,12 +181,15 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
 github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
 github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
 github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
@@ -185,6 +197,7 @@ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@@ -216,6 +229,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -232,8 +246,8 @@ github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
-github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA=
-github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY=
+github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU=
+github.com/go-rod/rod v0.101.8/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@@ -247,11 +261,13 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
 github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -328,8 +344,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
-github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@@ -385,8 +399,9 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/
 github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
 github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
-github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
 github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
+github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0=
+github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE=
 github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
 github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
@@ -479,8 +494,6 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
-github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY=
-github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -555,6 +568,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -572,9 +586,8 @@ github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydK
 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U=
 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc=
 github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542 h1:wvHtITg+Y2eMVdN/4OR8wx7747RFTZX/UcxxJOE35F4=
 github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ=
-github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
-github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew=
 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
 github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
@@ -584,16 +597,16 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h
 github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs=
 github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
+github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk=
+github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
 github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
 github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
+github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f h1:lLBOzyKoIZTUoMIi3hxa8fj6EtK0EozlftfR4A8Mc4w=
 github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54=
-github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
-github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0=
-github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo=
 github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
 github.com/projectdiscovery/interactsh v0.0.6 h1:z2nVuc2FDRtOVjoYySiG0by/JA1Dwmgzb2rBFJEFR/c=
 github.com/projectdiscovery/interactsh v0.0.6/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U=
@@ -609,23 +622,21 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo
 github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
 github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I=
 github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
-github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ=
-github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
 github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
 github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/retryabledns v1.0.12 h1:OzCsUaipN75OwjtH62FxBIhKye1NmnfG4DxtQclOtns=
 github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
-github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE=
-github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U=
 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
 github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
-github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw=
-github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4=
 github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ=
 github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -664,8 +675,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -674,8 +683,9 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU=
 github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
+github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
+github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -726,10 +736,12 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
-github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg=
 github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4=
-github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8=
+github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
+github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E=
+github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
+github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
 github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
@@ -752,8 +764,9 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
-github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY=
 github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
+github.com/xanzy/go-gitlab v0.51.1 h1:wWKLalwx4omxFoHh3PLs9zDgAD4GXDP/uoxwMRCSiWM=
+github.com/xanzy/go-gitlab v0.51.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
@@ -846,6 +859,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
+golang.org/x/exp v0.0.0-20210826195003-46c773283d9d h1:0jjV4zcNl3aKvtqwdWq0orBLGpQB9TZ9qH45oQhI5MM=
 golang.org/x/exp v0.0.0-20210826195003-46c773283d9d/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -922,16 +936,17 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA=
 golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE=
+golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1006,9 +1021,10 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
-golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index 84f2837a8..4f8ef9753 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -18,7 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v2/pkg/core"
-	"github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hybrid"
+	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go
index 3593123a2..883707875 100644
--- a/v2/pkg/core/execute.go
+++ b/v2/pkg/core/execute.go
@@ -49,6 +49,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input Inpu
 			// All other request types are executed here
 			e.executeModelWithInput(templateType, template, input, results)
 		}
+		wg.Done()
 	}
 	e.workPool.Wait()
 	return results

From 1ca2cf3beace4bb5b876b50e8b13a4289c7091f9 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Thu, 28 Oct 2021 23:17:05 +0530
Subject: [PATCH 8/9] Misc

---
 v2/internal/runner/runner.go    | 118 +++++++++++++++-----------------
 v2/pkg/catalog/loader/loader.go |  21 ++++++
 2 files changed, 75 insertions(+), 64 deletions(-)

diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index 4f8ef9753..d27713a35 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -267,22 +267,7 @@ func (r *Runner) RunEnumeration() error {
 	}
 	executerOpts.WorkflowLoader = workflowLoader
 
-	loaderConfig := loader.Config{
-		Templates:          r.options.Templates,
-		Workflows:          r.options.Workflows,
-		ExcludeTemplates:   r.options.ExcludedTemplates,
-		Tags:               r.options.Tags,
-		ExcludeTags:        r.options.ExcludeTags,
-		IncludeTemplates:   r.options.IncludeTemplates,
-		Authors:            r.options.Author,
-		Severities:         r.options.Severities,
-		ExcludeSeverities:  r.options.ExcludeSeverities,
-		IncludeTags:        r.options.IncludeTags,
-		TemplatesDirectory: r.options.TemplatesDirectory,
-		Catalog:            r.catalog,
-		ExecutorOptions:    executerOpts,
-	}
-	store, err := loader.New(&loaderConfig)
+	store, err := loader.New(loader.NewConfig(r.options, r.catalog, executerOpts))
 	if err != nil {
 		return errors.Wrap(err, "could not load templates from config")
 	}
@@ -300,54 +285,7 @@ func (r *Runner) RunEnumeration() error {
 		return nil // exit
 	}
 
-	// Display stats for any loaded templates' syntax warnings or errors
-	stats.Display(parsers.SyntaxWarningStats)
-	stats.Display(parsers.SyntaxErrorStats)
-
-	builder := &strings.Builder{}
-	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
-		builder.WriteString(" (")
-
-		if strings.Contains(config.Version, "-dev") {
-			builder.WriteString(r.colorizer.Blue("development").String())
-		} else if config.Version == r.templatesConfig.NucleiLatestVersion {
-			builder.WriteString(r.colorizer.Green("latest").String())
-		} else {
-			builder.WriteString(r.colorizer.Red("outdated").String())
-		}
-		builder.WriteString(")")
-	}
-	messageStr := builder.String()
-	builder.Reset()
-
-	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
-
-	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
-		builder.WriteString(" (")
-
-		if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
-			builder.WriteString(r.colorizer.Green("latest").String())
-		} else {
-			builder.WriteString(r.colorizer.Red("outdated").String())
-		}
-		builder.WriteString(")")
-	}
-	messageStr = builder.String()
-	builder.Reset()
-
-	if r.templatesConfig != nil {
-		gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
-	}
-	if r.interactsh != nil {
-		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
-	}
-	if len(store.Templates()) > 0 {
-		gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates())
-		gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates()))
-	}
-	if len(store.Workflows()) > 0 {
-		gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
-	}
+	r.displayExecutionInfo(store)
 
 	var unclusteredRequests int64
 	for _, template := range store.Templates() {
@@ -417,6 +355,58 @@ func (r *Runner) RunEnumeration() error {
 	return nil
 }
 
+// displayExecutionInfo displays misc info about the nuclei engine execution
+func (r *Runner) displayExecutionInfo(store *loader.Store) {
+	// Display stats for any loaded templates' syntax warnings or errors
+	stats.Display(parsers.SyntaxWarningStats)
+	stats.Display(parsers.SyntaxErrorStats)
+
+	builder := &strings.Builder{}
+	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if strings.Contains(config.Version, "-dev") {
+			builder.WriteString(r.colorizer.Blue("development").String())
+		} else if config.Version == r.templatesConfig.NucleiLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr := builder.String()
+	builder.Reset()
+
+	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
+
+	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
+		builder.WriteString(" (")
+
+		if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr = builder.String()
+	builder.Reset()
+
+	if r.templatesConfig != nil {
+		gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
+	}
+	if r.interactsh != nil {
+		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
+	}
+	if len(store.Templates()) > 0 {
+		gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates())
+		gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates()))
+	}
+	if len(store.Workflows()) > 0 {
+		gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
+	}
+}
+
 // readNewTemplatesFile reads newly added templates from directory if it exists
 func (r *Runner) readNewTemplatesFile() ([]string, error) {
 	if r.templatesConfig == nil {
diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go
index eac22b4ae..386898b12 100644
--- a/v2/pkg/catalog/loader/loader.go
+++ b/v2/pkg/catalog/loader/loader.go
@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
 // Config contains the configuration options for the loader
@@ -44,6 +45,26 @@ type Store struct {
 	preprocessor templates.Preprocessor
 }
 
+// NewConfig returns a new loader config
+func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
+	loaderConfig := Config{
+		Templates:          options.Templates,
+		Workflows:          options.Workflows,
+		ExcludeTemplates:   options.ExcludedTemplates,
+		Tags:               options.Tags,
+		ExcludeTags:        options.ExcludeTags,
+		IncludeTemplates:   options.IncludeTemplates,
+		Authors:            options.Author,
+		Severities:         options.Severities,
+		ExcludeSeverities:  options.ExcludeSeverities,
+		IncludeTags:        options.IncludeTags,
+		TemplatesDirectory: options.TemplatesDirectory,
+		Catalog:            catalog,
+		ExecutorOptions:    executerOpts,
+	}
+	return &loaderConfig
+}
+
 // New creates a new template store based on provided configuration
 func New(config *Config) (*Store, error) {
 	// Create a tag filter based on provided configuration

From 5393cc4cd509c95eda6dc7e543f5384feec09268 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Fri, 29 Oct 2021 03:19:43 +0530
Subject: [PATCH 9/9] Adjusting packages for more API-type design

---
 v2/internal/testutils/testutils.go            | 38 +------------------
 v2/pkg/core/inputs/inputs.go                  | 17 +++++++++
 v2/pkg/output/output.go                       | 35 +++++++++++++++++
 v2/pkg/progress/progress.go                   | 24 ++++++++++++
 .../protocols/common/interactsh/interactsh.go | 14 +++++++
 v2/pkg/types/types.go                         | 14 +++++++
 6 files changed, 105 insertions(+), 37 deletions(-)
 create mode 100644 v2/pkg/core/inputs/inputs.go

diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go
index d62b1d630..e58c70791 100644
--- a/v2/internal/testutils/testutils.go
+++ b/v2/internal/testutils/testutils.go
@@ -1,7 +1,6 @@
 package testutils
 
 import (
-	"github.com/logrusorgru/aurora"
 	"go.uber.org/ratelimit"
 
 	"github.com/projectdiscovery/gologger/levels"
@@ -60,41 +59,6 @@ var DefaultOptions = &types.Options{
 	CustomHeaders:      []string{},
 }
 
-// MockOutputWriter is a mocked output writer.
-type MockOutputWriter struct {
-	aurora          aurora.Aurora
-	RequestCallback func(templateID, url, requestType string, err error)
-	WriteCallback   func(o *output.ResultEvent)
-}
-
-// NewMockOutputWriter creates a new mock output writer
-func NewMockOutputWriter() *MockOutputWriter {
-	return &MockOutputWriter{aurora: aurora.NewAurora(false)}
-}
-
-// Close closes the output writer interface
-func (m *MockOutputWriter) Close() {}
-
-// Colorizer returns the colorizer instance for writer
-func (m *MockOutputWriter) Colorizer() aurora.Aurora {
-	return m.aurora
-}
-
-// Write writes the event to file and/or screen.
-func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
-	if m.WriteCallback != nil {
-		m.WriteCallback(result)
-	}
-	return nil
-}
-
-// Request writes a log the requests trace log
-func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
-	if m.RequestCallback != nil {
-		m.RequestCallback(templateID, url, requestType, err)
-	}
-}
-
 // TemplateInfo contains info for a mock executed template.
 type TemplateInfo struct {
 	ID   string
@@ -109,7 +73,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
 		TemplateID:   info.ID,
 		TemplateInfo: info.Info,
 		TemplatePath: info.Path,
-		Output:       NewMockOutputWriter(),
+		Output:       output.NewMockOutputWriter(),
 		Options:      options,
 		Progress:     progressImpl,
 		ProjectFile:  nil,
diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go
new file mode 100644
index 000000000..6237dfb99
--- /dev/null
+++ b/v2/pkg/core/inputs/inputs.go
@@ -0,0 +1,17 @@
+package inputs
+
+type SimpleInputProvider struct {
+	Inputs []string
+}
+
+// Count returns the number of items for input provider
+func (s *SimpleInputProvider) Count() int64 {
+	return int64(len(s.Inputs))
+}
+
+// Scan calls a callback function till the input provider is exhausted
+func (s *SimpleInputProvider) Scan(callback func(value string)) {
+	for _, v := range s.Inputs {
+		callback(v)
+	}
+}
diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go
index b90be29c3..26555719d 100644
--- a/v2/pkg/output/output.go
+++ b/v2/pkg/output/output.go
@@ -209,3 +209,38 @@ func (w *StandardWriter) Close() {
 		w.traceFile.Close()
 	}
 }
+
+// MockOutputWriter is a mocked output writer.
+type MockOutputWriter struct {
+	aurora          aurora.Aurora
+	RequestCallback func(templateID, url, requestType string, err error)
+	WriteCallback   func(o *ResultEvent)
+}
+
+// NewMockOutputWriter creates a new mock output writer
+func NewMockOutputWriter() *MockOutputWriter {
+	return &MockOutputWriter{aurora: aurora.NewAurora(false)}
+}
+
+// Close closes the output writer interface
+func (m *MockOutputWriter) Close() {}
+
+// Colorizer returns the colorizer instance for writer
+func (m *MockOutputWriter) Colorizer() aurora.Aurora {
+	return m.aurora
+}
+
+// Write writes the event to file and/or screen.
+func (m *MockOutputWriter) Write(result *ResultEvent) error {
+	if m.WriteCallback != nil {
+		m.WriteCallback(result)
+	}
+	return nil
+}
+
+// Request writes a log the requests trace log
+func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
+	if m.RequestCallback != nil {
+		m.RequestCallback(templateID, url, requestType, err)
+	}
+}
diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go
index 220ff49a6..fcd34340d 100644
--- a/v2/pkg/progress/progress.go
+++ b/v2/pkg/progress/progress.go
@@ -247,3 +247,27 @@ func (p *StatsTicker) Stop() {
 		_ = p.server.Shutdown(context.Background())
 	}
 }
+
+type MockProgressClient struct{}
+
+// Stop stops the progress recorder.
+func (m *MockProgressClient) Stop() {}
+
+// Init inits the progress bar with initial details for scan
+func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {}
+
+// AddToTotal adds a value to the total request count
+func (m *MockProgressClient) AddToTotal(delta int64) {}
+
+// IncrementRequests increments the requests counter by 1.
+func (m *MockProgressClient) IncrementRequests() {}
+
+// IncrementMatched increments the matched counter by 1.
+func (m *MockProgressClient) IncrementMatched() {}
+
+// IncrementErrorsBy increments the error counter by count.
+func (m *MockProgressClient) IncrementErrorsBy(count int64) {}
+
+// IncrementFailedRequestsBy increments the number of requests counter by count
+// along with errors.
+func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {}
diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go
index 5c2914ff5..0ea42d460 100644
--- a/v2/pkg/protocols/common/interactsh/interactsh.go
+++ b/v2/pkg/protocols/common/interactsh/interactsh.go
@@ -103,6 +103,20 @@ func New(options *Options) (*Client, error) {
 	return interactClient, nil
 }
 
+// NewDefaultOptions returns the default options for interactsh client
+func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options {
+	return &Options{
+		ServerURL:      "https://interactsh.com",
+		CacheSize:      5000,
+		Eviction:       60 * time.Second,
+		ColldownPeriod: 5 * time.Second,
+		PollDuration:   5 * time.Second,
+		Output:         output,
+		IssuesClient:   reporting,
+		Progress:       progress,
+	}
+}
+
 func (c *Client) firstTimeInitializeClient() error {
 	interactsh, err := client.New(&client.Options{
 		ServerURL:         c.options.ServerURL,
diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
index 9f8f784b1..7f5a1851b 100644
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@@ -181,3 +181,17 @@ func (options *Options) AddVarPayload(key string, value interface{}) {
 func (options *Options) VarsPayload() map[string]interface{} {
 	return options.varsPayload
 }
+
+// DefaultOptions returns default options for nuclei
+func DefaultOptions() *Options {
+	return &Options{
+		RateLimit:               150,
+		BulkSize:                25,
+		TemplateThreads:         25,
+		HeadlessBulkSize:        10,
+		HeadlessTemplateThreads: 10,
+		Timeout:                 5,
+		Retries:                 1,
+		MaxHostError:            30,
+	}
+}