wip

v2/go.sum

@@ -47,8 +47,6 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
-github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.33 h1:8KUVEKrUw2dmu1Ys0aWnkEJgoRaLAzNysfCh2KSMWiI=
 github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
@@ -73,8 +71,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/httpx v1.0.2 h1:g7EeRAPckZgWcHkcAH2Qzv9MkRACVRLF+T2LJcM7SCk=
 github.com/projectdiscovery/httpx v1.0.2/go.mod h1:OwvMc5ogx69xukKXY6kIrDP6dgOYr4VtEWyr6o573Xs=
-github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1 h1:I3aE8ta92M2XbrYKNYOTlXhodxyH+zQOt1jIatorhQA=
-github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto=
 github.com/projectdiscovery/rawhttp v0.0.3/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
@@ -112,8 +108,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

v2/internal/runner/options.go

@@ -40,7 +40,9 @@ type Options struct {
 	Stdin              bool                   // Stdin specifies whether stdin input was given to the process
 	StopAtFirstMatch   bool                   // Stop processing template at first full match (this may break chained requests)
 	BulkSize           int                    // Number of targets analyzed in parallel for each template
-	ProjectFile        bool                   // Nuclei uses a project-file to avoid sending same HTTP request multiple times
+	TemplateThreads 	int // Number of templates executed in parallel
+	Project            bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
+	ProjectPath        string                 // Nuclei uses a user defined project folder
 }
 
 type multiStringFlag []string
@@ -82,8 +84,10 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
 	flag.IntVar(&options.RateLimit, "rate-limit", -1, "Per Target Rate-Limit")
 	flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
-	flag.IntVar(&options.BulkSize, "bulk-size", 150, "Number of hosts analyzed in parallel per template")
+	flag.IntVar(&options.BulkSize, "bulk-size", 25, "Number of hosts analyzed in parallel per template")
-	flag.BoolVar(&options.ProjectFile, "project-file", false, "Use a project file to avoid sending same request multiple times")
+	flag.IntVar(&options.TemplateThreads, "c", 10, "Number of templates executed in parallel")
+	flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
+	flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
 
 	flag.Parse()
 

v2/internal/runner/runner.go

@@ -8,9 +8,9 @@ import (
 	"os"
 	"regexp"
 	"strings"
-	"sync"
 
 	"github.com/logrusorgru/aurora"
+	"github.com/remeh/sizedwaitgroup"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
@@ -167,10 +167,12 @@ func New(options *Options) (*Runner, error) {
 	// Creates the progress tracking object
 	runner.progress = progress.NewProgress(runner.colorizer.Colorizer, options.EnableProgressBar)
 
-	// create project file if requested
+	// create project file if requested or load existing one
-	if options.ProjectFile {
+	if options.Project {
+		hOptions := hybrid.DefaultDiskOptions
+		hOptions.Path = options.ProjectPath
 		var err error
-		runner.hm, err = hybrid.New(hybrid.DefaultDiskOptions)
+		runner.hm, err = hybrid.New(hOptions)
 		if err != nil {
 			return runner, err
 		}
@@ -244,10 +246,8 @@ func (r *Runner) RunEnumeration() {
 		} // nolint:wsl // comment
 	}
 
-	var (
+	results := atomicboolean.New()
-		wgtemplates sync.WaitGroup
+	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
-		results     atomicboolean.AtomBool
-	)
 
 	if r.inputCount == 0 {
 		gologger.Errorf("Could not find any valid input URLs.")
@@ -257,7 +257,7 @@ func (r *Runner) RunEnumeration() {
 		p.InitProgressbar(r.inputCount, templateCount, totalRequests)
 
 		for _, t := range availableTemplates {
-			wgtemplates.Add(1)
+			wgtemplates.Add()
 			go func(template interface{}) {
 				defer wgtemplates.Done()
 				switch tt := template.(type) {

v2/pkg/atomicboolean/bool.go

@@ -9,6 +9,10 @@ type AtomBool struct {
 	flag bool
 }
 
+func New() *AtomBool {
+	return &AtomBool{}
+}
+
 func (b *AtomBool) Or(value bool) {
 	b.Lock()
 	defer b.Unlock()

v2/pkg/executer/executer_http.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
@@ -367,13 +366,15 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 		// if nuclei-project is available check if the request was already sent previously
 		if e.hm != nil {
 			reqHash, err := hash(dumpedRequest)
-			log.Println(reqURL, reqHash)
 			// if the computation was successful check within the cache
 			if err == nil {
 				data, ok := e.hm.Get(reqHash)
 				// if found reuse the item
 				if ok {
-					unmarshal(data, resp)
+					var httprecord HTTPRecord
+					httprecord.Response = newInternalResponse()
+					unmarshal(data, &httprecord)
+					resp = fromInternalResponse(httprecord.Response)
 					fromcache = true
 				}
 			}
@@ -430,8 +431,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 		reqHash, err := hash(dumpedRequest)
 		// if the computation was successful store within the cache
 		if err == nil {
+			var httprecord HTTPRecord
 			intResp := toInternalResponse(resp, data)
-			data, err := marshal(intResp)
+			httprecord.Request = dumpedRequest
+			httprecord.Response = intResp
+			data, err := marshal(httprecord)
 			// once marshaled without errors store in the cache
 			if err == nil {
 				e.hm.Set(reqHash, data)

v2/pkg/executer/http_utils.go

@@ -90,6 +90,20 @@ func unmarshal(data []byte, obj interface{}) error {
 	return nil
 }
 
+type HTTPRecord struct {
+	Request  []byte
+	Response *InternalResponse
+}
+
+type InternalRequest struct {
+	Target    string
+	HTTPMajor int
+	HTTPMinor int
+	Method    string
+	Headers   map[string][]string
+	Body      []byte
+}
+
 type InternalResponse struct {
 	HTTPMajor    int
 	HTTPMinor    int
@@ -99,12 +113,34 @@ type InternalResponse struct {
 	Body         []byte
 }
 
+func newInternalRquest() *InternalRequest {
+	return &InternalRequest{
+		Headers: make(map[string][]string),
+	}
+}
+
 func newInternalResponse() *InternalResponse {
 	return &InternalResponse{
 		Headers: make(map[string][]string),
 	}
 }
 
+func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest {
+	intReq := newInternalRquest()
+
+	intReq.Target = target
+	intReq.HTTPMajor = req.ProtoMajor
+	intReq.HTTPMinor = req.ProtoMinor
+	for k, v := range req.Header {
+		intReq.Headers[k] = v
+	}
+	intReq.Headers = req.Header
+	intReq.Method = req.Method
+	intReq.Body = body
+
+	return intReq
+}
+
 func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
 	intResp := newInternalResponse()
 
@@ -120,13 +156,27 @@ func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
 }
 
 func fromInternalResponse(intResp *InternalResponse) *http.Response {
+	var contentLength int64
+	if intResp.Body != nil {
+		contentLength = int64(len(intResp.Body))
+	}
 	return &http.Response{
 		ProtoMinor:    intResp.HTTPMinor,
 		ProtoMajor:    intResp.HTTPMajor,
 		Status:        intResp.StatusReason,
 		StatusCode:    intResp.StatusCode,
 		Header:        intResp.Headers,
-		ContentLength: int64(len(intResp.Body)),
+		ContentLength: contentLength,
 		Body:          ioutil.NopCloser(bytes.NewReader(intResp.Body)),
 	}
 }
+
+func fromInternalRequest(intReq *InternalRequest) *http.Request {
+	return &http.Request{
+		ProtoMinor:    intReq.HTTPMinor,
+		ProtoMajor:    intReq.HTTPMajor,
+		Header:        intReq.Headers,
+		ContentLength: int64(len(intReq.Body)),
+		Body:          ioutil.NopCloser(bytes.NewReader(intReq.Body)),
+	}
+}