diff --git a/v2/go.sum b/v2/go.sum index 4fa384923..f96c0e9ee 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -47,8 +47,6 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= -github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.33 h1:8KUVEKrUw2dmu1Ys0aWnkEJgoRaLAzNysfCh2KSMWiI= github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -73,8 +71,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/httpx v1.0.2 h1:g7EeRAPckZgWcHkcAH2Qzv9MkRACVRLF+T2LJcM7SCk= github.com/projectdiscovery/httpx v1.0.2/go.mod h1:OwvMc5ogx69xukKXY6kIrDP6dgOYr4VtEWyr6o573Xs= -github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1 h1:I3aE8ta92M2XbrYKNYOTlXhodxyH+zQOt1jIatorhQA= -github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto= github.com/projectdiscovery/rawhttp v0.0.3/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= @@ -112,8 +108,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 811134bc1..0b8db0362 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -40,7 +40,9 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests) BulkSize int // Number of targets analyzed in parallel for each template - ProjectFile bool // Nuclei uses a project-file to avoid sending same HTTP request multiple times + TemplateThreads int // Number of templates executed in parallel + Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times + ProjectPath string // Nuclei uses a user defined project folder } type multiStringFlag []string @@ -82,8 +84,10 @@ func ParseOptions() *Options { flag.BoolVar(&options.TemplateList, "tl", false, "List available templates") flag.IntVar(&options.RateLimit, "rate-limit", -1, "Per Target Rate-Limit") flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)") - flag.IntVar(&options.BulkSize, "bulk-size", 150, "Number of hosts analyzed in parallel per template") - flag.BoolVar(&options.ProjectFile, "project-file", false, "Use a project file to avoid sending same request multiple times") + flag.IntVar(&options.BulkSize, "bulk-size", 25, "Number of hosts analyzed in parallel per template") + flag.IntVar(&options.TemplateThreads, "c", 10, "Number of templates executed in parallel") + flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times") + flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled") flag.Parse() diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c9418128d..fd780e621 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -8,9 +8,9 @@ import ( "os" "regexp" "strings" - "sync" "github.com/logrusorgru/aurora" + "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" @@ -167,10 +167,12 @@ func New(options *Options) (*Runner, error) { // Creates the progress tracking object runner.progress = progress.NewProgress(runner.colorizer.Colorizer, options.EnableProgressBar) - // create project file if requested - if options.ProjectFile { + // create project file if requested or load existing one + if options.Project { + hOptions := hybrid.DefaultDiskOptions + hOptions.Path = options.ProjectPath var err error - runner.hm, err = hybrid.New(hybrid.DefaultDiskOptions) + runner.hm, err = hybrid.New(hOptions) if err != nil { return runner, err } @@ -244,10 +246,8 @@ func (r *Runner) RunEnumeration() { } // nolint:wsl // comment } - var ( - wgtemplates sync.WaitGroup - results atomicboolean.AtomBool - ) + results := atomicboolean.New() + wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) if r.inputCount == 0 { gologger.Errorf("Could not find any valid input URLs.") @@ -257,7 +257,7 @@ func (r *Runner) RunEnumeration() { p.InitProgressbar(r.inputCount, templateCount, totalRequests) for _, t := range availableTemplates { - wgtemplates.Add(1) + wgtemplates.Add() go func(template interface{}) { defer wgtemplates.Done() switch tt := template.(type) { diff --git a/v2/pkg/atomicboolean/bool.go b/v2/pkg/atomicboolean/bool.go index 690b24f5d..829b78ec6 100644 --- a/v2/pkg/atomicboolean/bool.go +++ b/v2/pkg/atomicboolean/bool.go @@ -9,6 +9,10 @@ type AtomBool struct { flag bool } +func New() *AtomBool { + return &AtomBool{} +} + func (b *AtomBool) Or(value bool) { b.Lock() defer b.Unlock() diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 8370854d0..91d8d11dc 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net" "net/http" "net/http/cookiejar" @@ -367,13 +366,15 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // if nuclei-project is available check if the request was already sent previously if e.hm != nil { reqHash, err := hash(dumpedRequest) - log.Println(reqURL, reqHash) // if the computation was successful check within the cache if err == nil { data, ok := e.hm.Get(reqHash) // if found reuse the item if ok { - unmarshal(data, resp) + var httprecord HTTPRecord + httprecord.Response = newInternalResponse() + unmarshal(data, &httprecord) + resp = fromInternalResponse(httprecord.Response) fromcache = true } } @@ -430,8 +431,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, reqHash, err := hash(dumpedRequest) // if the computation was successful store within the cache if err == nil { + var httprecord HTTPRecord intResp := toInternalResponse(resp, data) - data, err := marshal(intResp) + httprecord.Request = dumpedRequest + httprecord.Response = intResp + data, err := marshal(httprecord) // once marshaled without errors store in the cache if err == nil { e.hm.Set(reqHash, data) diff --git a/v2/pkg/executer/http_utils.go b/v2/pkg/executer/http_utils.go index 8247a4b59..fc1692e22 100644 --- a/v2/pkg/executer/http_utils.go +++ b/v2/pkg/executer/http_utils.go @@ -90,6 +90,20 @@ func unmarshal(data []byte, obj interface{}) error { return nil } +type HTTPRecord struct { + Request []byte + Response *InternalResponse +} + +type InternalRequest struct { + Target string + HTTPMajor int + HTTPMinor int + Method string + Headers map[string][]string + Body []byte +} + type InternalResponse struct { HTTPMajor int HTTPMinor int @@ -99,12 +113,34 @@ type InternalResponse struct { Body []byte } +func newInternalRquest() *InternalRequest { + return &InternalRequest{ + Headers: make(map[string][]string), + } +} + func newInternalResponse() *InternalResponse { return &InternalResponse{ Headers: make(map[string][]string), } } +func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest { + intReq := newInternalRquest() + + intReq.Target = target + intReq.HTTPMajor = req.ProtoMajor + intReq.HTTPMinor = req.ProtoMinor + for k, v := range req.Header { + intReq.Headers[k] = v + } + intReq.Headers = req.Header + intReq.Method = req.Method + intReq.Body = body + + return intReq +} + func toInternalResponse(resp *http.Response, body []byte) *InternalResponse { intResp := newInternalResponse() @@ -120,13 +156,27 @@ func toInternalResponse(resp *http.Response, body []byte) *InternalResponse { } func fromInternalResponse(intResp *InternalResponse) *http.Response { + var contentLength int64 + if intResp.Body != nil { + contentLength = int64(len(intResp.Body)) + } return &http.Response{ ProtoMinor: intResp.HTTPMinor, ProtoMajor: intResp.HTTPMajor, Status: intResp.StatusReason, StatusCode: intResp.StatusCode, Header: intResp.Headers, - ContentLength: int64(len(intResp.Body)), + ContentLength: contentLength, Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)), } } + +func fromInternalRequest(intReq *InternalRequest) *http.Request { + return &http.Request{ + ProtoMinor: intReq.HTTPMinor, + ProtoMajor: intReq.HTTPMajor, + Header: intReq.Headers, + ContentLength: int64(len(intReq.Body)), + Body: ioutil.NopCloser(bytes.NewReader(intReq.Body)), + } +}