External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 17:55:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mzack9999 2020-10-17 02:10:47 +02:00
parent 1fba3b0637
commit d5cd01d43b
6 changed files with 79 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
v2/go.sum
View File

@ -47,8 +47,6 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.33 h1:8KUVEKrUw2dmu1Ys0aWnkEJgoRaLAzNysfCh2KSMWiI=
github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
@ -73,8 +71,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/httpx v1.0.2 h1:g7EeRAPckZgWcHkcAH2Qzv9MkRACVRLF+T2LJcM7SCk=
github.com/projectdiscovery/httpx v1.0.2/go.mod h1:OwvMc5ogx69xukKXY6kIrDP6dgOYr4VtEWyr6o573Xs=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1 h1:I3aE8ta92M2XbrYKNYOTlXhodxyH+zQOt1jIatorhQA=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto=
github.com/projectdiscovery/rawhttp v0.0.3/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
@ -112,8 +108,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

10
v2/internal/runner/options.go
View File

@ -40,7 +40,9 @@ type Options struct {
Stdin bool // Stdin specifies whether stdin input was given to the process
StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests)
BulkSize int // Number of targets analyzed in parallel for each template
ProjectFile bool // Nuclei uses a project-file to avoid sending same HTTP request multiple times
TemplateThreads int // Number of templates executed in parallel
Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times
ProjectPath string // Nuclei uses a user defined project folder
}
type multiStringFlag []string
@ -82,8 +84,10 @@ func ParseOptions() *Options {
flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
flag.IntVar(&options.RateLimit, "rate-limit", -1, "Per Target Rate-Limit")
flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
flag.IntVar(&options.BulkSize, "bulk-size", 150, "Number of hosts analyzed in parallel per template")
flag.BoolVar(&options.ProjectFile, "project-file", false, "Use a project file to avoid sending same request multiple times")
flag.IntVar(&options.BulkSize, "bulk-size", 25, "Number of hosts analyzed in parallel per template")
flag.IntVar(&options.TemplateThreads, "c", 10, "Number of templates executed in parallel")
flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
flag.Parse()

18
v2/internal/runner/runner.go
View File

@ -8,9 +8,9 @@ import (
"os"
"regexp"
"strings"
"sync"
"github.com/logrusorgru/aurora"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
@ -167,10 +167,12 @@ func New(options *Options) (*Runner, error) {
// Creates the progress tracking object
runner.progress = progress.NewProgress(runner.colorizer.Colorizer, options.EnableProgressBar)
// create project file if requested
if options.ProjectFile {
// create project file if requested or load existing one
if options.Project {
hOptions := hybrid.DefaultDiskOptions
hOptions.Path = options.ProjectPath
var err error
runner.hm, err = hybrid.New(hybrid.DefaultDiskOptions)
runner.hm, err = hybrid.New(hOptions)
if err != nil {
return runner, err
}
@ -244,10 +246,8 @@ func (r *Runner) RunEnumeration() {
} // nolint:wsl // comment
}
var (
wgtemplates sync.WaitGroup
results atomicboolean.AtomBool
)
results := atomicboolean.New()
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
if r.inputCount == 0 {
gologger.Errorf("Could not find any valid input URLs.")
@ -257,7 +257,7 @@ func (r *Runner) RunEnumeration() {
p.InitProgressbar(r.inputCount, templateCount, totalRequests)
for _, t := range availableTemplates {
wgtemplates.Add(1)
wgtemplates.Add()
go func(template interface{}) {
defer wgtemplates.Done()
switch tt := template.(type) {

4
v2/pkg/atomicboolean/bool.go
View File

@ -9,6 +9,10 @@ type AtomBool struct {
flag bool
}
func New() *AtomBool {
return &AtomBool{}
}
func (b *AtomBool) Or(value bool) {
b.Lock()
defer b.Unlock()

12
v2/pkg/executer/executer_http.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"net/http/cookiejar"
@ -367,13 +366,15 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
// if nuclei-project is available check if the request was already sent previously
if e.hm != nil {
reqHash, err := hash(dumpedRequest)
log.Println(reqURL, reqHash)
// if the computation was successful check within the cache
if err == nil {
data, ok := e.hm.Get(reqHash)
// if found reuse the item
if ok {
unmarshal(data, resp)
var httprecord HTTPRecord
httprecord.Response = newInternalResponse()
unmarshal(data, &httprecord)
resp = fromInternalResponse(httprecord.Response)
fromcache = true
}
}
@ -430,8 +431,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
reqHash, err := hash(dumpedRequest)
// if the computation was successful store within the cache
if err == nil {
var httprecord HTTPRecord
intResp := toInternalResponse(resp, data)
data, err := marshal(intResp)
httprecord.Request = dumpedRequest
httprecord.Response = intResp
data, err := marshal(httprecord)
// once marshaled without errors store in the cache
if err == nil {
e.hm.Set(reqHash, data)

52
v2/pkg/executer/http_utils.go
View File

@ -90,6 +90,20 @@ func unmarshal(data []byte, obj interface{}) error {
return nil
}
type HTTPRecord struct {
Request []byte
Response *InternalResponse
}
type InternalRequest struct {
Target string
HTTPMajor int
HTTPMinor int
Method string
Headers map[string][]string
Body []byte
}
type InternalResponse struct {
HTTPMajor int
HTTPMinor int
@ -99,12 +113,34 @@ type InternalResponse struct {
Body []byte
}
func newInternalRquest() *InternalRequest {
return &InternalRequest{
Headers: make(map[string][]string),
}
}
func newInternalResponse() *InternalResponse {
return &InternalResponse{
Headers: make(map[string][]string),
}
}
func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest {
intReq := newInternalRquest()
intReq.Target = target
intReq.HTTPMajor = req.ProtoMajor
intReq.HTTPMinor = req.ProtoMinor
for k, v := range req.Header {
intReq.Headers[k] = v
}
intReq.Headers = req.Header
intReq.Method = req.Method
intReq.Body = body
return intReq
}
func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
intResp := newInternalResponse()
@ -120,13 +156,27 @@ func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
}
func fromInternalResponse(intResp *InternalResponse) *http.Response {
var contentLength int64
if intResp.Body != nil {
contentLength = int64(len(intResp.Body))
}
return &http.Response{
ProtoMinor: intResp.HTTPMinor,
ProtoMajor: intResp.HTTPMajor,
Status: intResp.StatusReason,
StatusCode: intResp.StatusCode,
Header: intResp.Headers,
ContentLength: int64(len(intResp.Body)),
ContentLength: contentLength,
Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)),
}
}
func fromInternalRequest(intReq *InternalRequest) *http.Request {
return &http.Request{
ProtoMinor: intReq.HTTPMinor,
ProtoMajor: intReq.HTTPMajor,
Header: intReq.Headers,
ContentLength: int64(len(intReq.Body)),
Body: ioutil.NopCloser(bytes.NewReader(intReq.Body)),
}
}