diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go deleted file mode 100644 index 408918d38..000000000 --- a/v2/pkg/executer/executer_http.go +++ /dev/null @@ -1,577 +0,0 @@ -package executer - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/cookiejar" - "net/http/httputil" - "net/url" - "os" - "regexp" - "strconv" - "strings" - "time" - - "github.com/corpix/uarand" - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/internal/tracelog" - "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/rawhttp" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/ratelimit" -) - -const ( - two = 2 - ten = 10 - defaultMaxWorkers = 150 - defaultMaxHistorydata = 150 -) - -// HTTPExecuter is client for performing HTTP requests -// for a template. -type HTTPExecuter struct { - pf *projetctfile.ProjectFile - customHeaders requests.CustomHeaders - colorizer colorizer.NucleiColorizer - httpClient *retryablehttp.Client - rawHTTPClient *rawhttp.Client - template *templates.Template - bulkHTTPRequest *requests.BulkHTTPRequest - writer *bufwriter.Writer - CookieJar *cookiejar.Jar - traceLog tracelog.Log - decolorizer *regexp.Regexp - randomAgent bool - vhost bool - coloredOutput bool - debug bool - Results bool - jsonOutput bool - jsonRequest bool - noMeta bool - stopAtFirstMatch bool - ratelimiter ratelimit.Limiter -} - -// HTTPOptions contains configuration options for the HTTP executer. -type HTTPOptions struct { - Template *templates.Template - BulkHTTPRequest *requests.BulkHTTPRequest - CookieJar *cookiejar.Jar - PF *projetctfile.ProjectFile -} - -// NewHTTPExecuter creates a new HTTP executer from a template -// and a HTTP request query. -func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { - var ( - proxyURL *url.URL - err error - ) - - if err != nil { - return nil, err - } - - // Create the HTTP Client - client := makeHTTPClient(proxyURL, options) - // nolint:bodyclose // false positive there is no body to close yet - - if options.CookieJar != nil { - client.HTTPClient.Jar = options.CookieJar - } else if options.CookieReuse { - jar, err := cookiejar.New(nil) - if err != nil { - return nil, err - } - client.HTTPClient.Jar = jar - } - - executer := &HTTPExecuter{ - debug: options.Debug, - jsonOutput: options.JSON, - jsonRequest: options.JSONRequests, - noMeta: options.NoMeta, - httpClient: client, - rawHTTPClient: rawClient, - traceLog: options.TraceLog, - template: options.Template, - bulkHTTPRequest: options.BulkHTTPRequest, - writer: options.Writer, - randomAgent: options.RandomAgent, - customHeaders: options.CustomHeaders, - CookieJar: options.CookieJar, - coloredOutput: options.ColoredOutput, - colorizer: *options.Colorizer, - decolorizer: options.Decolorizer, - stopAtFirstMatch: options.StopAtFirstMatch, - pf: options.PF, - vhost: options.Vhost, - ratelimiter: options.RateLimiter, - } - return executer, nil -} - -func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - e.bulkHTTPRequest.CreateGenerator(reqURL) - - // Workers that keeps enqueuing new requests - maxWorkers := e.bulkHTTPRequest.RaceNumberRequests - swg := sizedwaitgroup.New(maxWorkers) - for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ { - swg.Add() - // base request - result.Lock() - request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - } - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - - // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - result.Error = errors.Wrap(err, "could not handle http request") - } - }(request) - } - - swg.Wait() - - return result -} - -func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - remaining := e.bulkHTTPRequest.GetRequestCount() - e.bulkHTTPRequest.CreateGenerator(reqURL) - - // Workers that keeps enqueuing new requests - maxWorkers := e.bulkHTTPRequest.Threads - swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) { - result.Lock() - request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - p.Drop(remaining) - } else { - swg.Add() - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - - e.ratelimiter.Take() - - // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - e.traceLog.Request(e.template.ID, reqURL, "http", err) - result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - }(request) - } - p.Update() - e.bulkHTTPRequest.Increment(reqURL) - } - swg.Wait() - - return result -} - -func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - e.bulkHTTPRequest.CreateGenerator(reqURL) - - // need to extract the target from the url - URL, err := url.Parse(reqURL) - if err != nil { - return result - } - - // defaultMaxWorkers should be a sufficient value to keep queues always full - maxWorkers := defaultMaxWorkers - // in case the queue is bigger increase the workers - if pipeOptions.MaxPendingRequests > maxWorkers { - maxWorkers = pipeOptions.MaxPendingRequests - } - swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) { - result.Lock() - request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - } else { - swg.Add() - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - - // HTTP pipelining ignores rate limit - // If the request was built correctly then execute it - request.Pipeline = true - request.PipelineClient = pipeclient - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - e.traceLog.Request(e.template.ID, reqURL, "http", err) - result.Error = errors.Wrap(err, "could not handle http request") - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - request.PipelineClient = nil - }(request) - } - - e.bulkHTTPRequest.Increment(reqURL) - } - swg.Wait() - return result -} - -// ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result { - var customHost string - if e.vhost { - parts := strings.Split(reqURL, ",") - reqURL = parts[0] - customHost = parts[1] - } - - // verify if pipeline was requested - if e.bulkHTTPRequest.Pipeline { - return e.ExecuteTurboHTTP(reqURL) - } - - // verify if a basic race condition was requested - if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 { - return e.ExecuteRaceRequest(reqURL) - } - - // verify if parallel elaboration was requested - if e.bulkHTTPRequest.Threads > 0 { - return e.ExecuteParallelHTTP(p, reqURL) - } - - var requestNumber int - - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - historyData: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - remaining := e.bulkHTTPRequest.GetRequestCount() - e.bulkHTTPRequest.CreateGenerator(reqURL) - - for e.bulkHTTPRequest.Next(reqURL) { - requestNumber++ - result.Lock() - httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - p.Drop(remaining) - } else { - if e.vhost { - if httpRequest.Request != nil { - httpRequest.Request.Host = customHost - } - if httpRequest.RawRequest != nil && httpRequest.RawRequest.Headers != nil { - httpRequest.RawRequest.Headers["Host"] = customHost - } - } - - e.ratelimiter.Take() - // If the request was built correctly then execute it - format := "%s_" + strconv.Itoa(requestNumber) - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format) - if err != nil { - result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) - e.traceLog.Request(e.template.ID, reqURL, "http", err) - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - } - p.Update() - - // Check if has to stop processing at first valid result - if e.stopAtFirstMatch && result.GotResults { - p.Drop(remaining) - break - } - - // move always forward with requests - e.bulkHTTPRequest.Increment(reqURL) - remaining-- - } - gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL) - return result -} - -func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error { - // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if e.randomAgent { - // nolint:errcheck // ignoring error - e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) - } - - e.setCustomHeaders(request) - - var ( - resp *http.Response - err error - dumpedRequest []byte - fromcache bool - ) - - if e.debug || e.pf != nil { - dumpedRequest, err = requests.Dump(request, reqURL) - if err != nil { - return err - } - } - - if e.debug { - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) - } - - timeStart := time.Now() - - if request.Pipeline { - resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data))) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } else if request.Unsafe { - // rawhttp - // burp uses "\r\n" as new line character - request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n") - options := e.rawHTTPClient.Options - options.AutomaticContentLength = request.AutomaticContentLengthHeader - options.AutomaticHostHeader = request.AutomaticHostHeader - options.FollowRedirects = request.FollowRedirects - resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } else { - // if nuclei-project is available check if the request was already sent previously - if e.pf != nil { - // if unavailable fail silently - fromcache = true - // nolint:bodyclose // false positive the response is generated at runtime - resp, err = e.pf.Get(dumpedRequest) - if err != nil { - fromcache = false - } - } - - // retryablehttp - if resp == nil { - resp, err = e.httpClient.Do(request.Request) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - } - - duration := time.Since(timeStart) - - // Dump response - Step 1 - Decompression not yet handled - var dumpedResponse []byte - if e.debug { - var dumpErr error - dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) - if dumpErr != nil { - return errors.Wrap(dumpErr, "could not dump http response") - } - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - _, copyErr := io.Copy(ioutil.Discard, resp.Body) - if copyErr != nil { - resp.Body.Close() - return copyErr - } - - resp.Body.Close() - - return errors.Wrap(err, "could not read http body") - } - - resp.Body.Close() - - // net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request - // so in case we have to manually do it - dataOrig := data - data, err = requests.HandleDecompression(request, data) - if err != nil { - return errors.Wrap(err, "could not decompress http body") - } - - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if e.debug { - dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) - } - - // if nuclei-project is enabled store the response if not previously done - if e.pf != nil && !fromcache { - err := e.pf.Set(dumpedRequest, resp, data) - if err != nil { - return errors.Wrap(err, "could not store in project file") - } - } - - // Convert response body from []byte to string with zero copy - body := unsafeToString(data) - - headers := headersToString(resp.Header) - - var matchData map[string]interface{} - if payloads != nil { - matchData = generators.MergeMaps(result.historyData, payloads) - } - - // store for internal purposes the DSL matcher data - // hardcode stopping storing data after defaultMaxHistorydata items - if len(result.historyData) < defaultMaxHistorydata { - result.Lock() - // update history data with current reqURL and hostname - result.historyData["reqURL"] = reqURL - if parsed, err := url.Parse(reqURL); err == nil { - result.historyData["Hostname"] = parsed.Host - } - result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) - if payloads == nil { - // merge them to history data - result.historyData = generators.MergeMaps(result.historyData, payloads) - } - result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) - - // complement match data with new one if necessary - matchData = generators.MergeMaps(matchData, result.historyData) - result.Unlock() - } - - return nil -} - -// Close closes the http executer for a template. -func (e *HTTPExecuter) Close() {} - -func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { - for _, customHeader := range e.customHeaders { - // This should be pre-computed somewhere and done only once - tokens := strings.SplitN(customHeader, ":", two) - // if it's an invalid header skip it - if len(tokens) < two { - continue - } - - headerName, headerValue := tokens[0], strings.Join(tokens[1:], "") - if r.RawRequest != nil { - // rawhttp - r.RawRequest.Headers[headerName] = headerValue - } else { - // retryablehttp - headerName = strings.TrimSpace(headerName) - headerValue = strings.TrimSpace(headerValue) - r.Request.Header[headerName] = []string{headerValue} - } - } -} diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go deleted file mode 100644 index 7ce421136..000000000 --- a/v2/pkg/executer/output_http.go +++ /dev/null @@ -1,147 +0,0 @@ -package executer - -import ( - "fmt" - "net/http" - "net/http/httputil" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" -) - -// writeOutputHTTP writes http output to streams -func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) { - var URL string - if req.RawRequest != nil { - URL = req.RawRequest.FullURL - } - if req.Request != nil { - URL = req.Request.URL.String() - } - - if e.jsonOutput { - output := make(jsonOutput) - - output["matched"] = URL - if !e.noMeta { - output["template"] = e.template.ID - output["type"] = "http" - output["host"] = reqURL - if len(meta) > 0 { - output["meta"] = meta - } - for k, v := range e.template.Info { - output[k] = v - } - if matcher != nil && len(matcher.Name) > 0 { - output["matcher_name"] = matcher.Name - } - if len(extractorResults) > 0 { - output["extracted_results"] = extractorResults - } - - // TODO: URL should be an argument - if e.jsonRequest { - dumpedRequest, err := requests.Dump(req, URL) - if err != nil { - gologger.Warningf("could not dump request: %s\n", err) - } else { - output["request"] = string(dumpedRequest) - } - - dumpedResponse, err := httputil.DumpResponse(resp, false) - if err != nil { - gologger.Warningf("could not dump response: %s\n", err) - } else { - output["response"] = string(dumpedResponse) + body - } - } - } - - data, err := jsoniter.Marshal(output) - if err != nil { - gologger.Warningf("Could not marshal json output: %s\n", err) - } - gologger.Silentf("%s", string(data)) - - if e.writer != nil { - if err := e.writer.Write(data); err != nil { - gologger.Errorf("Could not write output data: %s\n", err) - return - } - } - return - } - - builder := &strings.Builder{} - colorizer := e.colorizer - - if !e.noMeta { - builder.WriteRune('[') - builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String()) - - if matcher != nil && len(matcher.Name) > 0 { - builder.WriteString(":") - builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String()) - } - - builder.WriteString("] [") - builder.WriteString(colorizer.Colorizer.BrightBlue("http").String()) - builder.WriteString("] ") - - if e.template.Info["severity"] != "" { - builder.WriteString("[") - builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"])) - builder.WriteString("] ") - } - } - builder.WriteString(URL) - - // If any extractors, write the results - if len(extractorResults) > 0 && !e.noMeta { - builder.WriteString(" [") - - for i, result := range extractorResults { - builder.WriteString(colorizer.Colorizer.BrightCyan(result).String()) - - if i != len(extractorResults)-1 { - builder.WriteRune(',') - } - } - - builder.WriteString("]") - } - - // write meta if any - if len(req.Meta) > 0 && !e.noMeta { - builder.WriteString(" [") - - var metas []string - for name, value := range req.Meta { - metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String()) - } - - builder.WriteString(strings.Join(metas, ",")) - builder.WriteString("]") - } - - builder.WriteRune('\n') - - // Write output to screen as well as any output file - message := builder.String() - gologger.Silentf("%s", message) - - if e.writer != nil { - if e.coloredOutput { - message = e.decolorizer.ReplaceAllString(message, "") - } - - if err := e.writer.WriteString(message); err != nil { - gologger.Errorf("Could not write output data: %s\n", err) - return - } - } -} diff --git a/v2/pkg/executer/utils.go b/v2/pkg/executer/utils.go deleted file mode 100644 index 50954e5a2..000000000 --- a/v2/pkg/executer/utils.go +++ /dev/null @@ -1,59 +0,0 @@ -package executer - -import ( - "net/http" - "net/url" - "strings" - "unsafe" -) - -type jsonOutput map[string]interface{} - -// unsafeToString converts byte slice to string with zero allocations -func unsafeToString(bs []byte) string { - return *(*string)(unsafe.Pointer(&bs)) -} - -// headersToString converts http headers to string -func headersToString(headers http.Header) string { - builder := &strings.Builder{} - - for header, values := range headers { - builder.WriteString(header) - builder.WriteString(": ") - - for i, value := range values { - builder.WriteString(value) - - if i != len(values)-1 { - builder.WriteRune('\n') - builder.WriteString(header) - builder.WriteString(": ") - } - } - builder.WriteRune('\n') - } - return builder.String() -} - -// isURL tests a string to determine if it is a well-structured url or not. -func isURL(toTest string) bool { - _, err := url.ParseRequestURI(toTest) - if err != nil { - return false - } - u, err := url.Parse(toTest) - if err != nil || u.Scheme == "" || u.Host == "" { - return false - } - return true -} - -// extractDomain extracts the domain name of a URL -func extractDomain(theURL string) string { - u, err := url.Parse(theURL) - if err != nil { - return "" - } - return u.Hostname() -} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index b5f099411..971cb3a8c 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -32,11 +32,18 @@ var StringToType = map[string]Type{ } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, Type Type) (*Generator, error) { +func New(payloads map[string]interface{}, Type Type, templatePath string) (*Generator, error) { + generator := &Generator{} + if err := generator.validate(payloads, templatePath); err != nil { + return nil, err + } + compiled, err := loadPayloads(payloads) if err != nil { return nil, err } + generator.Type = Type + generator.payloads = compiled // Validate the payload types if Type == Sniper && len(compiled) > 1 { @@ -51,7 +58,6 @@ func New(payloads map[string]interface{}, Type Type) (*Generator, error) { totalLength = len(v) } } - generator := &Generator{Type: Type, payloads: compiled} return generator, nil } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 93843c856..1380dac90 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -9,7 +9,7 @@ import ( func TestSniperGenerator(t *testing.T) { usernames := []string{"admin", "password", "login", "test"} - generator, err := New(map[string]interface{}{"username": usernames}, Sniper) + generator, err := New(map[string]interface{}{"username": usernames}, Sniper, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -29,7 +29,7 @@ func TestPitchforkGenerator(t *testing.T) { usernames := []string{"admin", "token"} passwords := []string{"admin", "password"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -50,7 +50,7 @@ func TestClusterbombGenerator(t *testing.T) { usernames := []string{"admin"} passwords := []string{"admin", "password", "token"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go new file mode 100644 index 000000000..acc8a2da0 --- /dev/null +++ b/v2/pkg/protocols/common/generators/validate.go @@ -0,0 +1,61 @@ +package generators + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/cast" +) + +// validate validates the payloads if any. +func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error { + for name, payload := range payloads { + switch pt := payload.(type) { + case string: + // check if it's a multiline string list + if len(strings.Split(pt, "\n")) != 1 { + return errors.New("invalid number of lines in payload") + } + + // check if it's a worldlist file and try to load it + if fileExists(pt) { + continue + } + + changed := false + pathTokens := strings.Split(templatePath, "/") + + for i := range pathTokens { + tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt) + if fileExists(tpath) { + payloads[name] = tpath + changed = true + break + } + } + if !changed { + return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) + } + case interface{}: + loadedPayloads := cast.ToStringSlice(pt) + if len(loadedPayloads) == 0 { + return fmt.Errorf("the payload %s does not contain enough elements", name) + } + default: + return fmt.Errorf("the payload %s has invalid type", name) + } + } + return nil +} + +// fileExists checks if a file exists and is not a directory +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/v2/pkg/protocols/dns/executer_test.go b/v2/pkg/protocols/dns/executer_test.go deleted file mode 100644 index 9143eabaf..000000000 --- a/v2/pkg/protocols/dns/executer_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package dns - -import ( - "fmt" - "testing" - - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" -) - -func TestRequest(t *testing.T) { - err := dnsclientpool.Init(&types.Options{}) - require.Nil(t, err, "could not initialize dns client pool") - - writer, err := output.NewStandardWriter(true, false, false, "", "") - require.Nil(t, err, "could not create standard output writer") - - progress, err := progress.NewProgress(false, false, 0) - require.Nil(t, err, "could not create standard progress writer") - - protocolOpts := &protocols.ExecuterOptions{ - TemplateID: "testing-dns", - TemplateInfo: map[string]string{"author": "test"}, - Output: writer, - Options: &types.Options{}, - Progress: progress, - } - req := &Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{ - Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}}, - }} - err = req.Compile(protocolOpts) - require.Nil(t, err, "could not compile request") - - output, err := req.ExecuteWithResults("docs.hackerone.com.", nil) - require.Nil(t, err, "could not execute request") - - for _, result := range output { - fmt.Printf("%+v\n", result) - } -} - -func TestExecuter(t *testing.T) { - err := dnsclientpool.Init(&types.Options{}) - require.Nil(t, err, "could not initialize dns client pool") - - writer, err := output.NewStandardWriter(true, false, false, "", "") - require.Nil(t, err, "could not create standard output writer") - - progress, err := progress.NewProgress(false, false, 0) - require.Nil(t, err, "could not create standard progress writer") - - protocolOpts := &protocols.ExecuterOptions{ - TemplateID: "testing-dns", - TemplateInfo: map[string]string{"author": "test"}, - Output: writer, - Options: &types.Options{}, - Progress: progress, - } - executer := NewExecuter([]*Request{&Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{ - Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}}, - }}}, protocolOpts) - err = executer.Compile() - require.Nil(t, err, "could not compile request") - - _, err = executer.Execute("docs.hackerone.com") - require.Nil(t, err, "could not execute request") -} diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 846b8896c..b16f5c98b 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -31,7 +31,7 @@ func TestRequestGeneratorClusterSingle(t *testing.T) { attackType: generators.ClusterBomb, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType) + req.generator, err = generators.New(req.Payloads, req.attackType, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator() @@ -54,7 +54,7 @@ func TestRequestGeneratorClusterMultipleRaw(t *testing.T) { attackType: generators.ClusterBomb, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType) + req.generator, err = generators.New(req.Payloads, req.attackType, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator() diff --git a/v2/pkg/protocols/http/executer_test.go b/v2/pkg/protocols/http/executer_test.go deleted file mode 100644 index a8bafdb26..000000000 --- a/v2/pkg/protocols/http/executer_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package http - -import ( - "testing" - - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" - "go.uber.org/ratelimit" -) - -func TestRequest(t *testing.T) { - err := httpclientpool.Init(&types.Options{}) - require.Nil(t, err, "could not initialize dns client pool") - - writer, err := output.NewStandardWriter(true, false, false, "", "") - require.Nil(t, err, "could not create standard output writer") - - progress, err := progress.NewProgress(false, false, 0) - require.Nil(t, err, "could not create standard progress writer") - - protocolOpts := &protocols.ExecuterOptions{ - TemplateID: "testing-dns", - TemplateInfo: map[string]string{"author": "test"}, - Output: writer, - Options: &types.Options{}, - Progress: progress, - RateLimiter: ratelimit.New(100), - } - executer := NewExecuter([]*Request{&Request{Path: []string{"{{BaseURL}}"}, Method: "GET", Operators: &operators.Operators{ - Matchers: []*matchers.Matcher{{Type: "dsl", DSL: []string{"!contains(tolower(all_headers), 'x-frame-options')"}, Part: "body"}}, - }}}, protocolOpts) - err = executer.Compile() - require.Nil(t, err, "could not compile request") - - _, err = executer.Execute("https://example.com") - require.Nil(t, err, "could not execute request") - - // for _, result := range output { - // fmt.Printf("%+v\n", result) - // } -} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 949a210ba..00cb2131b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -91,8 +91,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } if len(r.Payloads) > 0 { - r.attackType = generators.StringToType[r.AttackType] - r.generator, err = generators.New(r.Payloads, r.attackType) + attackType := r.AttackType + if attackType == "" { + attackType = "sniper" + } + r.attackType = generators.StringToType[attackType] + + r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9be2ef1c1..c3c99973c 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -26,6 +26,8 @@ type Executer interface { type ExecuterOptions struct { // TemplateID is the ID of the template for the request TemplateID string + // TemplatePath is the path of the template for the request + TemplatePath string // TemplateInfo contains information block of the template request TemplateInfo map[string]string // Output is a writer interface for writing output events from executer. diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index f15d42b26..4ed579c2a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -3,11 +3,7 @@ package templates import ( "fmt" "os" - "path" - "strings" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "gopkg.in/yaml.v2" ) @@ -35,99 +31,9 @@ func Parse(file string) (*Template, error) { // Compile the matchers and the extractors for http requests for _, request := range template.BulkRequestsHTTP { - // Get the condition between the matchers - condition, ok := matchers.ConditionTypes[request.MatchersCondition] - if !ok { - request.SetMatchersCondition(matchers.ORCondition) - } else { - request.SetMatchersCondition(condition) - } - - // Set the attack type - used only in raw requests - attack, ok := generators.AttackTypes[request.AttackType] - if !ok { - request.SetAttackType(generators.Sniper) - } else { - request.SetAttackType(attack) - } - - // Validate the payloads if any - for name, payload := range request.Payloads { - switch pt := payload.(type) { - case string: - // check if it's a multiline string list - if len(strings.Split(pt, "\n")) <= 1 { - // check if it's a worldlist file - if !generators.FileExists(pt) { - // attempt to load the file by taking the full path, tokezining it and searching the template in such paths - changed := false - pathTokens := strings.Split(template.path, "/") - - for i := range pathTokens { - tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt) - if generators.FileExists(tpath) { - request.Payloads[name] = tpath - changed = true - - break - } - } - - if !changed { - return nil, fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) - } - } - } - case []string, []interface{}: - if len(payload.([]interface{})) == 0 { - return nil, fmt.Errorf("the payload %s does not contain enough elements", name) - } - default: - return nil, fmt.Errorf("the payload %s has invalid type", name) - } - } - - for _, matcher := range request.Matchers { - matchErr := matcher.CompileMatchers() - if matchErr != nil { - return nil, matchErr - } - } - - for _, extractor := range request.Extractors { - extractErr := extractor.CompileExtractors() - if extractErr != nil { - return nil, extractErr - } - } request.InitGenerator() } - // Compile the matchers and the extractors for dns requests - for _, request := range template.RequestsDNS { - // Get the condition between the matchers - condition, ok := matchers.ConditionTypes[request.MatchersCondition] - if !ok { - request.SetMatchersCondition(matchers.ORCondition) - } else { - request.SetMatchersCondition(condition) - } - - for _, matcher := range request.Matchers { - err = matcher.CompileMatchers() - if err != nil { - return nil, err - } - } - - for _, extractor := range request.Extractors { - err := extractor.CompileExtractors() - if err != nil { - return nil, err - } - } - } - return template, nil }