External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 05:05:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added payload validation + misc

Browse Source
This commit is contained in:
Ice3man543 2020-12-29 12:08:46 +05:30
parent 97ad8e592e
commit aefa2717f7
12 changed files with 83 additions and 1006 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

577
v2/pkg/executer/executer_http.go
View File

@ -1,577 +0,0 @@
package executer
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/cookiejar"
"net/http/httputil"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/corpix/uarand"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/ratelimit"
)
const (
two = 2
ten = 10
defaultMaxWorkers = 150
defaultMaxHistorydata = 150
)
// HTTPExecuter is client for performing HTTP requests
// for a template.
type HTTPExecuter struct {
pf *projetctfile.ProjectFile
customHeaders requests.CustomHeaders
colorizer colorizer.NucleiColorizer
httpClient *retryablehttp.Client
rawHTTPClient *rawhttp.Client
template *templates.Template
bulkHTTPRequest *requests.BulkHTTPRequest
writer *bufwriter.Writer
CookieJar *cookiejar.Jar
traceLog tracelog.Log
decolorizer *regexp.Regexp
randomAgent bool
vhost bool
coloredOutput bool
debug bool
Results bool
jsonOutput bool
jsonRequest bool
noMeta bool
stopAtFirstMatch bool
ratelimiter ratelimit.Limiter
}
// HTTPOptions contains configuration options for the HTTP executer.
type HTTPOptions struct {
Template *templates.Template
BulkHTTPRequest *requests.BulkHTTPRequest
CookieJar *cookiejar.Jar
PF *projetctfile.ProjectFile
}
// NewHTTPExecuter creates a new HTTP executer from a template
// and a HTTP request query.
func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
var (
proxyURL *url.URL
err error
)
if err != nil {
return nil, err
}
// Create the HTTP Client
client := makeHTTPClient(proxyURL, options)
// nolint:bodyclose // false positive there is no body to close yet
if options.CookieJar != nil {
client.HTTPClient.Jar = options.CookieJar
} else if options.CookieReuse {
jar, err := cookiejar.New(nil)
if err != nil {
return nil, err
}
client.HTTPClient.Jar = jar
}
executer := &HTTPExecuter{
debug: options.Debug,
jsonOutput: options.JSON,
jsonRequest: options.JSONRequests,
noMeta: options.NoMeta,
httpClient: client,
rawHTTPClient: rawClient,
traceLog: options.TraceLog,
template: options.Template,
bulkHTTPRequest: options.BulkHTTPRequest,
writer: options.Writer,
randomAgent: options.RandomAgent,
customHeaders: options.CustomHeaders,
CookieJar: options.CookieJar,
coloredOutput: options.ColoredOutput,
colorizer: *options.Colorizer,
decolorizer: options.Decolorizer,
stopAtFirstMatch: options.StopAtFirstMatch,
pf: options.PF,
vhost: options.Vhost,
ratelimiter: options.RateLimiter,
}
return executer, nil
}
func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
result := &Result{
Matches: make(map[string]interface{}),
Extractions: make(map[string]interface{}),
}
dynamicvalues := make(map[string]interface{})
// verify if the URL is already being processed
if e.bulkHTTPRequest.HasGenerator(reqURL) {
return result
}
e.bulkHTTPRequest.CreateGenerator(reqURL)
// Workers that keeps enqueuing new requests
maxWorkers := e.bulkHTTPRequest.RaceNumberRequests
swg := sizedwaitgroup.New(maxWorkers)
for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
swg.Add()
// base request
result.Lock()
request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
}
go func(httpRequest *requests.HTTPRequest) {
defer swg.Done()
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
}
}(request)
}
swg.Wait()
return result
}
func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result {
result := &Result{
Matches: make(map[string]interface{}),
Extractions: make(map[string]interface{}),
}
dynamicvalues := make(map[string]interface{})
// verify if the URL is already being processed
if e.bulkHTTPRequest.HasGenerator(reqURL) {
return result
}
remaining := e.bulkHTTPRequest.GetRequestCount()
e.bulkHTTPRequest.CreateGenerator(reqURL)
// Workers that keeps enqueuing new requests
maxWorkers := e.bulkHTTPRequest.Threads
swg := sizedwaitgroup.New(maxWorkers)
for e.bulkHTTPRequest.Next(reqURL) {
result.Lock()
request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
p.Drop(remaining)
} else {
swg.Add()
go func(httpRequest *requests.HTTPRequest) {
defer swg.Done()
e.ratelimiter.Take()
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}(request)
}
p.Update()
e.bulkHTTPRequest.Increment(reqURL)
}
swg.Wait()
return result
}
func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
result := &Result{
Matches: make(map[string]interface{}),
Extractions: make(map[string]interface{}),
}
dynamicvalues := make(map[string]interface{})
// verify if the URL is already being processed
if e.bulkHTTPRequest.HasGenerator(reqURL) {
return result
}
e.bulkHTTPRequest.CreateGenerator(reqURL)
// need to extract the target from the url
URL, err := url.Parse(reqURL)
if err != nil {
return result
}
// defaultMaxWorkers should be a sufficient value to keep queues always full
maxWorkers := defaultMaxWorkers
// in case the queue is bigger increase the workers
if pipeOptions.MaxPendingRequests > maxWorkers {
maxWorkers = pipeOptions.MaxPendingRequests
}
swg := sizedwaitgroup.New(maxWorkers)
for e.bulkHTTPRequest.Next(reqURL) {
result.Lock()
request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
} else {
swg.Add()
go func(httpRequest *requests.HTTPRequest) {
defer swg.Done()
// HTTP pipelining ignores rate limit
// If the request was built correctly then execute it
request.Pipeline = true
request.PipelineClient = pipeclient
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
request.PipelineClient = nil
}(request)
}
e.bulkHTTPRequest.Increment(reqURL)
}
swg.Wait()
return result
}
// ExecuteHTTP executes the HTTP request on a URL
func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result {
var customHost string
if e.vhost {
parts := strings.Split(reqURL, ",")
reqURL = parts[0]
customHost = parts[1]
}
// verify if pipeline was requested
if e.bulkHTTPRequest.Pipeline {
return e.ExecuteTurboHTTP(reqURL)
}
// verify if a basic race condition was requested
if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 {
return e.ExecuteRaceRequest(reqURL)
}
// verify if parallel elaboration was requested
if e.bulkHTTPRequest.Threads > 0 {
return e.ExecuteParallelHTTP(p, reqURL)
}
var requestNumber int
result := &Result{
Matches: make(map[string]interface{}),
Extractions: make(map[string]interface{}),
historyData: make(map[string]interface{}),
}
dynamicvalues := make(map[string]interface{})
// verify if the URL is already being processed
if e.bulkHTTPRequest.HasGenerator(reqURL) {
return result
}
remaining := e.bulkHTTPRequest.GetRequestCount()
e.bulkHTTPRequest.CreateGenerator(reqURL)
for e.bulkHTTPRequest.Next(reqURL) {
requestNumber++
result.Lock()
httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
p.Drop(remaining)
} else {
if e.vhost {
if httpRequest.Request != nil {
httpRequest.Request.Host = customHost
}
if httpRequest.RawRequest != nil && httpRequest.RawRequest.Headers != nil {
httpRequest.RawRequest.Headers["Host"] = customHost
}
}
e.ratelimiter.Take()
// If the request was built correctly then execute it
format := "%s_" + strconv.Itoa(requestNumber)
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format)
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
e.traceLog.Request(e.template.ID, reqURL, "http", err)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}
p.Update()
// Check if has to stop processing at first valid result
if e.stopAtFirstMatch && result.GotResults {
p.Drop(remaining)
break
}
// move always forward with requests
e.bulkHTTPRequest.Increment(reqURL)
remaining--
}
gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL)
return result
}
func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error {
// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
if e.randomAgent {
// nolint:errcheck // ignoring error
e.customHeaders.Set("User-Agent: " + uarand.GetRandom())
}
e.setCustomHeaders(request)
var (
resp *http.Response
err error
dumpedRequest []byte
fromcache bool
)
if e.debug || e.pf != nil {
dumpedRequest, err = requests.Dump(request, reqURL)
if err != nil {
return err
}
}
if e.debug {
gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
}
timeStart := time.Now()
if request.Pipeline {
resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
if err != nil {
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
} else if request.Unsafe {
// rawhttp
// burp uses "\r\n" as new line character
request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n")
options := e.rawHTTPClient.Options
options.AutomaticContentLength = request.AutomaticContentLengthHeader
options.AutomaticHostHeader = request.AutomaticHostHeader
options.FollowRedirects = request.FollowRedirects
resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options)
if err != nil {
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
} else {
// if nuclei-project is available check if the request was already sent previously
if e.pf != nil {
// if unavailable fail silently
fromcache = true
// nolint:bodyclose // false positive the response is generated at runtime
resp, err = e.pf.Get(dumpedRequest)
if err != nil {
fromcache = false
}
}
// retryablehttp
if resp == nil {
resp, err = e.httpClient.Do(request.Request)
if err != nil {
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}
duration := time.Since(timeStart)
// Dump response - Step 1 - Decompression not yet handled
var dumpedResponse []byte
if e.debug {
var dumpErr error
dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
if dumpErr != nil {
return errors.Wrap(dumpErr, "could not dump http response")
}
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
_, copyErr := io.Copy(ioutil.Discard, resp.Body)
if copyErr != nil {
resp.Body.Close()
return copyErr
}
resp.Body.Close()
return errors.Wrap(err, "could not read http body")
}
resp.Body.Close()
// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
// so in case we have to manually do it
dataOrig := data
data, err = requests.HandleDecompression(request, data)
if err != nil {
return errors.Wrap(err, "could not decompress http body")
}
// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
if e.debug {
dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
}
// if nuclei-project is enabled store the response if not previously done
if e.pf != nil && !fromcache {
err := e.pf.Set(dumpedRequest, resp, data)
if err != nil {
return errors.Wrap(err, "could not store in project file")
}
}
// Convert response body from []byte to string with zero copy
body := unsafeToString(data)
headers := headersToString(resp.Header)
var matchData map[string]interface{}
if payloads != nil {
matchData = generators.MergeMaps(result.historyData, payloads)
}
// store for internal purposes the DSL matcher data
// hardcode stopping storing data after defaultMaxHistorydata items
if len(result.historyData) < defaultMaxHistorydata {
result.Lock()
// update history data with current reqURL and hostname
result.historyData["reqURL"] = reqURL
if parsed, err := url.Parse(reqURL); err == nil {
result.historyData["Hostname"] = parsed.Host
}
result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
if payloads == nil {
// merge them to history data
result.historyData = generators.MergeMaps(result.historyData, payloads)
}
result.historyData = generators.MergeMaps(result.historyData, dynamicvalues)
// complement match data with new one if necessary
matchData = generators.MergeMaps(matchData, result.historyData)
result.Unlock()
}
return nil
}
// Close closes the http executer for a template.
func (e *HTTPExecuter) Close() {}
func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
for _, customHeader := range e.customHeaders {
// This should be pre-computed somewhere and done only once
tokens := strings.SplitN(customHeader, ":", two)
// if it's an invalid header skip it
if len(tokens) < two {
continue
}
headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
if r.RawRequest != nil {
// rawhttp
r.RawRequest.Headers[headerName] = headerValue
} else {
// retryablehttp
headerName = strings.TrimSpace(headerName)
headerValue = strings.TrimSpace(headerValue)
r.Request.Header[headerName] = []string{headerValue}
}
}
}

147
v2/pkg/executer/output_http.go
View File

@ -1,147 +0,0 @@
package executer
import (
"fmt"
"net/http"
"net/http/httputil"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
)
// writeOutputHTTP writes http output to streams
func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) {
var URL string
if req.RawRequest != nil {
URL = req.RawRequest.FullURL
}
if req.Request != nil {
URL = req.Request.URL.String()
}
if e.jsonOutput {
output := make(jsonOutput)
output["matched"] = URL
if !e.noMeta {
output["template"] = e.template.ID
output["type"] = "http"
output["host"] = reqURL
if len(meta) > 0 {
output["meta"] = meta
}
for k, v := range e.template.Info {
output[k] = v
}
if matcher != nil && len(matcher.Name) > 0 {
output["matcher_name"] = matcher.Name
}
if len(extractorResults) > 0 {
output["extracted_results"] = extractorResults
}
// TODO: URL should be an argument
if e.jsonRequest {
dumpedRequest, err := requests.Dump(req, URL)
if err != nil {
gologger.Warningf("could not dump request: %s\n", err)
} else {
output["request"] = string(dumpedRequest)
}
dumpedResponse, err := httputil.DumpResponse(resp, false)
if err != nil {
gologger.Warningf("could not dump response: %s\n", err)
} else {
output["response"] = string(dumpedResponse) + body
}
}
}
data, err := jsoniter.Marshal(output)
if err != nil {
gologger.Warningf("Could not marshal json output: %s\n", err)
}
gologger.Silentf("%s", string(data))
if e.writer != nil {
if err := e.writer.Write(data); err != nil {
gologger.Errorf("Could not write output data: %s\n", err)
return
}
}
return
}
builder := &strings.Builder{}
colorizer := e.colorizer
if !e.noMeta {
builder.WriteRune('[')
builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
if matcher != nil && len(matcher.Name) > 0 {
builder.WriteString(":")
builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
}
builder.WriteString("] [")
builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
builder.WriteString("] ")
if e.template.Info["severity"] != "" {
builder.WriteString("[")
builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
builder.WriteString("] ")
}
}
builder.WriteString(URL)
// If any extractors, write the results
if len(extractorResults) > 0 && !e.noMeta {
builder.WriteString(" [")
for i, result := range extractorResults {
builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
if i != len(extractorResults)-1 {
builder.WriteRune(',')
}
}
builder.WriteString("]")
}
// write meta if any
if len(req.Meta) > 0 && !e.noMeta {
builder.WriteString(" [")
var metas []string
for name, value := range req.Meta {
metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String())
}
builder.WriteString(strings.Join(metas, ","))
builder.WriteString("]")
}
builder.WriteRune('\n')
// Write output to screen as well as any output file
message := builder.String()
gologger.Silentf("%s", message)
if e.writer != nil {
if e.coloredOutput {
message = e.decolorizer.ReplaceAllString(message, "")
}
if err := e.writer.WriteString(message); err != nil {
gologger.Errorf("Could not write output data: %s\n", err)
return
}
}
}

59
v2/pkg/executer/utils.go
View File

@ -1,59 +0,0 @@
package executer
import (
"net/http"
"net/url"
"strings"
"unsafe"
)
type jsonOutput map[string]interface{}
// unsafeToString converts byte slice to string with zero allocations
func unsafeToString(bs []byte) string {
return *(*string)(unsafe.Pointer(&bs))
}
// headersToString converts http headers to string
func headersToString(headers http.Header) string {
builder := &strings.Builder{}
for header, values := range headers {
builder.WriteString(header)
builder.WriteString(": ")
for i, value := range values {
builder.WriteString(value)
if i != len(values)-1 {
builder.WriteRune('\n')
builder.WriteString(header)
builder.WriteString(": ")
}
}
builder.WriteRune('\n')
}
return builder.String()
}
// isURL tests a string to determine if it is a well-structured url or not.
func isURL(toTest string) bool {
_, err := url.ParseRequestURI(toTest)
if err != nil {
return false
}
u, err := url.Parse(toTest)
if err != nil || u.Scheme == "" || u.Host == "" {
return false
}
return true
}
// extractDomain extracts the domain name of a URL
func extractDomain(theURL string) string {
u, err := url.Parse(theURL)
if err != nil {
return ""
}
return u.Hostname()
}

10
v2/pkg/protocols/common/generators/generators.go
View File

@ -32,11 +32,18 @@ var StringToType = map[string]Type{
}
// New creates a new generator structure for payload generation
func New(payloads map[string]interface{}, Type Type) (*Generator, error) {
func New(payloads map[string]interface{}, Type Type, templatePath string) (*Generator, error) {
generator := &Generator{}
if err := generator.validate(payloads, templatePath); err != nil {
return nil, err
}
compiled, err := loadPayloads(payloads)
if err != nil {
return nil, err
}
generator.Type = Type
generator.payloads = compiled
// Validate the payload types
if Type == Sniper && len(compiled) > 1 {
@ -51,7 +58,6 @@ func New(payloads map[string]interface{}, Type Type) (*Generator, error) {
totalLength = len(v)
}
}
generator := &Generator{Type: Type, payloads: compiled}
return generator, nil
}

6
v2/pkg/protocols/common/generators/generators_test.go
View File

@ -9,7 +9,7 @@ import (
func TestSniperGenerator(t *testing.T) {
usernames := []string{"admin", "password", "login", "test"}
generator, err := New(map[string]interface{}{"username": usernames}, Sniper)
generator, err := New(map[string]interface{}{"username": usernames}, Sniper, "")
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -29,7 +29,7 @@ func TestPitchforkGenerator(t *testing.T) {
usernames := []string{"admin", "token"}
passwords := []string{"admin", "password"}
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork)
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "")
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -50,7 +50,7 @@ func TestClusterbombGenerator(t *testing.T) {
usernames := []string{"admin"}
passwords := []string{"admin", "password", "token"}
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb)
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb, "")
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()

61
v2/pkg/protocols/common/generators/validate.go Normal file
View File

@ -0,0 +1,61 @@
package generators
import (
"errors"
"fmt"
"os"
"path"
"strings"
"github.com/spf13/cast"
)
// validate validates the payloads if any.
func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error {
for name, payload := range payloads {
switch pt := payload.(type) {
case string:
// check if it's a multiline string list
if len(strings.Split(pt, "\n")) != 1 {
return errors.New("invalid number of lines in payload")
}
// check if it's a worldlist file and try to load it
if fileExists(pt) {
continue
}
changed := false
pathTokens := strings.Split(templatePath, "/")
for i := range pathTokens {
tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
if fileExists(tpath) {
payloads[name] = tpath
changed = true
break
}
}
if !changed {
return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name)
}
case interface{}:
loadedPayloads := cast.ToStringSlice(pt)
if len(loadedPayloads) == 0 {
return fmt.Errorf("the payload %s does not contain enough elements", name)
}
default:
return fmt.Errorf("the payload %s has invalid type", name)
}
}
return nil
}
// fileExists checks if a file exists and is not a directory
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}

73
v2/pkg/protocols/dns/executer_test.go
View File

@ -1,73 +0,0 @@
package dns
import (
"fmt"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/stretchr/testify/require"
)
func TestRequest(t *testing.T) {
err := dnsclientpool.Init(&types.Options{})
require.Nil(t, err, "could not initialize dns client pool")
writer, err := output.NewStandardWriter(true, false, false, "", "")
require.Nil(t, err, "could not create standard output writer")
progress, err := progress.NewProgress(false, false, 0)
require.Nil(t, err, "could not create standard progress writer")
protocolOpts := &protocols.ExecuterOptions{
TemplateID: "testing-dns",
TemplateInfo: map[string]string{"author": "test"},
Output: writer,
Options: &types.Options{},
Progress: progress,
}
req := &Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{
Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}},
}}
err = req.Compile(protocolOpts)
require.Nil(t, err, "could not compile request")
output, err := req.ExecuteWithResults("docs.hackerone.com.", nil)
require.Nil(t, err, "could not execute request")
for _, result := range output {
fmt.Printf("%+v\n", result)
}
}
func TestExecuter(t *testing.T) {
err := dnsclientpool.Init(&types.Options{})
require.Nil(t, err, "could not initialize dns client pool")
writer, err := output.NewStandardWriter(true, false, false, "", "")
require.Nil(t, err, "could not create standard output writer")
progress, err := progress.NewProgress(false, false, 0)
require.Nil(t, err, "could not create standard progress writer")
protocolOpts := &protocols.ExecuterOptions{
TemplateID: "testing-dns",
TemplateInfo: map[string]string{"author": "test"},
Output: writer,
Options: &types.Options{},
Progress: progress,
}
executer := NewExecuter([]*Request{&Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{
Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}},
}}}, protocolOpts)
err = executer.Compile()
require.Nil(t, err, "could not compile request")
_, err = executer.Execute("docs.hackerone.com")
require.Nil(t, err, "could not execute request")
}

4
v2/pkg/protocols/http/build_request_test.go
View File

@ -31,7 +31,7 @@ func TestRequestGeneratorClusterSingle(t *testing.T) {
attackType: generators.ClusterBomb,
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`},
}
req.generator, err = generators.New(req.Payloads, req.attackType)
req.generator, err = generators.New(req.Payloads, req.attackType, "")
require.Nil(t, err, "could not create generator")
generator := req.newGenerator()
@ -54,7 +54,7 @@ func TestRequestGeneratorClusterMultipleRaw(t *testing.T) {
attackType: generators.ClusterBomb,
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
}
req.generator, err = generators.New(req.Payloads, req.attackType)
req.generator, err = generators.New(req.Payloads, req.attackType, "")
require.Nil(t, err, "could not create generator")
generator := req.newGenerator()

47
v2/pkg/protocols/http/executer_test.go
View File

@ -1,47 +0,0 @@
package http
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/stretchr/testify/require"
"go.uber.org/ratelimit"
)
func TestRequest(t *testing.T) {
err := httpclientpool.Init(&types.Options{})
require.Nil(t, err, "could not initialize dns client pool")
writer, err := output.NewStandardWriter(true, false, false, "", "")
require.Nil(t, err, "could not create standard output writer")
progress, err := progress.NewProgress(false, false, 0)
require.Nil(t, err, "could not create standard progress writer")
protocolOpts := &protocols.ExecuterOptions{
TemplateID: "testing-dns",
TemplateInfo: map[string]string{"author": "test"},
Output: writer,
Options: &types.Options{},
Progress: progress,
RateLimiter: ratelimit.New(100),
}
executer := NewExecuter([]*Request{&Request{Path: []string{"{{BaseURL}}"}, Method: "GET", Operators: &operators.Operators{
Matchers: []*matchers.Matcher{{Type: "dsl", DSL: []string{"!contains(tolower(all_headers), 'x-frame-options')"}, Part: "body"}},
}}}, protocolOpts)
err = executer.Compile()
require.Nil(t, err, "could not compile request")
_, err = executer.Execute("https://example.com")
require.Nil(t, err, "could not execute request")
// for _, result := range output {
// fmt.Printf("%+v\n", result)
// }
}

9
v2/pkg/protocols/http/http.go
View File

@ -91,8 +91,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
}
if len(r.Payloads) > 0 {
r.attackType = generators.StringToType[r.AttackType]
r.generator, err = generators.New(r.Payloads, r.attackType)
attackType := r.AttackType
if attackType == "" {
attackType = "sniper"
}
r.attackType = generators.StringToType[attackType]
r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}

2
v2/pkg/protocols/protocols.go
View File

@ -26,6 +26,8 @@ type Executer interface {
type ExecuterOptions struct {
// TemplateID is the ID of the template for the request
TemplateID string
// TemplatePath is the path of the template for the request
TemplatePath string
// TemplateInfo contains information block of the template request
TemplateInfo map[string]string
// Output is a writer interface for writing output events from executer.

94
v2/pkg/templates/compile.go
View File

@ -3,11 +3,7 @@ package templates
import (
"fmt"
"os"
"path"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"gopkg.in/yaml.v2"
)
@ -35,99 +31,9 @@ func Parse(file string) (*Template, error) {
// Compile the matchers and the extractors for http requests
for _, request := range template.BulkRequestsHTTP {
// Get the condition between the matchers
condition, ok := matchers.ConditionTypes[request.MatchersCondition]
if !ok {
request.SetMatchersCondition(matchers.ORCondition)
} else {
request.SetMatchersCondition(condition)
}
// Set the attack type - used only in raw requests
attack, ok := generators.AttackTypes[request.AttackType]
if !ok {
request.SetAttackType(generators.Sniper)
} else {
request.SetAttackType(attack)
}
// Validate the payloads if any
for name, payload := range request.Payloads {
switch pt := payload.(type) {
case string:
// check if it's a multiline string list
if len(strings.Split(pt, "\n")) <= 1 {
// check if it's a worldlist file
if !generators.FileExists(pt) {
// attempt to load the file by taking the full path, tokezining it and searching the template in such paths
changed := false
pathTokens := strings.Split(template.path, "/")
for i := range pathTokens {
tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
if generators.FileExists(tpath) {
request.Payloads[name] = tpath
changed = true
break
}
}
if !changed {
return nil, fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name)
}
}
}
case []string, []interface{}:
if len(payload.([]interface{})) == 0 {
return nil, fmt.Errorf("the payload %s does not contain enough elements", name)
}
default:
return nil, fmt.Errorf("the payload %s has invalid type", name)
}
}
for _, matcher := range request.Matchers {
matchErr := matcher.CompileMatchers()
if matchErr != nil {
return nil, matchErr
}
}
for _, extractor := range request.Extractors {
extractErr := extractor.CompileExtractors()
if extractErr != nil {
return nil, extractErr
}
}
request.InitGenerator()
}
// Compile the matchers and the extractors for dns requests
for _, request := range template.RequestsDNS {
// Get the condition between the matchers
condition, ok := matchers.ConditionTypes[request.MatchersCondition]
if !ok {
request.SetMatchersCondition(matchers.ORCondition)
} else {
request.SetMatchersCondition(condition)
}
for _, matcher := range request.Matchers {
err = matcher.CompileMatchers()
if err != nil {
return nil, err
}
}
for _, extractor := range request.Extractors {
err := extractor.CompileExtractors()
if err != nil {
return nil, err
}
}
}
return template, nil
}