External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:55:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

code refactor

Browse Source
This commit is contained in:
Mzack9999 2020-07-18 21:42:23 +02:00
parent 6d7bb9a2c3
commit 57a496203f
8 changed files with 291 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
v2/internal/runner/runner.go
View File

@ -112,7 +112,7 @@ func (r *Runner) RunEnumeration() {
var results bool
template := t.(*templates.Template)
// process http requests
for _, request := range template.RequestsHTTP {
for _, request := range template.BulkRequestsHTTP {
results = r.processTemplateRequest(template, request)
}
// process dns requests
@ -176,7 +176,7 @@ func (r *Runner) RunEnumeration() {
results = dnsResults
}
}
for _, request := range template.RequestsHTTP {
for _, request := range template.BulkRequestsHTTP {
httpResults := r.processTemplateRequest(template, request)
if httpResults {
results = httpResults
@ -248,19 +248,19 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
Writer: writer,
JSON: r.options.JSON,
})
case *requests.HTTPRequest:
case *requests.BulkHTTPRequest:
httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
Debug: r.options.Debug,
Template: template,
HTTPRequest: value,
Writer: writer,
Timeout: r.options.Timeout,
Retries: r.options.Retries,
ProxyURL: r.options.ProxyURL,
ProxySocksURL: r.options.ProxySocksURL,
CustomHeaders: r.options.CustomHeaders,
JSON: r.options.JSON,
CookieReuse: value.CookieReuse,
Debug: r.options.Debug,
Template: template,
BulkHttpRequest: value,
Writer: writer,
Timeout: r.options.Timeout,
Retries: r.options.Retries,
ProxyURL: r.options.ProxyURL,
ProxySocksURL: r.options.ProxySocksURL,
CustomHeaders: r.options.CustomHeaders,
JSON: r.options.JSON,
CookieReuse: value.CookieReuse,
})
}
if err != nil {
@ -302,11 +302,11 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
// See if we got any results from the executers
var results bool
if httpExecuter != nil {
results = httpExecuter.GotResults()
results = httpExecuter.Results
}
if dnsExecuter != nil {
if !results {
results = dnsExecuter.GotResults()
results = dnsExecuter.Results
}
}
return results
@ -376,7 +376,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
return err
}
template := &workflows.Template{}
if len(t.RequestsHTTP) > 0 {
if len(t.BulkRequestsHTTP) > 0 {
template.HTTPOptions = &executer.HTTPOptions{
Debug: r.options.Debug,
Writer: writer,
@ -427,7 +427,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
return err
}
template := &workflows.Template{}
if len(t.RequestsHTTP) > 0 {
if len(t.BulkRequestsHTTP) > 0 {
template.HTTPOptions = &executer.HTTPOptions{
Debug: r.options.Debug,
Writer: writer,

18
v2/pkg/executer/executer_dns.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"os"
"sync"
"sync/atomic"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
@ -20,7 +19,7 @@ import (
type DNSExecuter struct {
debug bool
jsonOutput bool
results uint32
Results bool
dnsClient *retryabledns.Client
template *templates.Template
dnsRequest *requests.DNSRequest
@ -53,7 +52,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
executer := &DNSExecuter{
debug: options.Debug,
jsonOutput: options.JSON,
results: 0,
dnsClient: dnsClient,
template: options.Template,
dnsRequest: options.DNSRequest,
@ -63,14 +61,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
return executer
}
// GotResults returns true if there were any results for the executer
func (e *DNSExecuter) GotResults() bool {
if atomic.LoadUint32(&e.results) == 0 {
return false
}
return true
}
// ExecuteDNS executes the DNS request on a URL
func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
// Parse the URL and return domain if URL.
@ -120,7 +110,7 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
e.writeOutputDNS(domain, matcher, nil)
atomic.CompareAndSwapUint32(&e.results, 0, 1)
e.Results = true
}
}
}
@ -138,7 +128,7 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
// AND or if we have extractors for the mechanism too.
if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputDNS(domain, nil, extractorResults)
atomic.CompareAndSwapUint32(&e.results, 0, 1)
e.Results = true
}
return
@ -147,6 +137,6 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
// Close closes the dns executer for a template.
func (e *DNSExecuter) Close() {
e.outputMutex.Lock()
defer e.outputMutex.Unlock()
e.writer.Flush()
e.outputMutex.Unlock()
}

288
v2/pkg/executer/executer_http.go
View File

@ -13,7 +13,6 @@ import (
"os"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pkg/errors"
@ -28,32 +27,32 @@ import (
// HTTPExecuter is client for performing HTTP requests
// for a template.
type HTTPExecuter struct {
debug bool
results uint32
jsonOutput bool
httpClient *retryablehttp.Client
template *templates.Template
httpRequest *requests.HTTPRequest
writer *bufio.Writer
outputMutex *sync.Mutex
customHeaders requests.CustomHeaders
CookieJar *cookiejar.Jar
debug bool
Results bool
jsonOutput bool
httpClient *retryablehttp.Client
template *templates.Template
bulkHttpRequest *requests.BulkHTTPRequest
writer *bufio.Writer
outputMutex *sync.Mutex
customHeaders requests.CustomHeaders
CookieJar *cookiejar.Jar
}
// HTTPOptions contains configuration options for the HTTP executer.
type HTTPOptions struct {
Template *templates.Template
HTTPRequest *requests.HTTPRequest
Writer *bufio.Writer
Timeout int
Retries int
ProxyURL string
ProxySocksURL string
Debug bool
JSON bool
CustomHeaders requests.CustomHeaders
CookieReuse bool
CookieJar *cookiejar.Jar
Template *templates.Template
BulkHttpRequest *requests.BulkHTTPRequest
Writer *bufio.Writer
Timeout int
Retries int
ProxyURL string
ProxySocksURL string
Debug bool
JSON bool
CustomHeaders requests.CustomHeaders
CookieReuse bool
CookieJar *cookiejar.Jar
}
// NewHTTPExecuter creates a new HTTP executer from a template
@ -83,143 +82,36 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
}
executer := &HTTPExecuter{
debug: options.Debug,
jsonOutput: options.JSON,
results: 0,
httpClient: client,
template: options.Template,
httpRequest: options.HTTPRequest,
outputMutex: &sync.Mutex{},
writer: options.Writer,
customHeaders: options.CustomHeaders,
CookieJar: options.CookieJar,
debug: options.Debug,
jsonOutput: options.JSON,
httpClient: client,
template: options.Template,
bulkHttpRequest: options.BulkHttpRequest,
outputMutex: &sync.Mutex{},
writer: options.Writer,
customHeaders: options.CustomHeaders,
CookieJar: options.CookieJar,
}
return executer, nil
}
// GotResults returns true if there were any results for the executer
func (e *HTTPExecuter) GotResults() bool {
if atomic.LoadUint32(&e.results) == 0 {
return false
}
return true
}
// ExecuteHTTP executes the HTTP request on a URL
func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
result.Matches = make(map[string]interface{})
result.Extractions = make(map[string]interface{})
dynamicvalues := make(map[string]string)
// Compile each request for the template based on the URL
compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
if err != nil {
result.Error = errors.Wrap(err, "could not make http request")
return
}
// Send the request to the target servers
mainLoop:
for compiledRequest := range compiledRequest {
if compiledRequest.Error != nil {
// Requests defined via Model
for e.bulkHttpRequest.Next() {
httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, e.bulkHttpRequest.Current())
if err != nil {
result.Error = errors.Wrap(err, "could not make http request")
return
}
e.setCustomHeaders(compiledRequest)
e.setDynamicValues(compiledRequest, dynamicvalues)
req := compiledRequest.Request
if e.debug {
gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
dumpedRequest, err := httputil.DumpRequest(req.Request, true)
if err != nil {
result.Error = errors.Wrap(err, "could not make http request")
return
}
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
}
e.handleHTTP(URL, httpRequest, dynamicvalues, &result)
resp, err := e.httpClient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
gologger.Warningf("Could not do request: %s\n", err)
continue
}
if e.debug {
gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
result.Error = errors.Wrap(err, "could not dump http response")
return
}
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
result.Error = errors.Wrap(err, "could not read http body")
return
}
resp.Body.Close()
// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
// so in case we have to manually do it
data, err = requests.HandleDecompression(compiledRequest.Request, data)
if err != nil {
result.Error = errors.Wrap(err, "could not decompress http body")
return
}
// Convert response body from []byte to string with zero copy
body := unsafeToString(data)
headers := headersToString(resp.Header)
matcherCondition := e.httpRequest.GetMatchersCondition()
for _, matcher := range e.httpRequest.Matchers {
// Check if the matcher matched
if !matcher.Match(resp, body, headers) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
continue mainLoop
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
result.Matches[matcher.Name] = nil
// probably redundant but ensures we snapshot current payload values when matchers are valid
result.Meta = compiledRequest.Meta
e.writeOutputHTTP(compiledRequest, matcher, nil)
atomic.CompareAndSwapUint32(&e.results, 0, 1)
}
}
}
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults []string
for _, extractor := range e.httpRequest.Extractors {
for match := range extractor.Extract(resp, body, headers) {
if _, ok := dynamicvalues[extractor.Name]; !ok {
dynamicvalues[extractor.Name] = match
}
extractorResults = append(extractorResults, match)
}
// probably redundant but ensures we snapshot current payload values when extractors are valid
result.Meta = compiledRequest.Meta
result.Extractions[extractor.Name] = extractorResults
}
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputHTTP(compiledRequest, nil, extractorResults)
atomic.CompareAndSwapUint32(&e.results, 0, 1)
}
e.bulkHttpRequest.Increment()
}
gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
@ -227,11 +119,107 @@ mainLoop:
return
}
func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dynamicvalues map[string]string, result *Result) error {
e.setCustomHeaders(request)
e.setDynamicValues(request, dynamicvalues)
req := request.Request
if e.debug {
gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
dumpedRequest, err := httputil.DumpRequest(req.Request, true)
if err != nil {
return errors.Wrap(err, "could not make http request")
}
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
}
resp, err := e.httpClient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
gologger.Warningf("Could not do request: %s\n", err)
}
if e.debug {
gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
return errors.Wrap(err, "could not dump http response")
}
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
return errors.Wrap(err, "could not read http body")
}
resp.Body.Close()
// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
// so in case we have to manually do it
data, err = requests.HandleDecompression(req, data)
if err != nil {
return errors.Wrap(err, "could not decompress http body")
}
// Convert response body from []byte to string with zero copy
body := unsafeToString(data)
headers := headersToString(resp.Header)
matcherCondition := e.bulkHttpRequest.GetMatchersCondition()
for _, matcher := range e.bulkHttpRequest.Matchers {
// Check if the matcher matched
if !matcher.Match(resp, body, headers) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
return nil
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.bulkHttpRequest.Extractors) == 0 {
result.Matches[matcher.Name] = nil
// probably redundant but ensures we snapshot current payload values when matchers are valid
result.Meta = request.Meta
e.writeOutputHTTP(request, matcher, nil)
e.Results = true
}
}
}
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults []string
for _, extractor := range e.bulkHttpRequest.Extractors {
for match := range extractor.Extract(resp, body, headers) {
if _, ok := dynamicvalues[extractor.Name]; !ok {
dynamicvalues[extractor.Name] = match
}
extractorResults = append(extractorResults, match)
}
// probably redundant but ensures we snapshot current payload values when extractors are valid
result.Meta = request.Meta
result.Extractions[extractor.Name] = extractorResults
}
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.bulkHttpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputHTTP(request, nil, extractorResults)
e.Results = true
}
return nil
}
// Close closes the http executer for a template.
func (e *HTTPExecuter) Close() {
e.outputMutex.Lock()
defer e.outputMutex.Unlock()
e.writer.Flush()
e.outputMutex.Unlock()
}
// makeHTTPClient creates a http client
@ -239,8 +227,8 @@ func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Clie
retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
retryablehttpOptions.RetryWaitMax = 10 * time.Second
retryablehttpOptions.RetryMax = options.Retries
followRedirects := options.HTTPRequest.Redirects
maxRedirects := options.HTTPRequest.MaxRedirects
followRedirects := options.BulkHttpRequest.Redirects
maxRedirects := options.BulkHttpRequest.MaxRedirects
transport := &http.Transport{
MaxIdleConnsPerHost: -1,
@ -296,7 +284,7 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect
}
}
func (e *HTTPExecuter) setCustomHeaders(r *requests.CompiledHTTP) {
func (e *HTTPExecuter) setCustomHeaders(r *requests.HttpRequest) {
for _, customHeader := range e.customHeaders {
// This should be pre-computed somewhere and done only once
tokens := strings.Split(customHeader, ":")
@ -313,7 +301,7 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.CompiledHTTP) {
}
// for now supports only headers
func (e *HTTPExecuter) setDynamicValues(r *requests.CompiledHTTP, dynamicValues map[string]string) {
func (e *HTTPExecuter) setDynamicValues(r *requests.HttpRequest, dynamicValues map[string]string) {
for dk, dv := range dynamicValues {
// replace within header values
for k, v := range r.Request.Header {

2
v2/pkg/executer/output_http.go
View File

@ -10,7 +10,7 @@ import (
)
// writeOutputHTTP writes http output to streams
func (e *HTTPExecuter) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) {
func (e *HTTPExecuter) writeOutputHTTP(req *requests.HttpRequest, matcher *matchers.Matcher, extractorResults []string) {
URL := req.Request.URL.String()
if e.jsonOutput {

217
v2/pkg/requests/http-request.go → v2/pkg/requests/bulk-http-request.go
View File

@ -16,8 +16,8 @@ import (
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
)
// HTTPRequest contains a request to be made from a template
type HTTPRequest struct {
// BulkHTTPRequest contains a request to be made from a template
type BulkHTTPRequest struct {
Name string `yaml:"Name,omitempty"`
// AttackType is the attack type
// Sniper, PitchFork and ClusterBomb. Default is Sniper
@ -52,31 +52,37 @@ type HTTPRequest struct {
// MaxRedirects is the maximum number of redirects that should be followed.
MaxRedirects int `yaml:"max-redirects,omitempty"`
// Raw contains raw requests
Raw []string `yaml:"raw,omitempty"`
Raw []string `yaml:"raw,omitempty"`
positionPath int
positionRaw int
generator func(payloads map[string][]string) (out chan map[string]interface{})
currentPayloads map[string]interface{}
basePayloads map[string][]string
generatorChan chan map[string]interface{}
currentGeneratorValue map[string]interface{}
}
// GetMatchersCondition returns the condition for the matcher
func (r *HTTPRequest) GetMatchersCondition() matchers.ConditionType {
func (r *BulkHTTPRequest) GetMatchersCondition() matchers.ConditionType {
return r.matchersCondition
}
// SetMatchersCondition sets the condition for the matcher
func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
func (r *BulkHTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
r.matchersCondition = condition
}
// GetAttackType returns the attack
func (r *HTTPRequest) GetAttackType() generators.Type {
func (r *BulkHTTPRequest) GetAttackType() generators.Type {
return r.attackType
}
// SetAttackType sets the attack
func (r *HTTPRequest) SetAttackType(attack generators.Type) {
func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) {
r.attackType = attack
}
// MakeHTTPRequest creates a *http.Request from a request configuration
func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) {
func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, data string) (*HttpRequest, error) {
parsed, err := url.Parse(baseURL)
if err != nil {
return nil, err
@ -88,119 +94,102 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error
"Hostname": hostname,
}
if len(r.Raw) > 0 {
return r.makeHTTPRequestFromRaw(baseURL, values)
// if data contains \n it's a raw request
if strings.Contains(data, "\n") {
return r.makeHTTPRequestFromRaw(baseURL, data, values)
}
return r.makeHTTPRequestFromModel(baseURL, values)
return r.makeHTTPRequestFromModel(baseURL, data, values)
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
requests = make(chan *CompiledHTTP)
func (r *BulkHTTPRequest) makeHTTPRequestFromModel(baseURL string, data string, values map[string]interface{}) (*HttpRequest, error) {
replacer := newReplacer(values)
URL := replacer.Replace(data)
// request generator
go func() {
defer close(requests)
for _, path := range r.Path {
// process base request
replacer := newReplacer(values)
// Build a request on the specified URL
req, err := http.NewRequest(r.Method, URL, nil)
if err != nil {
return nil, err
}
// Replace the dynamic variables in the URL if any
URL := replacer.Replace(path)
request, err := r.fillRequest(req, values)
if err != nil {
return nil, err
}
// Build a request on the specified URL
req, err := http.NewRequest(r.Method, URL, nil)
if err != nil {
requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return
}
return &HttpRequest{Request: request}, nil
}
request, err := r.fillRequest(req, values)
if err != nil {
requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return
}
func (r *BulkHTTPRequest) StartGenerator() {
r.generatorChan = r.generator(r.basePayloads)
}
requests <- &CompiledHTTP{Request: request, Error: nil, Meta: nil}
}
}()
return
func (r *BulkHTTPRequest) PickOne() {
var ok bool
r.currentGeneratorValue, ok = <-r.generatorChan
if !ok {
r.generator = nil
}
}
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
requests = make(chan *CompiledHTTP)
// request generator
go func() {
defer close(requests)
for _, raw := range r.Raw {
// Add trailing line
raw += "\n"
if len(r.Payloads) > 0 {
basePayloads := generators.LoadPayloads(r.Payloads)
generatorFunc := generators.SniperGenerator
switch r.attackType {
case generators.PitchFork:
generatorFunc = generators.PitchforkGenerator
case generators.ClusterBomb:
generatorFunc = generators.ClusterbombGenerator
}
for genValues := range generatorFunc(basePayloads) {
compiledHTTP := r.handleRawWithPaylods(raw, baseURL, values, genValues)
requests <- compiledHTTP
if compiledHTTP.Error != nil {
return
}
}
} else {
// otherwise continue with normal flow
compiledHTTP := r.handleSimpleRaw(raw, baseURL, values)
requests <- compiledHTTP
if compiledHTTP.Error != nil {
return
}
func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(baseURL string, data string, values map[string]interface{}) (*HttpRequest, error) {
// Add trailing line
data += "\n"
if len(r.Payloads) > 0 {
if r.generator == nil {
r.basePayloads = generators.LoadPayloads(r.Payloads)
generatorFunc := generators.SniperGenerator
switch r.attackType {
case generators.PitchFork:
generatorFunc = generators.PitchforkGenerator
case generators.ClusterBomb:
generatorFunc = generators.ClusterbombGenerator
}
r.generator = generatorFunc
r.StartGenerator()
}
}()
return requests, nil
r.PickOne()
return r.handleRawWithPaylods(data, baseURL, values, r.currentGeneratorValue)
}
// otherwise continue with normal flow
return r.handleSimpleRaw(data, baseURL, values)
}
func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) *CompiledHTTP {
func (r *BulkHTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) (*HttpRequest, error) {
// base request
replacer := newReplacer(values)
// Replace the dynamic variables in the request if any
raw = replacer.Replace(raw)
compiledRequest, err := r.parseRawRequest(raw, baseURL)
rawRequest, err := r.parseRawRequest(raw, baseURL)
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
req, err := http.NewRequest(compiledRequest.Method, compiledRequest.FullURL, strings.NewReader(compiledRequest.Data))
req, err := http.NewRequest(rawRequest.Method, rawRequest.FullURL, strings.NewReader(rawRequest.Data))
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
// copy headers
for key, value := range compiledRequest.Headers {
for key, value := range rawRequest.Headers {
req.Header[key] = []string{value}
}
request, err := r.fillRequest(req, values)
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
return &CompiledHTTP{Request: request, Error: nil, Meta: nil}
return &HttpRequest{Request: request}, nil
}
func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) *CompiledHTTP {
func (r *BulkHTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) (*HttpRequest, error) {
baseValues := generators.CopyMap(values)
finValues := generators.MergeMaps(baseValues, genValues)
@ -218,11 +207,11 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
result, err := compiled.Evaluate(finValues)
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
dynamicValues[expr] = result
}
@ -234,12 +223,12 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
compiledRequest, err := r.parseRawRequest(raw, baseURL)
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
req, err := http.NewRequest(compiledRequest.Method, compiledRequest.FullURL, strings.NewReader(compiledRequest.Data))
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
// copy headers
@ -249,13 +238,13 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
request, err := r.fillRequest(req, values)
if err != nil {
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
return nil, err
}
return &CompiledHTTP{Request: request, Error: nil, Meta: genValues}
return &HttpRequest{Request: request, Meta: genValues}, nil
}
func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
req.Header.Set("Connection", "close")
req.Close = true
replacer := newReplacer(values)
@ -290,10 +279,8 @@ func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface
return retryablehttp.FromRequest(req)
}
// CompiledHTTP contains Generated HTTP Request or error
type CompiledHTTP struct {
type HttpRequest struct {
Request *retryablehttp.Request
Error error
Meta map[string]interface{}
}
@ -311,7 +298,7 @@ func (c *CustomHeaders) Set(value string) error {
return nil
}
type compiledRawRequest struct {
type RawRequest struct {
FullURL string
Method string
Path string
@ -320,10 +307,10 @@ type compiledRawRequest struct {
}
// parseRawRequest parses the raw request as supplied by the user
func (r *HTTPRequest) parseRawRequest(request string, baseURL string) (*compiledRawRequest, error) {
func (r *BulkHTTPRequest) parseRawRequest(request string, baseURL string) (*RawRequest, error) {
reader := bufio.NewReader(strings.NewReader(request))
rawRequest := compiledRawRequest{
rawRequest := RawRequest{
Headers: make(map[string]string),
}
@ -406,3 +393,41 @@ func (r *HTTPRequest) parseRawRequest(request string, baseURL string) (*compiled
rawRequest.Data = string(b)
return &rawRequest, nil
}
func (r *BulkHTTPRequest) Next() bool {
if r.positionPath+r.positionRaw >= len(r.Path)+len(r.Raw) {
return false
}
return true
}
func (r *BulkHTTPRequest) Position() int {
return r.positionPath + r.positionRaw
}
func (r *BulkHTTPRequest) Reset() {
r.positionPath = 0
r.positionRaw = 0
}
func (r *BulkHTTPRequest) Current() string {
if r.positionPath <= len(r.Path) && len(r.Path) != 0 && r.positionRaw == 0 {
return r.Path[r.positionPath]
}
return r.Raw[r.positionRaw]
}
func (r *BulkHTTPRequest) Total() int {
return len(r.Path) + len(r.Raw)
}
func (r *BulkHTTPRequest) Increment() int {
if r.positionPath <= len(r.Path) && len(r.Path) != 0 && r.positionRaw == 0 {
r.positionPath++
return r.positionPath
}
// if we have payloads increment only when the generators are done
if r.generator == nil {
r.positionRaw++
}
return r.positionPath + r.positionRaw
}

4
v2/pkg/templates/compile.go
View File

@ -27,12 +27,12 @@ func Parse(file string) (*Template, error) {
defer f.Close()
// If no requests, and it is also not a workflow, return error.
if len(template.RequestsHTTP)+len(template.RequestsDNS) <= 0 {
if len(template.BulkRequestsHTTP)+len(template.RequestsDNS) <= 0 {
return nil, errors.New("No requests defined")
}
// Compile the matchers and the extractors for http requests
for _, request := range template.RequestsHTTP {
for _, request := range template.BulkRequestsHTTP {
// Get the condition between the matchers
condition, ok := matchers.ConditionTypes[request.MatchersCondition]
if !ok {

6
v2/pkg/templates/templates.go
View File

@ -10,9 +10,9 @@ type Template struct {
ID string `yaml:"id"`
// Info contains information about the template
Info Info `yaml:"info"`
// RequestHTTP contains the http request to make in the template
RequestsHTTP []*requests.HTTPRequest `yaml:"requests,omitempty"`
// RequestDNS contains the dns request to make in the template
// BulkRequestsHTTP contains the http request to make in the template
BulkRequestsHTTP []*requests.BulkHTTPRequest `yaml:"requests,omitempty"`
// RequestsDNS contains the dns request to make in the template
RequestsDNS []*requests.DNSRequest `yaml:"dns,omitempty"`
}

8
v2/pkg/workflows/var.go
View File

@ -53,12 +53,12 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
var gotResult bool
for _, template := range n.Templates {
if template.HTTPOptions != nil {
for _, request := range template.HTTPOptions.Template.RequestsHTTP {
for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
// apply externally supplied payloads if any
request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
// apply externally supplied payloads if any
request.Payloads = generators.MergeMaps(request.Payloads, externalVars)
template.HTTPOptions.HTTPRequest = request
template.HTTPOptions.BulkHttpRequest = request
httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions)
if err != nil {
gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
@ -70,7 +70,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
continue
}
if httpExecuter.GotResults() {
if httpExecuter.Results {
gotResult = true
n.addResults(&result)
}
@ -87,7 +87,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
continue
}
if dnsExecuter.GotResults() {
if dnsExecuter.Results {
gotResult = true
n.addResults(&result)
}