diff --git a/go.mod b/go.mod index ba73258d5..471211d02 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 + github.com/elastic/go-lumber v0.1.0 github.com/karrick/godirwalk v1.15.6 github.com/miekg/dns v1.1.29 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index b2699167c..c8f2b1c5d 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8L github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-lumber v0.1.0 h1:HUjpyg36v2HoKtXlEC53EJ3zDFiDRn65d7B8dBHNius= +github.com/elastic/go-lumber v0.1.0/go.mod h1:8YvjMIRYypWuPvpxx7WoijBYdbB7XIh/9FqSYQZTtxQ= github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= diff --git a/pkg/executor/executer_http.go b/pkg/executor/executer_http.go index c09a4511c..5b1a92f07 100644 --- a/pkg/executor/executer_http.go +++ b/pkg/executor/executer_http.go @@ -78,7 +78,11 @@ func (e *HTTPExecutor) ExecuteHTTP(URL string) error { // Send the request to the target servers mainLoop: - for req := range compiledRequest { + for compiledRequest := range compiledRequest { + if compiledRequest.Error != nil { + return errors.Wrap(err, "could not make http request") + } + req := compiledRequest.Request resp, err := e.httpClient.Do(req) if err != nil { if resp != nil { diff --git a/pkg/generators/dsl.go b/pkg/generators/dsl.go index 929a85065..87f51f73c 100644 --- a/pkg/generators/dsl.go +++ b/pkg/generators/dsl.go @@ -2,6 +2,7 @@ package generators import ( "crypto/md5" + "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -85,7 +86,14 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { return hex.EncodeToString(hash[:]), nil } functions["sha256"] = func(args ...interface{}) (interface{}, error) { - return sha256.Sum256([]byte(args[0].(string))), nil + h := sha256.New() + h.Write([]byte(args[0].(string))) + return hex.EncodeToString(h.Sum(nil)), nil + } + functions["sha1"] = func(args ...interface{}) (interface{}, error) { + h := sha1.New() + h.Write([]byte(args[0].(string))) + return hex.EncodeToString(h.Sum(nil)), nil } // search functions["contains"] = func(args ...interface{}) (interface{}, error) { diff --git a/pkg/generators/util.go b/pkg/generators/util.go index 144f273dd..82dd63736 100644 --- a/pkg/generators/util.go +++ b/pkg/generators/util.go @@ -106,3 +106,12 @@ func StringContainsAnyMapItem(m map[string]interface{}, s string) bool { func TrimDelimiters(s string) string { return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}") } + +// FileExists checks if a file exists and is not a directory +func FileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/pkg/requests/http-request.go b/pkg/requests/http-request.go index c5326f9cb..b683caa64 100644 --- a/pkg/requests/http-request.go +++ b/pkg/requests/http-request.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "io/ioutil" - "log" "net/http" "net/url" "regexp" @@ -74,7 +73,7 @@ func (r *HTTPRequest) SetAttackType(attack generators.Type) { } // MakeHTTPRequest creates a *http.Request from a request configuration -func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *retryablehttp.Request, error) { +func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) { parsed, err := url.Parse(baseURL) if err != nil { return nil, err @@ -94,8 +93,8 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *retryablehttp.Reque } // MakeHTTPRequestFromModel creates a *http.Request from a request template -func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *retryablehttp.Request, err error) { - requests = make(chan *retryablehttp.Request) +func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) { + requests = make(chan *CompiledHTTP) // request generator go func() { @@ -110,17 +109,17 @@ func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string // Build a request on the specified URL req, err := http.NewRequest(r.Method, URL, nil) if err != nil { - // find a way to pass the error + requests <- &CompiledHTTP{Request: nil, Error: err} return } request, err := r.fillRequest(req, values) if err != nil { - // find a way to pass the error + requests <- &CompiledHTTP{Request: nil, Error: err} return } - requests <- request + requests <- &CompiledHTTP{Request: request, Error: nil} } }() @@ -128,8 +127,8 @@ func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string } // makeHTTPRequestFromRaw creates a *http.Request from a raw request -func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *retryablehttp.Request, err error) { - requests = make(chan *retryablehttp.Request) +func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) { + requests = make(chan *CompiledHTTP) // request generator go func() { defer close(requests) @@ -149,101 +148,20 @@ func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]i } for genValues := range generatorFunc(basePayloads) { - baseValues := generators.CopyMap(values) - finValues := generators.MergeMaps(baseValues, genValues) - - replacer := newReplacer(finValues) - - // Replace the dynamic variables in the URL if any - raw := replacer.Replace(raw) - - dynamicValues := make(map[string]interface{}) - // find all potentials tokens between {{}} - var re = regexp.MustCompile(`(?m)\{\{.+}}`) - for _, match := range re.FindAllString(raw, -1) { - // check if the match contains a dynamic variable - if generators.StringContainsAnyMapItem(finValues, match) { - expr := generators.TrimDelimiters(match) - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions()) - if err != nil { - // Debug only - Remove - log.Fatal(err) - } - result, err := compiled.Evaluate(finValues) - if err != nil { - // Debug only - Remove - log.Fatal(err) - } - dynamicValues[expr] = result - } - } - - // replace dynamic values - dynamicReplacer := newReplacer(dynamicValues) - raw = dynamicReplacer.Replace(raw) - - // log.Println(raw) - - // Build a parsed request from raw - parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw))) - if err != nil { + // otherwise continue with normal flow + compiledHTTP := r.handleRawWithPaylods(raw, baseURL, values, genValues) + requests <- compiledHTTP + if compiledHTTP.Error != nil { return } - - // requests generated from http.ReadRequest have incorrect RequestURI, so they - // cannot be used to perform another request directly, we need to generate a new one - // with the new target url - finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL) - req, err := http.NewRequest(r.Method, finalURL, parsedReq.Body) - if err != nil { - return - } - - // copy headers - req.Header = parsedReq.Header.Clone() - - request, err := r.fillRequest(req, values) - if err != nil { - return - } - - requests <- request } } else { // otherwise continue with normal flow - - // base request - replacer := newReplacer(values) - // Replace the dynamic variables in the request if any - raw = replacer.Replace(raw) - - // Build a parsed request from raw - parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw))) - if err != nil { - // find a way to pass the error + compiledHTTP := r.handleSimpleRaw(raw, baseURL, values) + requests <- compiledHTTP + if compiledHTTP.Error != nil { return } - - // requests generated from http.ReadRequest have incorrect RequestURI, so they - // cannot be used to perform another request directly, we need to generate a new one - // with the new target url - finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL) - req, err := http.NewRequest(r.Method, finalURL, parsedReq.Body) - if err != nil { - // find a way to pass the error - return - } - - // copy headers - req.Header = parsedReq.Header.Clone() - - request, err := r.fillRequest(req, values) - if err != nil { - // find a way to pass the error - return - } - - requests <- request } } }() @@ -251,7 +169,109 @@ func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]i return requests, nil } +func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) *CompiledHTTP { + // base request + replacer := newReplacer(values) + // Replace the dynamic variables in the request if any + raw = replacer.Replace(raw) + + // Build a parsed request from raw + parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw))) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + + // requests generated from http.ReadRequest have incorrect RequestURI, so they + // cannot be used to perform another request directly, we need to generate a new one + // with the new target url + finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL) + req, err := http.NewRequest(parsedReq.Method, finalURL, parsedReq.Body) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + + // copy headers + req.Header = parsedReq.Header.Clone() + + request, err := r.fillRequest(req, values) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + + return &CompiledHTTP{Request: request, Error: nil} +} + +func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) *CompiledHTTP { + baseValues := generators.CopyMap(values) + finValues := generators.MergeMaps(baseValues, genValues) + + replacer := newReplacer(finValues) + + // Replace the dynamic variables in the URL if any + raw = replacer.Replace(raw) + + dynamicValues := make(map[string]interface{}) + // find all potentials tokens between {{}} + var re = regexp.MustCompile(`(?m)\{\{.+}}`) + for _, match := range re.FindAllString(raw, -1) { + // check if the match contains a dynamic variable + if generators.StringContainsAnyMapItem(finValues, match) { + expr := generators.TrimDelimiters(match) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions()) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + result, err := compiled.Evaluate(finValues) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + dynamicValues[expr] = result + } + } + + // replace dynamic values + dynamicReplacer := newReplacer(dynamicValues) + raw = dynamicReplacer.Replace(raw) + + // Build a parsed request from raw + parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw))) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + + // Bug: http.ReadRequest does not process request body, so building it manually + // need to read from first \n\n till end + body := raw[strings.Index(raw, "\n\n"):] + + // requests generated from http.ReadRequest have incorrect RequestURI, so they + // cannot be used to perform another request directly, we need to generate a new one + // with the new target url + finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL) + req, err := http.NewRequest(parsedReq.Method, finalURL, strings.NewReader(body)) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + + // copy headers + req.Header = parsedReq.Header.Clone() + + request, err := r.fillRequest(req, values) + if err != nil { + return &CompiledHTTP{Request: nil, Error: err} + } + + return &CompiledHTTP{Request: request, Error: nil} +} + func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { + req.Header.Set("Connection", "close") + req.Close = true + + // raw requests are left untouched + if len(r.Raw) > 0 { + return retryablehttp.FromRequest(req) + } + replacer := newReplacer(values) // Check if the user requested a request body if r.Body != "" { @@ -274,8 +294,12 @@ func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface if _, ok := req.Header["Accept-Language"]; !ok { req.Header.Set("Accept-Language", "en") } - req.Header.Set("Connection", "close") - req.Close = true return retryablehttp.FromRequest(req) } + +// CompiledHTTP contains Generated HTTP Request or error +type CompiledHTTP struct { + Request *retryablehttp.Request + Error error +} diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index c39568011..9d7b2f3a2 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -1,6 +1,7 @@ package templates import ( + "fmt" "os" "github.com/projectdiscovery/nuclei/pkg/generators" @@ -34,6 +35,7 @@ func ParseTemplate(file string) (*Template, error) { request.SetMatchersCondition(condition) } + // Set the attack type - used only in raw requests attack, ok := generators.AttackTypes[request.AttackType] if !ok { request.SetAttackType(generators.Sniper) @@ -41,6 +43,13 @@ func ParseTemplate(file string) (*Template, error) { request.SetAttackType(attack) } + // Validate the payloads if any + for name, wordlist := range request.Payloads { + if !generators.FileExists(wordlist) { + return nil, fmt.Errorf("The %s file for payload %s does not exist", wordlist, name) + } + } + for _, matcher := range request.Matchers { if err = matcher.CompileMatchers(); err != nil { return nil, err