solved many issues - yet to improve the output verbosity

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
+	github.com/elastic/go-lumber v0.1.0
 	github.com/karrick/godirwalk v1.15.6
 	github.com/miekg/dns v1.1.29
 	github.com/pkg/errors v0.9.1

go.sum

@@ -4,6 +4,8 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8L
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/go-lumber v0.1.0 h1:HUjpyg36v2HoKtXlEC53EJ3zDFiDRn65d7B8dBHNius=
+github.com/elastic/go-lumber v0.1.0/go.mod h1:8YvjMIRYypWuPvpxx7WoijBYdbB7XIh/9FqSYQZTtxQ=
 github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
 github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=

pkg/executor/executer_http.go

@@ -78,7 +78,11 @@ func (e *HTTPExecutor) ExecuteHTTP(URL string) error {
 
 	// Send the request to the target servers
 mainLoop:
-	for req := range compiledRequest {
+	for compiledRequest := range compiledRequest {
+		if compiledRequest.Error != nil {
+			return errors.Wrap(err, "could not make http request")
+		}
+		req := compiledRequest.Request
 		resp, err := e.httpClient.Do(req)
 		if err != nil {
 			if resp != nil {

pkg/generators/dsl.go

@@ -2,6 +2,7 @@ package generators
 
 import (
 	"crypto/md5"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
@@ -85,7 +86,14 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 		return hex.EncodeToString(hash[:]), nil
 	}
 	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
-		return sha256.Sum256([]byte(args[0].(string))), nil
+		h := sha256.New()
+		h.Write([]byte(args[0].(string)))
+		return hex.EncodeToString(h.Sum(nil)), nil
+	}
+	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
+		h := sha1.New()
+		h.Write([]byte(args[0].(string)))
+		return hex.EncodeToString(h.Sum(nil)), nil
 	}
 	// search
 	functions["contains"] = func(args ...interface{}) (interface{}, error) {

pkg/generators/util.go

@@ -106,3 +106,12 @@ func StringContainsAnyMapItem(m map[string]interface{}, s string) bool {
 func TrimDelimiters(s string) string {
 	return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}")
 }
+
+// FileExists checks if a file exists and is not a directory
+func FileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}

pkg/requests/http-request.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -74,7 +73,7 @@ func (r *HTTPRequest) SetAttackType(attack generators.Type) {
 }
 
 // MakeHTTPRequest creates a *http.Request from a request configuration
-func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *retryablehttp.Request, error) {
+func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) {
 	parsed, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
@@ -94,8 +93,8 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *retryablehttp.Reque
 }
 
 // MakeHTTPRequestFromModel creates a *http.Request from a request template
-func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *retryablehttp.Request, err error) {
+func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
-	requests = make(chan *retryablehttp.Request)
+	requests = make(chan *CompiledHTTP)
 
 	// request generator
 	go func() {
@@ -110,17 +109,17 @@ func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string
 			// Build a request on the specified URL
 			req, err := http.NewRequest(r.Method, URL, nil)
 			if err != nil {
-				// find a way to pass the error
+				requests <- &CompiledHTTP{Request: nil, Error: err}
 				return
 			}
 
 			request, err := r.fillRequest(req, values)
 			if err != nil {
-				// find a way to pass the error
+				requests <- &CompiledHTTP{Request: nil, Error: err}
 				return
 			}
 
-			requests <- request
+			requests <- &CompiledHTTP{Request: request, Error: nil}
 		}
 	}()
 
@@ -128,8 +127,8 @@ func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string
 }
 
 // makeHTTPRequestFromRaw creates a *http.Request from a raw request
-func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *retryablehttp.Request, err error) {
+func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
-	requests = make(chan *retryablehttp.Request)
+	requests = make(chan *CompiledHTTP)
 	// request generator
 	go func() {
 		defer close(requests)
@@ -149,101 +148,20 @@ func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]i
 				}
 
 				for genValues := range generatorFunc(basePayloads) {
-					baseValues := generators.CopyMap(values)
+					// otherwise continue with normal flow
-					finValues := generators.MergeMaps(baseValues, genValues)
+					compiledHTTP := r.handleRawWithPaylods(raw, baseURL, values, genValues)
-
+					requests <- compiledHTTP
-					replacer := newReplacer(finValues)
+					if compiledHTTP.Error != nil {
-
-					// Replace the dynamic variables in the URL if any
-					raw := replacer.Replace(raw)
-
-					dynamicValues := make(map[string]interface{})
-					// find all potentials tokens between {{}}
-					var re = regexp.MustCompile(`(?m)\{\{.+}}`)
-					for _, match := range re.FindAllString(raw, -1) {
-						// check if the match contains a dynamic variable
-						if generators.StringContainsAnyMapItem(finValues, match) {
-							expr := generators.TrimDelimiters(match)
-							compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
-							if err != nil {
-								// Debug only - Remove
-								log.Fatal(err)
-							}
-							result, err := compiled.Evaluate(finValues)
-							if err != nil {
-								// Debug only - Remove
-								log.Fatal(err)
-							}
-							dynamicValues[expr] = result
-						}
-					}
-
-					// replace dynamic values
-					dynamicReplacer := newReplacer(dynamicValues)
-					raw = dynamicReplacer.Replace(raw)
-
-					// log.Println(raw)
-
-					// Build a parsed request from raw
-					parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
-					if err != nil {
 						return
 					}
-
-					// requests generated from http.ReadRequest have incorrect RequestURI, so they
-					// cannot be used to perform another request directly, we need to generate a new one
-					// with the new target url
-					finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
-					req, err := http.NewRequest(r.Method, finalURL, parsedReq.Body)
-					if err != nil {
-						return
-					}
-
-					// copy headers
-					req.Header = parsedReq.Header.Clone()
-
-					request, err := r.fillRequest(req, values)
-					if err != nil {
-						return
-					}
-
-					requests <- request
 				}
 			} else {
 				// otherwise continue with normal flow
-
+				compiledHTTP := r.handleSimpleRaw(raw, baseURL, values)
-				// base request
+				requests <- compiledHTTP
-				replacer := newReplacer(values)
+				if compiledHTTP.Error != nil {
-				// Replace the dynamic variables in the request if any
-				raw = replacer.Replace(raw)
-
-				// Build a parsed request from raw
-				parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
-				if err != nil {
-					// find a way to pass the error
 					return
 				}
-
-				// requests generated from http.ReadRequest have incorrect RequestURI, so they
-				// cannot be used to perform another request directly, we need to generate a new one
-				// with the new target url
-				finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
-				req, err := http.NewRequest(r.Method, finalURL, parsedReq.Body)
-				if err != nil {
-					// find a way to pass the error
-					return
-				}
-
-				// copy headers
-				req.Header = parsedReq.Header.Clone()
-
-				request, err := r.fillRequest(req, values)
-				if err != nil {
-					// find a way to pass the error
-					return
-				}
-
-				requests <- request
 			}
 		}
 	}()
@@ -251,7 +169,109 @@ func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]i
 	return requests, nil
 }
 
+func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) *CompiledHTTP {
+	// base request
+	replacer := newReplacer(values)
+	// Replace the dynamic variables in the request if any
+	raw = replacer.Replace(raw)
+
+	// Build a parsed request from raw
+	parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
+	if err != nil {
+		return &CompiledHTTP{Request: nil, Error: err}
+	}
+
+	// requests generated from http.ReadRequest have incorrect RequestURI, so they
+	// cannot be used to perform another request directly, we need to generate a new one
+	// with the new target url
+	finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
+	req, err := http.NewRequest(parsedReq.Method, finalURL, parsedReq.Body)
+	if err != nil {
+		return &CompiledHTTP{Request: nil, Error: err}
+	}
+
+	// copy headers
+	req.Header = parsedReq.Header.Clone()
+
+	request, err := r.fillRequest(req, values)
+	if err != nil {
+		return &CompiledHTTP{Request: nil, Error: err}
+	}
+
+	return &CompiledHTTP{Request: request, Error: nil}
+}
+
+func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) *CompiledHTTP {
+	baseValues := generators.CopyMap(values)
+	finValues := generators.MergeMaps(baseValues, genValues)
+
+	replacer := newReplacer(finValues)
+
+	// Replace the dynamic variables in the URL if any
+	raw = replacer.Replace(raw)
+
+	dynamicValues := make(map[string]interface{})
+	// find all potentials tokens between {{}}
+	var re = regexp.MustCompile(`(?m)\{\{.+}}`)
+	for _, match := range re.FindAllString(raw, -1) {
+		// check if the match contains a dynamic variable
+		if generators.StringContainsAnyMapItem(finValues, match) {
+			expr := generators.TrimDelimiters(match)
+			compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
+			if err != nil {
+				return &CompiledHTTP{Request: nil, Error: err}
+			}
+			result, err := compiled.Evaluate(finValues)
+			if err != nil {
+				return &CompiledHTTP{Request: nil, Error: err}
+			}
+			dynamicValues[expr] = result
+		}
+	}
+
+	// replace dynamic values
+	dynamicReplacer := newReplacer(dynamicValues)
+	raw = dynamicReplacer.Replace(raw)
+
+	// Build a parsed request from raw
+	parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
+	if err != nil {
+		return &CompiledHTTP{Request: nil, Error: err}
+	}
+
+	// Bug: http.ReadRequest does not process request body, so building it manually
+	// need to read from first \n\n till end
+	body := raw[strings.Index(raw, "\n\n"):]
+
+	// requests generated from http.ReadRequest have incorrect RequestURI, so they
+	// cannot be used to perform another request directly, we need to generate a new one
+	// with the new target url
+	finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
+	req, err := http.NewRequest(parsedReq.Method, finalURL, strings.NewReader(body))
+	if err != nil {
+		return &CompiledHTTP{Request: nil, Error: err}
+	}
+
+	// copy headers
+	req.Header = parsedReq.Header.Clone()
+
+	request, err := r.fillRequest(req, values)
+	if err != nil {
+		return &CompiledHTTP{Request: nil, Error: err}
+	}
+
+	return &CompiledHTTP{Request: request, Error: nil}
+}
+
 func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
+	req.Header.Set("Connection", "close")
+	req.Close = true
+
+	// raw requests are left untouched
+	if len(r.Raw) > 0 {
+		return retryablehttp.FromRequest(req)
+	}
+
 	replacer := newReplacer(values)
 	// Check if the user requested a request body
 	if r.Body != "" {
@@ -274,8 +294,12 @@ func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface
 	if _, ok := req.Header["Accept-Language"]; !ok {
 		req.Header.Set("Accept-Language", "en")
 	}
-	req.Header.Set("Connection", "close")
-	req.Close = true
 
 	return retryablehttp.FromRequest(req)
 }
+
+// CompiledHTTP contains Generated HTTP Request or error
+type CompiledHTTP struct {
+	Request *retryablehttp.Request
+	Error   error
+}

pkg/templates/compile.go

@@ -1,6 +1,7 @@
 package templates
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/projectdiscovery/nuclei/pkg/generators"
@@ -34,6 +35,7 @@ func ParseTemplate(file string) (*Template, error) {
 			request.SetMatchersCondition(condition)
 		}
 
+		// Set the attack type - used only in raw requests
 		attack, ok := generators.AttackTypes[request.AttackType]
 		if !ok {
 			request.SetAttackType(generators.Sniper)
@@ -41,6 +43,13 @@ func ParseTemplate(file string) (*Template, error) {
 			request.SetAttackType(attack)
 		}
 
+		// Validate the payloads if any
+		for name, wordlist := range request.Payloads {
+			if !generators.FileExists(wordlist) {
+				return nil, fmt.Errorf("The %s file for payload %s does not exist", wordlist, name)
+			}
+		}
+
 		for _, matcher := range request.Matchers {
 			if err = matcher.CompileMatchers(); err != nil {
 				return nil, err