improving error/args handling

v2/pkg/protocols/common/expressions/variables.go

@@ -16,49 +16,66 @@ func ContainsUnresolvedVariables(items ...string) error {
 		if len(matches) == 0 {
 			return nil
 		}
-		errorString := &strings.Builder{}
+		var unresolvedVariables []string
-		errorString.WriteString("unresolved variables found: ")
+		for _, match := range matches {
-
-		for i, match := range matches {
 			if len(match) < 2 {
 				continue
 			}
-			errorString.WriteString(match[1])
+			unresolvedVariables = append(unresolvedVariables, match[1])
-			if i != len(matches)-1 {
-				errorString.WriteString(",")
-			}
 		}
-		errorMessage := errorString.String()
+		return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ","))
-		return errors.New(errorMessage)
 	}
 
 	return nil
 }
 
+// ContainsVariablesWithNames returns an error with variable names if the passed
+// input contains unresolved {{<pattern-here>}} variables within the provided list
 func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error {
 	for _, data := range items {
 		matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
 		if len(matches) == 0 {
 			return nil
 		}
-		errorString := &strings.Builder{}
+		var unresolvedVariables []string
-		errorString.WriteString("unresolved variables with values found: ")
+		for _, match := range matches {
-
-		for i, match := range matches {
 			if len(match) < 2 {
 				continue
 			}
 			matchName := match[1]
 			if _, ok := names[matchName]; !ok {
-				errorString.WriteString(matchName)
+				unresolvedVariables = append(unresolvedVariables, matchName)
-				if i != len(matches)-1 {
-					errorString.WriteString(",")
-				}
 			}
 		}
-		errorMessage := errorString.String()
+		return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
-		return errors.New(errorMessage)
 	}
 
 	return nil
 }
+
+// ContainsVariablesWithIgnoreList returns an error with variable names if the passed
+// input contains unresolved {{<pattern-here>}} other than the ones listed in the ignore list
+func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...string) error {
+	var unresolvedVariables []string
+	for _, data := range items {
+		matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
+		if len(matches) == 0 {
+			return nil
+		}
+		for _, match := range matches {
+			if len(match) < 2 {
+				continue
+			}
+			matchName := match[1]
+			if _, ok := skipNames[matchName]; ok {
+				continue
+			}
+			unresolvedVariables = append(unresolvedVariables, matchName)
+		}
+	}
+
+	if len(unresolvedVariables) > 0 {
+		return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
+	}
+	return nil
+}

v2/pkg/protocols/http/build_request.go

@@ -123,10 +123,22 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam
 			return nil, fmt.Errorf("malformed request supplied")
 		}
 
-		// the url might contain placeholders
+		payloads := generators.BuildPayloadFromOptions(r.request.options.Options)
-		parts[1] = replacer.Replace(parts[1], generators.BuildPayloadFromOptions(r.request.options.Options))
+		// in case cases (eg requests signing, some variables uses default values if missing)
-		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+		if defaultList := GetVariablesDefault(r.request.Signature.Value); defaultList != nil {
-			return nil, err
+			payloads = generators.MergeMaps(defaultList, payloads)
+		}
+		parts[1] = replacer.Replace(parts[1], payloads)
+
+		// the url might contain placeholders with ignore list
+		if ignoreList := GetVariablesNamesSkipList(r.request.Signature.Value); ignoreList != nil {
+			if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil {
+				return nil, err
+			}
+		} else { // the url might contain placeholders
+			if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+				return nil, err
+			}
 		}
 
 		parsed, err := url.Parse(parts[1])
@@ -135,7 +147,7 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam
 		}
 		values := generators.MergeMaps(
 			generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
-			generators.BuildPayloadFromOptions(r.request.options.Options),
+			payloads,
 		)
 
 		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)

v2/pkg/protocols/http/request.go

@@ -350,10 +350,16 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		}
 		dumpedRequestString := string(dumpedRequest)
 
-		// Check if are there any unresolved variables. If yes, skip unless overridden by user.
+		if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
-		if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
+			if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
-			gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
+				gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
-			return errStopExecution
+				return errStopExecution
+			}
+		} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
+			if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
+				gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
+				return errStopExecution
+			}
 		}
 	}
 	var formedURL string
@@ -562,9 +568,9 @@ func (request *Request) handleSignature(generatedRequest *generatedRequest) erro
 		awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey}
 		service := types.ToString(payloads["service"])
 		region := types.ToString(payloads["region"])
-		// if region is empty default to "us-east-2"
+		// if region is empty use default value
 		if region == "" {
-			region = "us-east-2"
+			region = types.ToString(signer.AwsDefaultVars["region"])
 		}
 		awsSignatureArguments := signer.AwsSignatureArguments{
 			Service: types.ToString(service),

v2/pkg/protocols/http/signature.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/alecthomas/jsonschema"
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
 )
 
 // SignatureType is the type of signature
@@ -82,3 +83,25 @@ func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) {
 func (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) {
 	return holder.Value.String(), nil
 }
+
+var ErrNoIgnoreList = errors.New("uknown signature types")
+
+// GetVariablesNamesSkipList depending on the signature type
+func GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {
+	switch signature {
+	case AWSSignature:
+		return signer.AwsSkipList
+	default:
+		return nil
+	}
+}
+
+// GetVariablesNamesSkipList depending on the signature type
+func GetVariablesDefault(signature SignatureType) map[string]interface{} {
+	switch signature {
+	case AWSSignature:
+		return signer.AwsDefaultVars
+	default:
+		return nil
+	}
+}

v2/pkg/protocols/http/signer/aws.go

@@ -58,9 +58,7 @@ func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) {
 	if creds == nil {
 		return nil, errors.New("couldn't create the credentials structure")
 	}
-
 	signer := v4.NewSigner(creds)
-
 	return &AwsSigner{creds: creds, signer: signer}, nil
 }
 
@@ -69,7 +67,8 @@ func NewAwsSignerFromEnv() (*AwsSigner, error) {
 	if creds == nil {
 		return nil, errors.New("couldn't create the credentials structure")
 	}
-	return &AwsSigner{creds: creds}, nil
+	signer := v4.NewSigner(creds)
+	return &AwsSigner{creds: creds, signer: signer}, nil
 }
 
 func NewAwsSignerFromFile() (*AwsSigner, error) {
@@ -77,7 +76,8 @@ func NewAwsSignerFromFile() (*AwsSigner, error) {
 	if creds == nil {
 		return nil, errors.New("couldn't create the credentials structure")
 	}
-	return &AwsSigner{creds: creds}, nil
+	signer := v4.NewSigner(creds)
+	return &AwsSigner{creds: creds, signer: signer}, nil
 }
 
 func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) error {
@@ -96,7 +96,6 @@ func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) er
 		request.Body.Close()
 		body = bytes.NewReader(bodyBytes)
 	}
-
 	if _, err := awsSigner.signer.Sign(request, body, signatureArgs.Service, signatureArgs.Region, signatureArgs.Time); err != nil {
 		return err
 	}
@@ -131,3 +130,11 @@ func (awsSigner *AwsSigner) checkSignatureArgs(args interface{}) (AwsSignatureAr
 func (awsSigner *AwsSigner) prepareRequest(request *http.Request) {
 	request.Header.Del("Host")
 }
+
+var AwsSkipList = map[string]interface{}{
+	"region": struct{}{},
+}
+
+var AwsDefaultVars = map[string]interface{}{
+	"region": "us-east-2",
+}

v2/pkg/protocols/http/signer/signer.go

@@ -23,11 +23,11 @@ func NewSigner(args SignerArgs) (signer Signer, err error) {
 	case AwsSignerArgs:
 		awsSigner, err := NewAwsSigner(signerArgs)
 		if err != nil {
-			// env variables
+			// $HOME/.aws/credentials
-			awsSigner, err = NewAwsSignerFromEnv()
+			awsSigner, err = NewAwsSignerFromFile()
 			if err != nil {
-				// $HOME/.aws/credentials
+				// env variables
-				awsSigner, err = NewAwsSignerFromFile()
+				awsSigner, err = NewAwsSignerFromEnv()
 				if err != nil {
 					return nil, err
 				}