feat: auth file support enhancements for more complex scenarios + misc

internal/runner/lazy.go

@@ -92,8 +92,13 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
 			}
 			// dynamic values
 			for k, v := range e.OperatorsResult.DynamicValues {
-				if len(v) > 0 {
+				// Iterate through all the values and choose the
-					data[k] = v[0]
+				// largest value as the extracted value
+				for _, value := range v {
+					oldVal, ok := data[k]
+					if !ok || len(value) > len(oldVal.(string)) {
+						data[k] = value
+					}
 				}
 			}
 			// named extractors

pkg/authprovider/authx/cookies_auth.go

@@ -33,6 +33,9 @@ func (s *CookiesAuthStrategy) Apply(req *http.Request) {
 
 // ApplyOnRR applies the cookies auth strategy to the retryable request
 func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
+	// Before adding new cookies, remove existing cookies
+	req.Header.Del("Cookie")
+
 	for _, cookie := range s.Data.Cookies {
 		c := &http.Cookie{
 			Name:  cookie.Key,

pkg/authprovider/authx/dynamic.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 )
 
 type LazyFetchSecret func(d *Dynamic) error
@@ -22,7 +23,8 @@ var (
 // ex: username and password are dynamic secrets, the actual secret is the token obtained
 // after authenticating with the username and password
 type Dynamic struct {
-	Secret        `yaml:",inline"`       // this is a static secret that will be generated after the dynamic secret is resolved
+	*Secret       `yaml:",inline"`       // this is a static secret that will be generated after the dynamic secret is resolved
+	Secrets       []*Secret              `yaml:"secrets"`
 	TemplatePath  string                 `json:"template" yaml:"template"`
 	Variables     []KV                   `json:"variables" yaml:"variables"`
 	Input         string                 `json:"input" yaml:"input"` // (optional) target for the dynamic secret
@@ -33,6 +35,22 @@ type Dynamic struct {
 	error         error                  `json:"-" yaml:"-"` // error if any
 }
 
+func (d *Dynamic) GetDomainAndDomainRegex() ([]string, []string) {
+	var domains []string
+	var domainRegex []string
+	for _, secret := range d.Secrets {
+		domains = append(domains, secret.Domains...)
+		domainRegex = append(domainRegex, secret.DomainsRegex...)
+	}
+	if d.Secret != nil {
+		domains = append(domains, d.Secret.Domains...)
+		domainRegex = append(domainRegex, d.Secret.DomainsRegex...)
+	}
+	uniqueDomains := sliceutil.Dedupe(domains)
+	uniqueDomainRegex := sliceutil.Dedupe(domainRegex)
+	return uniqueDomains, uniqueDomainRegex
+}
+
 func (d *Dynamic) UnmarshalJSON(data []byte) error {
 	if err := json.Unmarshal(data, &d); err != nil {
 		return err
@@ -41,7 +59,7 @@ func (d *Dynamic) UnmarshalJSON(data []byte) error {
 	if err := json.Unmarshal(data, &s); err != nil {
 		return err
 	}
-	d.Secret = s
+	d.Secret = &s
 	return nil
 }
 
@@ -54,9 +72,18 @@ func (d *Dynamic) Validate() error {
 	if len(d.Variables) == 0 {
 		return errorutil.New("variables are required for dynamic secret")
 	}
-	d.skipCookieParse = true // skip cookie parsing in dynamic secrets during validation
+
-	if err := d.Secret.Validate(); err != nil {
+	if d.Secret != nil {
-		return err
+		d.Secret.skipCookieParse = true // skip cookie parsing in dynamic secrets during validation
+		if err := d.Secret.Validate(); err != nil {
+			return err
+		}
+	}
+	for _, secret := range d.Secrets {
+		secret.skipCookieParse = true
+		if err := secret.Validate(); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -74,76 +101,98 @@ func (d *Dynamic) SetLazyFetchCallback(callback LazyFetchSecret) {
 			return fmt.Errorf("no extracted values found for dynamic secret")
 		}
 
-		// evaluate headers
+		if d.Secret != nil {
-		for i, header := range d.Headers {
+			if err := d.applyValuesToSecret(d.Secret); err != nil {
-			if strings.Contains(header.Value, "{{") {
+				return err
-				header.Value = replacer.Replace(header.Value, d.Extracted)
 			}
-			if strings.Contains(header.Key, "{{") {
-				header.Key = replacer.Replace(header.Key, d.Extracted)
-			}
-			d.Headers[i] = header
 		}
 
-		// evaluate cookies
+		for _, secret := range d.Secrets {
-		for i, cookie := range d.Cookies {
+			if err := d.applyValuesToSecret(secret); err != nil {
-			if strings.Contains(cookie.Value, "{{") {
+				return err
-				cookie.Value = replacer.Replace(cookie.Value, d.Extracted)
-			}
-			if strings.Contains(cookie.Key, "{{") {
-				cookie.Key = replacer.Replace(cookie.Key, d.Extracted)
-			}
-			if strings.Contains(cookie.Raw, "{{") {
-				cookie.Raw = replacer.Replace(cookie.Raw, d.Extracted)
-			}
-			d.Cookies[i] = cookie
-		}
-
-		// evaluate query params
-		for i, query := range d.Params {
-			if strings.Contains(query.Value, "{{") {
-				query.Value = replacer.Replace(query.Value, d.Extracted)
-			}
-			if strings.Contains(query.Key, "{{") {
-				query.Key = replacer.Replace(query.Key, d.Extracted)
-			}
-			d.Params[i] = query
-		}
-
-		// check username, password and token
-		if strings.Contains(d.Username, "{{") {
-			d.Username = replacer.Replace(d.Username, d.Extracted)
-		}
-		if strings.Contains(d.Password, "{{") {
-			d.Password = replacer.Replace(d.Password, d.Extracted)
-		}
-		if strings.Contains(d.Token, "{{") {
-			d.Token = replacer.Replace(d.Token, d.Extracted)
-		}
-
-		// now attempt to parse the cookies
-		d.skipCookieParse = false
-		for i, cookie := range d.Cookies {
-			if cookie.Raw != "" {
-				if err := cookie.Parse(); err != nil {
-					return fmt.Errorf("[%s] invalid raw cookie in cookiesAuth: %s", d.TemplatePath, err)
-				}
-				d.Cookies[i] = cookie
 			}
 		}
 		return nil
 	}
 }
 
-// GetStrategy returns the auth strategy for the dynamic secret
+func (d *Dynamic) applyValuesToSecret(secret *Secret) error {
-func (d *Dynamic) GetStrategy() AuthStrategy {
+	// evaluate headers
+	for i, header := range secret.Headers {
+		if strings.Contains(header.Value, "{{") {
+			header.Value = replacer.Replace(header.Value, d.Extracted)
+		}
+		if strings.Contains(header.Key, "{{") {
+			header.Key = replacer.Replace(header.Key, d.Extracted)
+		}
+		secret.Headers[i] = header
+	}
+
+	// evaluate cookies
+	for i, cookie := range secret.Cookies {
+		if strings.Contains(cookie.Value, "{{") {
+			cookie.Value = replacer.Replace(cookie.Value, d.Extracted)
+		}
+		if strings.Contains(cookie.Key, "{{") {
+			cookie.Key = replacer.Replace(cookie.Key, d.Extracted)
+		}
+		if strings.Contains(cookie.Raw, "{{") {
+			cookie.Raw = replacer.Replace(cookie.Raw, d.Extracted)
+		}
+		secret.Cookies[i] = cookie
+	}
+
+	// evaluate query params
+	for i, query := range secret.Params {
+		if strings.Contains(query.Value, "{{") {
+			query.Value = replacer.Replace(query.Value, d.Extracted)
+		}
+		if strings.Contains(query.Key, "{{") {
+			query.Key = replacer.Replace(query.Key, d.Extracted)
+		}
+		secret.Params[i] = query
+	}
+
+	// check username, password and token
+	if strings.Contains(secret.Username, "{{") {
+		secret.Username = replacer.Replace(secret.Username, d.Extracted)
+	}
+	if strings.Contains(secret.Password, "{{") {
+		secret.Password = replacer.Replace(secret.Password, d.Extracted)
+	}
+	if strings.Contains(secret.Token, "{{") {
+		secret.Token = replacer.Replace(secret.Token, d.Extracted)
+	}
+
+	// now attempt to parse the cookies
+	secret.skipCookieParse = false
+	for i, cookie := range secret.Cookies {
+		if cookie.Raw != "" {
+			if err := cookie.Parse(); err != nil {
+				return fmt.Errorf("[%s] invalid raw cookie in cookiesAuth: %s", d.TemplatePath, err)
+			}
+			secret.Cookies[i] = cookie
+		}
+	}
+	return nil
+}
+
+// GetStrategy returns the auth strategies for the dynamic secret
+func (d *Dynamic) GetStrategies() []AuthStrategy {
 	if !d.fetched {
 		_ = d.Fetch(true)
 	}
 	if d.error != nil {
 		return nil
 	}
-	return d.Secret.GetStrategy()
+	var strategies []AuthStrategy
+	if d.Secret != nil {
+		strategies = append(strategies, d.Secret.GetStrategy())
+	}
+	for _, secret := range d.Secrets {
+		strategies = append(strategies, secret.GetStrategy())
+	}
+	return strategies
 }
 
 // Fetch fetches the dynamic secret

pkg/authprovider/authx/strategy.go

@@ -24,16 +24,16 @@ type DynamicAuthStrategy struct {
 
 // Apply applies the strategy to the request
 func (d *DynamicAuthStrategy) Apply(req *http.Request) {
-	strategy := d.Dynamic.GetStrategy()
+	strategy := d.Dynamic.GetStrategies()
-	if strategy != nil {
+	for _, s := range strategy {
-		strategy.Apply(req)
+		s.Apply(req)
 	}
 }
 
 // ApplyOnRR applies the strategy to the retryable request
 func (d *DynamicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
-	strategy := d.Dynamic.GetStrategy()
+	strategy := d.Dynamic.GetStrategies()
-	if strategy != nil {
+	for _, s := range strategy {
-		strategy.ApplyOnRR(req)
+		s.ApplyOnRR(req)
 	}
 }

pkg/authprovider/file.go

@@ -85,8 +85,10 @@ func (f *FileAuthProvider) init() {
 		}
 	}
 	for _, dynamic := range f.store.Dynamic {
-		if len(dynamic.DomainsRegex) > 0 {
+		domain, domainsRegex := dynamic.GetDomainAndDomainRegex()
-			for _, domain := range dynamic.DomainsRegex {
+
+		if len(domainsRegex) > 0 {
+			for _, domain := range domainsRegex {
 				if f.compiled == nil {
 					f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy)
 				}
@@ -101,7 +103,7 @@ func (f *FileAuthProvider) init() {
 				}
 			}
 		}
-		for _, domain := range dynamic.Domains {
+		for _, domain := range domain {
 			if f.domains == nil {
 				f.domains = make(map[string][]authx.AuthStrategy)
 			}