headless: automerge and other improvements (#3958)

* headless: automerge and other improvements

* fix typo in function signature

v2/pkg/protocols/headless/engine/instance.go

@@ -17,6 +17,7 @@ type Instance struct {
 
 	// redundant due to dependency cycle
 	interactsh *interactsh.Client
+	requestLog map[string]string // contains actual request that was sent
 }
 
 // NewInstance creates a new instance for the current browser.
@@ -35,7 +36,14 @@ func (b *Browser) NewInstance() (*Instance, error) {
 	// We use a custom sleeper that sleeps from 100ms to 500 ms waiting
 	// for an interaction. Used throughout rod for clicking, etc.
 	browser = browser.Sleeper(func() utils.Sleeper { return maxBackoffSleeper(10) })
-	return &Instance{browser: b, engine: browser}, nil
+	return &Instance{browser: b, engine: browser, requestLog: map[string]string{}}, nil
+}
+
+// returns a map of [template-defined-urls] -> [actual-request-sent]
+// Note: this does not include CORS or other requests while rendering that were not explicitly
+// specified in template
+func (i *Instance) GetRequestLog() map[string]string {
+	return i.requestLog
 }
 
 // Close closes all the tabs and pages for a browser instance

v2/pkg/protocols/headless/engine/page.go

@@ -134,7 +134,7 @@ func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads m
 		}
 	}
 
-	data, err := createdPage.ExecuteActions(input, actions)
+	data, err := createdPage.ExecuteActions(input, actions, payloads)
 	if err != nil {
 		return nil, nil, err
 	}

v2/pkg/protocols/headless/engine/page_actions.go

@@ -2,11 +2,8 @@ package engine
 
 import (
 	"context"
-	"net"
-	"net/url"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -19,17 +16,21 @@ import (
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
+	protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+	httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
 	"github.com/segmentio/ksuid"
 )
 
 var (
 	errinvalidArguments = errors.New("invalid arguments provided")
-	reUrlWithPort       = regexp.MustCompile(`{{BaseURL}}:(\d+)`)
 )
 
 const (
@@ -39,17 +40,13 @@ const (
 )
 
 // ExecuteActions executes a list of actions on a page.
-func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (map[string]string, error) {
+func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (map[string]string, error) {
-	baseURL, err := url.Parse(input.MetaInput.Input)
-	if err != nil {
-		return nil, err
-	}
-
 	outData := make(map[string]string)
+	var err error
 	for _, act := range actions {
 		switch act.ActionType.ActionType {
 		case ActionNavigate:
-			err = p.NavigateURL(act, outData, baseURL)
+			err = p.NavigateURL(act, outData, variables)
 		case ActionScript:
 			err = p.RunScript(act, outData)
 		case ActionClick:
@@ -237,25 +234,57 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error {
 }
 
 // NavigateURL executes an ActionLoadURL actions loading a URL for the page.
-func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error {
+func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[string]interface{}) error {
-	URL := p.getActionArgWithDefaultValues(action, "url")
+	// input <- is input url from cli
-	if URL == "" {
+	// target <- is the url from template (ex: {{BaseURL}}/test)
+	input, err := urlutil.Parse(p.input.MetaInput.Input)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("could not parse url %s", p.input.MetaInput.Input)
+	}
+	target := p.getActionArgWithDefaultValues(action, "url")
+	if target == "" {
 		return errinvalidArguments
 	}
 
-	// Handle the dynamic value substitution here.
+	// if target contains port ex: {{BaseURL}}:8080 use port specified in input
-	URL, parsed = baseURLWithTemplatePrefs(URL, parsed)
+	input, target = httputil.UpdateURLPortFromPayload(input, target)
-	if strings.HasSuffix(parsed.Path, "/") && strings.Contains(URL, "{{BaseURL}}/") {
+	hasTrailingSlash := httputil.HasTrailingSlash(target)
-		parsed.Path = strings.TrimSuffix(parsed.Path, "/")
-	}
-	parsedString := parsed.String()
-	final := replaceWithValues(URL, map[string]interface{}{
-		"Hostname": parsed.Hostname(),
-		"BaseURL":  parsedString,
-	})
 
-	if err := p.page.Navigate(final); err != nil {
+	// create vars from input url
-		return errors.Wrap(err, "could not navigate")
+	defaultReqVars := protocolutils.GenerateVariables(input, hasTrailingSlash, contextargs.GenerateVariables(p.input))
+	// merge all variables
+	// Note: ideally we should evaluate all available variables with reqvars
+	// but due to cyclic dependency between packages `engine` and `protocols`
+	// allvars are evaluated,merged and passed from headless package itself
+	// TODO: remove cyclic dependency between packages `engine` and `protocols`
+	allvars = generators.MergeMaps(allvars, defaultReqVars)
+
+	if vardump.EnableVarDump {
+		gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(allvars))
+	}
+
+	// Evaluate the target url with all variables
+	target, err = expressions.Evaluate(target, allvars)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("could not evaluate url %s", target)
+	}
+
+	reqURL, err := urlutil.ParseURL(target, true)
+	if err != nil {
+		return errorutil.NewWithTag("http", "failed to parse url %v while creating http request", target)
+	}
+
+	// ===== parameter automerge =====
+	// while merging parameters first preference is given to target params
+	finalparams := input.Params.Clone()
+	finalparams.Merge(reqURL.Params.Encode())
+	reqURL.Params = finalparams
+
+	// log all navigated requests
+	p.instance.requestLog[action.GetArg("url")] = reqURL.String()
+
+	if err := p.page.Navigate(reqURL.String()); err != nil {
+		return errorutil.NewWithErr(err).Msgf("could not navigate to url %s", reqURL.String())
 	}
 	return nil
 }
@@ -609,23 +638,6 @@ func selectorBy(selector string) rod.SelectorType {
 	}
 }
 
-// baseURLWithTemplatePrefs returns the url for BaseURL keeping
-// the template port and path preference over the user provided one.
-func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
-	// template port preference over input URL port if template has a port
-	matches := reUrlWithPort.FindAllStringSubmatch(data, -1)
-	if len(matches) == 0 {
-		return data, parsed
-	}
-	port := matches[0][1]
-	parsed.Host = net.JoinHostPort(parsed.Hostname(), port)
-	data = strings.ReplaceAll(data, ":"+port, "")
-	if parsed.Path == "" {
-		parsed.Path = "/"
-	}
-	return data, parsed
-}
-
 func (p *Page) getActionArg(action *Action, arg string) string {
 	return p.getActionArgWithValues(action, arg, nil)
 }

v2/pkg/protocols/headless/request.go

@@ -1,6 +1,7 @@
 package headless
 
 import (
+	"fmt"
 	"net/url"
 	"strings"
 	"time"
@@ -119,21 +120,31 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
 	}
 	defer page.Close()
 
+	reqLog := instance.GetRequestLog()
+	navigatedURL := request.getLastNavigationURLWithLog(reqLog) // also known as matchedURL if there is a match
+
 	request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil)
 	request.options.Progress.IncrementRequests()
-	gologger.Verbose().Msgf("Sent Headless request to %s", input.MetaInput.Input)
+	gologger.Verbose().Msgf("Sent Headless request to %s", navigatedURL)
 
 	reqBuilder := &strings.Builder{}
 	if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse {
-		gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input.MetaInput.Input)
+		gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, navigatedURL)
 
 		for _, act := range request.Steps {
-			actStepStr := act.String()
+			if act.ActionType.ActionType == engine.ActionNavigate {
-			actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", input.MetaInput.Input)
+				value := act.GetArg("url")
-			reqBuilder.WriteString("\t" + actStepStr + "\n")
+				if reqLog[value] != "" {
+					reqBuilder.WriteString(fmt.Sprintf("\tnavigate => %v\n", reqLog[value]))
+				} else {
+					reqBuilder.WriteString(fmt.Sprintf("%v not found in %v\n", value, reqLog))
+				}
+			} else {
+				actStepStr := act.String()
+				reqBuilder.WriteString("\t" + actStepStr + "\n")
+			}
 		}
 		gologger.Debug().Msgf(reqBuilder.String())
-
 	}
 
 	var responseBody string
@@ -142,7 +153,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
 		responseBody, _ = html.HTML()
 	}
 
-	outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory())
+	outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory())
 	for k, v := range out {
 		outputEvent[k] = v
 	}
@@ -215,3 +226,16 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
 	}
 	return nil
 }
+
+// getLastNaviationURL returns last successfully navigated URL
+func (request *Request) getLastNavigationURLWithLog(reqLog map[string]string) string {
+	for i := len(request.Steps) - 1; i >= 0; i-- {
+		if request.Steps[i].ActionType.ActionType == engine.ActionNavigate {
+			templateURL := request.Steps[i].GetArg("url")
+			if reqLog[templateURL] != "" {
+				return reqLog[templateURL]
+			}
+		}
+	}
+	return ""
+}

v2/pkg/protocols/http/build_request.go

@@ -17,8 +17,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/utils"
 	protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+	httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -97,8 +97,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	hasTrailingSlash := false
 	if !isRawRequest {
 		// if path contains port ex: {{BaseURL}}:8080 use port specified in reqData
-		parsed, reqData = utils.UpdateURLPortFromPayload(parsed, reqData)
+		parsed, reqData = httputil.UpdateURLPortFromPayload(parsed, reqData)
-		hasTrailingSlash = utils.HasTrailingSlash(reqData)
+		hasTrailingSlash = httputil.HasTrailingSlash(reqData)
 	}
 
 	// defaultreqvars are vars generated from request/input ex: {{baseURL}}, {{Host}} etc
@@ -362,13 +362,13 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
 		req.Body = bodyReader
 	}
 	if !r.request.Unsafe {
-		utils.SetHeader(req, "User-Agent", uarand.GetRandom())
+		httputil.SetHeader(req, "User-Agent", uarand.GetRandom())
 	}
 
 	// Only set these headers on non-raw requests
 	if len(r.request.Raw) == 0 && !r.request.Unsafe {
-		utils.SetHeader(req, "Accept", "*/*")
+		httputil.SetHeader(req, "Accept", "*/*")
-		utils.SetHeader(req, "Accept-Language", "en")
+		httputil.SetHeader(req, "Accept-Language", "en")
 	}
 
 	if !LeaveDefaultPorts {

v2/pkg/protocols/utils/http/requtils.go

@@ -1,4 +1,4 @@
-package utils
+package httputil
 
 import (
 	"regexp"

v2/pkg/protocols/utils/http/requtils_test.go

@@ -1,4 +1,4 @@
-package utils
+package httputil
 
 import (
 	"testing"