Add headless header and status matchers (#3794)

* add headless header and status matchers

* rename headers as header

* add integration test for header+status

* fix typo

integration_tests/headless/headless-header-status-test.yaml

@@ -0,0 +1,24 @@
+id: headless-header-status-test
+
+info:
+  name: headless header + status test
+  author: pdteam
+  severity: info
+
+headless:
+  - steps:
+      - args:
+          url: "{{BaseURL}}"
+        action: navigate
+      - action: waitload
+
+    matchers-condition: and
+    matchers:
+      - type: word
+        part: header
+        words:
+          - text/plain
+
+      - type: status
+        status:
+          - 200

v2/cmd/integration-test/headless.go

@@ -11,12 +11,13 @@ import (
 )
 
 var headlessTestcases = map[string]testutils.TestCase{
-	"headless/headless-basic.yaml":          &headlessBasic{},
+	"headless/headless-basic.yaml":              &headlessBasic{},
-	"headless/headless-header-action.yaml":  &headlessHeaderActions{},
+	"headless/headless-header-action.yaml":      &headlessHeaderActions{},
-	"headless/headless-extract-values.yaml": &headlessExtractValues{},
+	"headless/headless-extract-values.yaml":     &headlessExtractValues{},
-	"headless/headless-payloads.yaml":       &headlessPayloads{},
+	"headless/headless-payloads.yaml":           &headlessPayloads{},
-	"headless/variables.yaml":               &headlessVariables{},
+	"headless/variables.yaml":                   &headlessVariables{},
-	"headless/file-upload.yaml":             &headlessFileUpload{},
+	"headless/file-upload.yaml":                 &headlessFileUpload{},
+	"headless/headless-header-status-test.yaml": &headlessHeaderStatus{},
 }
 
 type headlessBasic struct{}
@@ -158,3 +159,15 @@ func (h *headlessFileUpload) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type headlessHeaderStatus struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessHeaderStatus) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-headless")
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}

v2/pkg/protocols/headless/engine/page.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"net/url"
 	"strings"
 	"sync"
@@ -78,10 +79,19 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]
 		return nil, nil, err
 	}
 
+	//FIXME: this is a hack, make sure to fix this in the future. See: https://github.com/go-rod/rod/issues/188
+	var e proto.NetworkResponseReceived
+	wait := page.WaitEvent(&e)
+
 	data, err := createdPage.ExecuteActions(baseURL, actions)
 	if err != nil {
 		return nil, nil, err
 	}
+
+	wait()
+	data["header"] = headersToString(e.Response.Headers)
+	data["status_code"] = fmt.Sprint(e.Response.Status)
+
 	return data, createdPage, nil
 }
 
@@ -178,3 +188,15 @@ func containsAnyModificationActionType(actionTypes ...ActionType) bool {
 	}
 	return false
 }
+
+// headersToString converts network headers to string
+func headersToString(headers proto.NetworkHeaders) string {
+	builder := &strings.Builder{}
+	for header, value := range headers {
+		builder.WriteString(header)
+		builder.WriteString(": ")
+		builder.WriteString(value.String())
+		builder.WriteRune('\n')
+	}
+	return builder.String()
+}

v2/pkg/protocols/headless/operators.go

@@ -1,6 +1,7 @@
 package headless
 
 import (
+	"strconv"
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
@@ -20,6 +21,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 	}
 
 	switch matcher.GetType() {
+	case matchers.StatusMatcher:
+		statusCode, ok := getStatusCode(data)
+		if !ok {
+			return false, []string{}
+		}
+		return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{}
 	case matchers.SizeMatcher:
 		return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
 	case matchers.WordsMatcher:
@@ -34,6 +41,24 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 	return false, []string{}
 }
 
+func getStatusCode(data map[string]interface{}) (int, bool) {
+	statusCodeValue, ok := data["status_code"]
+	if !ok {
+		return 0, false
+	}
+	statusCodeStr, ok := statusCodeValue.(string)
+	if !ok {
+		return 0, false
+	}
+
+	statusCode, err := strconv.Atoi(statusCodeStr)
+	if err != nil {
+		return 0, false
+	}
+
+	return statusCode, true
+}
+
 // Extract performs extracting operation for an extractor on model and returns true or false.
 func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
 	itemStr, ok := request.getMatchPart(extractor.Part, data)
@@ -58,6 +83,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 		part = "data"
 	case "history":
 		part = "history"
+	case "header":
+		part = "header"
 	}
 
 	item, ok := data[part]
@@ -70,12 +97,14 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 }
 
 // responseToDSLMap converts a headless response to a map for use in DSL matching
-func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent {
+func (request *Request) responseToDSLMap(resp, headers, status_code, req, host, matched string, history string) output.InternalEvent {
 	return output.InternalEvent{
 		"host":          host,
 		"matched":       matched,
 		"req":           req,
 		"data":          resp,
+		"header":        headers,
+		"status_code":   status_code,
 		"history":       history,
 		"type":          request.Type().String(),
 		"template-id":   request.options.TemplateID,

v2/pkg/protocols/headless/request.go

@@ -135,7 +135,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
 		responseBody, _ = html.HTML()
 	}
 
-	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
+	outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
 	for k, v := range out {
 		outputEvent[k] = v
 	}