From a4ca2021cdfdbc8fd7f561b1d5854837f793f976 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 9 Jun 2023 12:33:03 +0300 Subject: [PATCH] Add headless header and status matchers (#3794) * add headless header and status matchers * rename headers as header * add integration test for header+status * fix typo --- .../headless/headless-header-status-test.yaml | 24 ++++++++++++++ v2/cmd/integration-test/headless.go | 25 +++++++++++---- v2/pkg/protocols/headless/engine/page.go | 22 +++++++++++++ v2/pkg/protocols/headless/operators.go | 31 ++++++++++++++++++- v2/pkg/protocols/headless/request.go | 2 +- 5 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 integration_tests/headless/headless-header-status-test.yaml diff --git a/integration_tests/headless/headless-header-status-test.yaml b/integration_tests/headless/headless-header-status-test.yaml new file mode 100644 index 000000000..1b53d2252 --- /dev/null +++ b/integration_tests/headless/headless-header-status-test.yaml @@ -0,0 +1,24 @@ +id: headless-header-status-test + +info: + name: headless header + status test + author: pdteam + severity: info + +headless: + - steps: + - args: + url: "{{BaseURL}}" + action: navigate + - action: waitload + + matchers-condition: and + matchers: + - type: word + part: header + words: + - text/plain + + - type: status + status: + - 200 diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go index 2c4cf5066..e4c70543d 100644 --- a/v2/cmd/integration-test/headless.go +++ b/v2/cmd/integration-test/headless.go @@ -11,12 +11,13 @@ import ( ) var headlessTestcases = map[string]testutils.TestCase{ - "headless/headless-basic.yaml": &headlessBasic{}, - "headless/headless-header-action.yaml": &headlessHeaderActions{}, - "headless/headless-extract-values.yaml": &headlessExtractValues{}, - "headless/headless-payloads.yaml": &headlessPayloads{}, - "headless/variables.yaml": &headlessVariables{}, - "headless/file-upload.yaml": &headlessFileUpload{}, + "headless/headless-basic.yaml": &headlessBasic{}, + "headless/headless-header-action.yaml": &headlessHeaderActions{}, + "headless/headless-extract-values.yaml": &headlessExtractValues{}, + "headless/headless-payloads.yaml": &headlessPayloads{}, + "headless/variables.yaml": &headlessVariables{}, + "headless/file-upload.yaml": &headlessFileUpload{}, + "headless/headless-header-status-test.yaml": &headlessHeaderStatus{}, } type headlessBasic struct{} @@ -158,3 +159,15 @@ func (h *headlessFileUpload) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type headlessHeaderStatus struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessHeaderStatus) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-headless") + if err != nil { + return err + } + + return expectResultsCount(results, 1) +} diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index a3ad01964..96bbb15af 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -1,6 +1,7 @@ package engine import ( + "fmt" "net/url" "strings" "sync" @@ -78,10 +79,19 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string] return nil, nil, err } + //FIXME: this is a hack, make sure to fix this in the future. See: https://github.com/go-rod/rod/issues/188 + var e proto.NetworkResponseReceived + wait := page.WaitEvent(&e) + data, err := createdPage.ExecuteActions(baseURL, actions) if err != nil { return nil, nil, err } + + wait() + data["header"] = headersToString(e.Response.Headers) + data["status_code"] = fmt.Sprint(e.Response.Status) + return data, createdPage, nil } @@ -178,3 +188,15 @@ func containsAnyModificationActionType(actionTypes ...ActionType) bool { } return false } + +// headersToString converts network headers to string +func headersToString(headers proto.NetworkHeaders) string { + builder := &strings.Builder{} + for header, value := range headers { + builder.WriteString(header) + builder.WriteString(": ") + builder.WriteString(value.String()) + builder.WriteRune('\n') + } + return builder.String() +} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 9359fc30a..ddf5f672c 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -1,6 +1,7 @@ package headless import ( + "strconv" "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -20,6 +21,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat } switch matcher.GetType() { + case matchers.StatusMatcher: + statusCode, ok := getStatusCode(data) + if !ok { + return false, []string{} + } + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{} case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: @@ -34,6 +41,24 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return false, []string{} } +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCodeStr, ok := statusCodeValue.(string) + if !ok { + return 0, false + } + + statusCode, err := strconv.Atoi(statusCodeStr) + if err != nil { + return 0, false + } + + return statusCode, true +} + // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { itemStr, ok := request.getMatchPart(extractor.Part, data) @@ -58,6 +83,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st part = "data" case "history": part = "history" + case "header": + part = "header" } item, ok := data[part] @@ -70,12 +97,14 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st } // responseToDSLMap converts a headless response to a map for use in DSL matching -func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent { +func (request *Request) responseToDSLMap(resp, headers, status_code, req, host, matched string, history string) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, "req": req, "data": resp, + "header": headers, + "status_code": status_code, "history": history, "type": request.Type().String(), "template-id": request.options.TemplateID, diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index b58b1c1ae..814769d11 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -135,7 +135,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory()) + outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), inputURL, inputURL, page.DumpHistory()) for k, v := range out { outputEvent[k] = v }