From a34b94e62f18d6920a984f3ee2cfaf91e4f10644 Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Fri, 9 Jun 2023 05:50:44 +0530 Subject: [PATCH] Issue 3339 headless fuzz (#3790) * Basic headless fuzzing * Remove debug statements * Add integration tests * Update template * Fix recognize payload value in matcher * Update tempalte * use req.SetURL() --------- Co-authored-by: Tarun Koyalwar --- integration_tests/fuzz/fuzz-headless.yaml | 31 +++++++++++++ v2/cmd/integration-test/fuzz.go | 27 ++++++++++-- v2/pkg/protocols/{http => common}/fuzz/doc.go | 0 .../{http => common}/fuzz/execute.go | 0 .../{http => common}/fuzz/execute_test.go | 0 .../protocols/{http => common}/fuzz/fuzz.go | 0 .../{http => common}/fuzz/fuzz_test.go | 0 .../protocols/{http => common}/fuzz/parts.go | 5 +-- .../{http => common}/fuzz/parts_test.go | 0 v2/pkg/protocols/headless/headless.go | 19 ++++++++ v2/pkg/protocols/headless/request.go | 43 ++++++++++++++++++- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/http/request.go | 2 +- 13 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 integration_tests/fuzz/fuzz-headless.yaml rename v2/pkg/protocols/{http => common}/fuzz/doc.go (100%) rename v2/pkg/protocols/{http => common}/fuzz/execute.go (100%) rename v2/pkg/protocols/{http => common}/fuzz/execute_test.go (100%) rename v2/pkg/protocols/{http => common}/fuzz/fuzz.go (100%) rename v2/pkg/protocols/{http => common}/fuzz/fuzz_test.go (100%) rename v2/pkg/protocols/{http => common}/fuzz/parts.go (96%) rename v2/pkg/protocols/{http => common}/fuzz/parts_test.go (100%) diff --git a/integration_tests/fuzz/fuzz-headless.yaml b/integration_tests/fuzz/fuzz-headless.yaml new file mode 100644 index 000000000..39d4bce69 --- /dev/null +++ b/integration_tests/fuzz/fuzz-headless.yaml @@ -0,0 +1,31 @@ +id: headless-query-fuzzing + +info: + name: Example Query Fuzzing + author: pdteam + severity: info + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + - action: waitload + + payloads: + redirect: + - "blog.com" + - "portal.com" + + fuzzing: + - part: query + mode: single + type: replace + fuzz: + - "https://{{redirect}}" + + matchers: + - type: word + part: body + words: + - "{{redirect}}" diff --git a/v2/cmd/integration-test/fuzz.go b/v2/cmd/integration-test/fuzz.go index 8fe4b9581..2a9573d23 100644 --- a/v2/cmd/integration-test/fuzz.go +++ b/v2/cmd/integration-test/fuzz.go @@ -13,9 +13,10 @@ import ( ) var fuzzingTestCases = map[string]testutils.TestCase{ - "fuzz/fuzz-mode.yaml": &fuzzModeOverride{}, - "fuzz/fuzz-type.yaml": &fuzzTypeOverride{}, - "fuzz/fuzz-query.yaml": &httpFuzzQuery{}, + "fuzz/fuzz-mode.yaml": &fuzzModeOverride{}, + "fuzz/fuzz-type.yaml": &fuzzTypeOverride{}, + "fuzz/fuzz-query.yaml": &httpFuzzQuery{}, + "fuzz/fuzz-headless.yaml": &HeadlessFuzzingQuery{}, } type httpFuzzQuery struct{} @@ -126,3 +127,23 @@ func (h *fuzzTypeOverride) Execute(filePath string) error { } return nil } + +// HeadlessFuzzingQuery tests fuzzing is working not in headless mode +type HeadlessFuzzingQuery struct{} + +// Execute executes a test case and returns an error if occurred +func (h *HeadlessFuzzingQuery) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + resp := fmt.Sprintf("%s", r.URL.Query().Get("url")) + fmt.Fprint(w, resp) + }) + ts := httptest.NewTLSServer(router) + defer ts.Close() + + got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless") + if err != nil { + return err + } + return expectResultsCount(got, 2) +} diff --git a/v2/pkg/protocols/http/fuzz/doc.go b/v2/pkg/protocols/common/fuzz/doc.go similarity index 100% rename from v2/pkg/protocols/http/fuzz/doc.go rename to v2/pkg/protocols/common/fuzz/doc.go diff --git a/v2/pkg/protocols/http/fuzz/execute.go b/v2/pkg/protocols/common/fuzz/execute.go similarity index 100% rename from v2/pkg/protocols/http/fuzz/execute.go rename to v2/pkg/protocols/common/fuzz/execute.go diff --git a/v2/pkg/protocols/http/fuzz/execute_test.go b/v2/pkg/protocols/common/fuzz/execute_test.go similarity index 100% rename from v2/pkg/protocols/http/fuzz/execute_test.go rename to v2/pkg/protocols/common/fuzz/execute_test.go diff --git a/v2/pkg/protocols/http/fuzz/fuzz.go b/v2/pkg/protocols/common/fuzz/fuzz.go similarity index 100% rename from v2/pkg/protocols/http/fuzz/fuzz.go rename to v2/pkg/protocols/common/fuzz/fuzz.go diff --git a/v2/pkg/protocols/http/fuzz/fuzz_test.go b/v2/pkg/protocols/common/fuzz/fuzz_test.go similarity index 100% rename from v2/pkg/protocols/http/fuzz/fuzz_test.go rename to v2/pkg/protocols/common/fuzz/fuzz_test.go diff --git a/v2/pkg/protocols/http/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go similarity index 96% rename from v2/pkg/protocols/http/fuzz/parts.go rename to v2/pkg/protocols/common/fuzz/parts.go index 83d9992b6..9c0c43025 100644 --- a/v2/pkg/protocols/http/fuzz/parts.go +++ b/v2/pkg/protocols/common/fuzz/parts.go @@ -73,10 +73,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL, req.Header.Set("User-Agent", uarand.GetRandom()) } else { req = input.BaseRequest.Clone(context.TODO()) - //TODO: abstract below 3 lines with `req.UpdateURL(xx *urlutil.URL)` - req.URL = parsed - req.Request.URL = parsed.URL - req.Update() + req.SetURL(parsed) } request := GeneratedRequest{ Request: req, diff --git a/v2/pkg/protocols/http/fuzz/parts_test.go b/v2/pkg/protocols/common/fuzz/parts_test.go similarity index 100% rename from v2/pkg/protocols/http/fuzz/parts_test.go rename to v2/pkg/protocols/common/fuzz/parts_test.go diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 869a028c0..d4ba60b4f 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -7,6 +7,7 @@ import ( useragent "github.com/projectdiscovery/nuclei/v2/pkg/model/types/userAgent" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" fileutil "github.com/projectdiscovery/utils/file" @@ -54,6 +55,9 @@ type Request struct { // cache any variables that may be needed for operation. options *protocols.ExecutorOptions generator *generators.PayloadGenerator + + // Fuzzing describes schema to fuzz headless requests + Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" json:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz headless requests"` } // RequestPartDefinitions contains a mapping of request part definitions and their @@ -129,6 +133,21 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.CompiledOperators = compiled } request.options = options + + if len(request.Fuzzing) > 0 { + for _, rule := range request.Fuzzing { + if fuzzingMode := options.Options.FuzzingMode; fuzzingMode != "" { + rule.Mode = fuzzingMode + } + if fuzzingType := options.Options.FuzzingType; fuzzingType != "" { + rule.Type = fuzzingType + } + if err := rule.Compile(request.generator, request.options); err != nil { + return errors.Wrap(err, "could not compile fuzzing rule") + } + } + } + return nil } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index e8ee75a5d..b58b1c1ae 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -1,6 +1,7 @@ package headless import ( + "io" "net/url" "strings" "time" @@ -12,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" @@ -19,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + urlutil "github.com/projectdiscovery/utils/url" ) var _ protocols.Request = &Request{} @@ -51,7 +54,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, gotmatches = results.OperatorsResult.Matched } } - + // verify if fuzz elaboration was requested + if len(request.Fuzzing) > 0 { + return request.executeFuzzingRule(inputURL, payloads, previous, wrappedCallback) + } if request.generator != nil { iterator := request.generator.NewIterator() for { @@ -166,3 +172,38 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols. gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse) } } + +// executeFuzzingRule executes a fuzzing rule in the template request +func (request *Request) executeFuzzingRule(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + // check for operator matches by wrapping callback + gotmatches := false + fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool { + if gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) { + return true + } + if err := request.executeRequestWithPayloads(gr.Request.URL.String(), gr.DynamicValues, previous, callback); err != nil { + return false + } + return true + } + + parsedURL, err := urlutil.Parse(inputURL) + if err != nil { + return errors.Wrap(err, "could not parse url") + } + for _, rule := range request.Fuzzing { + err := rule.Execute(&fuzz.ExecuteRuleInput{ + URL: parsedURL, + Callback: fuzzRequestCallback, + Values: payloads, + BaseRequest: nil, + }) + if err == io.EOF { + return nil + } + if err != nil { + return errors.Wrap(err, "could not execute rule") + } + } + return nil +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index cca76f512..a593d0a89 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -11,8 +11,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/fuzz" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index df008ac11..de1ffedd8 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -24,12 +24,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/fuzz" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"