From d0d65f8d6b199077982420c03d45c1ba886061db Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 2 Apr 2022 02:14:00 +0530 Subject: [PATCH] Added integration tests for variables + misc changes --- integration_tests/dns/variables.yaml | 21 +++++++++++++ integration_tests/headless/variables.yaml | 20 +++++++++++++ integration_tests/http/variables.yaml | 26 ++++++++++++++++ integration_tests/network/variables.yaml | 23 +++++++++++++++ v2/cmd/integration-test/dns.go | 23 +++++++++++++-- v2/cmd/integration-test/headless.go | 19 ++++++++++++ v2/cmd/integration-test/network.go | 32 ++++++++++++++++++++ v2/pkg/protocols/dns/dns.go | 24 +-------------- v2/pkg/protocols/dns/dns_test.go | 2 +- v2/pkg/protocols/dns/operators.go | 2 +- v2/pkg/protocols/dns/request.go | 36 +++++++++++++++++++++-- v2/pkg/protocols/headless/operators.go | 2 +- v2/pkg/protocols/headless/request.go | 3 ++ v2/pkg/protocols/network/request.go | 17 ++++++----- 14 files changed, 211 insertions(+), 39 deletions(-) create mode 100644 integration_tests/dns/variables.yaml create mode 100644 integration_tests/headless/variables.yaml create mode 100644 integration_tests/http/variables.yaml create mode 100644 integration_tests/network/variables.yaml diff --git a/integration_tests/dns/variables.yaml b/integration_tests/dns/variables.yaml new file mode 100644 index 000000000..13b10bbb9 --- /dev/null +++ b/integration_tests/dns/variables.yaml @@ -0,0 +1,21 @@ +id: variables-example + +info: + name: Variables Example + author: pdteam + severity: info + +variables: + a1: "FQDN" + a2: "IN" + +dns: + - name: "{{a1}}" + type: A + class: inet + recursion: true + retries: 3 + matchers: + - type: word + words: + - "{{a2}}" \ No newline at end of file diff --git a/integration_tests/headless/variables.yaml b/integration_tests/headless/variables.yaml new file mode 100644 index 000000000..d64cba974 --- /dev/null +++ b/integration_tests/headless/variables.yaml @@ -0,0 +1,20 @@ +id: variables-example + +info: + name: Variables Example + author: pdteam + severity: info + +variables: + a1: "value" + +headless: + - steps: + - args: + url: "{{BaseURL}}" + action: navigate + - action: waitload + matchers: + - type: word + words: + - "{{a1}}" \ No newline at end of file diff --git a/integration_tests/http/variables.yaml b/integration_tests/http/variables.yaml new file mode 100644 index 000000000..97137f192 --- /dev/null +++ b/integration_tests/http/variables.yaml @@ -0,0 +1,26 @@ +id: variables-example + +info: + name: Variables Example + author: pdteam + severity: info + +variables: + a1: "value" + a2: "rand_base(5)" + +requests: + - raw: + - | + GET / HTTP/1.1 + Host: {{FQDN}} + Test: {{a1}} + Another: {{a2}} + + stop-at-first-match: true + matchers-condition: or + matchers: + - type: word + words: + - "{{a1}}" + - "{{a2}}" \ No newline at end of file diff --git a/integration_tests/network/variables.yaml b/integration_tests/network/variables.yaml new file mode 100644 index 000000000..3c6a06fd2 --- /dev/null +++ b/integration_tests/network/variables.yaml @@ -0,0 +1,23 @@ +id: variables-example + +info: + name: Variables Example + author: pdteam + severity: info + +variables: + a1: "Hostname" + a2: "PING" + a3: "PONG" + +network: + - host: + - "{{a1}}" + inputs: + - data: "{{a2}}\r\n" + read-size: 4 + matchers: + - type: word + part: data + words: + - "{{a3}}" \ No newline at end of file diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index e0068fa90..b980689e4 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -5,9 +5,10 @@ import ( ) var dnsTestCases = map[string]testutils.TestCase{ - "dns/basic.yaml": &dnsBasic{}, - "dns/ptr.yaml": &dnsPtr{}, - "dns/caa.yaml": &dnsCAA{}, + "dns/basic.yaml": &dnsBasic{}, + "dns/ptr.yaml": &dnsPtr{}, + "dns/caa.yaml": &dnsCAA{}, + "dns/variables.yaml": &dnsVariables{}, } type dnsBasic struct{} @@ -57,3 +58,19 @@ func (h *dnsCAA) Execute(filePath string) error { } return expectResultsCount(results, 1) } + +type dnsVariables struct{} + +// Execute executes a test case and returns an error if occurred +func (h *dnsVariables) Execute(filePath string) error { + var routerErr error + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + return expectResultsCount(results, 1) +} diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go index 5d1c28cfc..3f75c628a 100644 --- a/v2/cmd/integration-test/headless.go +++ b/v2/cmd/integration-test/headless.go @@ -14,6 +14,7 @@ var headlessTestcases = map[string]testutils.TestCase{ "headless/headless-header-action.yaml": &headlessHeaderActions{}, "headless/headless-extract-values.yaml": &headlessExtractValues{}, "headless/headless-payloads.yaml": &headlessPayloads{}, + "headless/variables.yaml": &headlessVariables{}, } type headlessBasic struct{} @@ -92,3 +93,21 @@ func (h *headlessPayloads) Execute(filePath string) error { return expectResultsCount(results, 4) } + +type headlessVariables struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessVariables) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("value")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + + return expectResultsCount(results, 1) +} diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 6ab1cac77..fc49b299f 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -11,6 +11,7 @@ var networkTestcases = map[string]testutils.TestCase{ "network/hex.yaml": &networkBasic{}, "network/multi-step.yaml": &networkMultiStep{}, "network/self-contained.yaml": &networkRequestSelContained{}, + "network/variables.yaml": &networkVariables{}, } const defaultStaticPort = 5431 @@ -116,3 +117,34 @@ func (h *networkRequestSelContained) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type networkVariables struct{} + +// Execute executes a test case and returns an error if occurred +func (h *networkVariables) Execute(filePath string) error { + var routerErr error + + ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { + defer conn.Close() + + data := make([]byte, 4) + if _, err := conn.Read(data); err != nil { + routerErr = err + return + } + if string(data) == "PING" { + _, _ = conn.Write([]byte("PONG")) + } + }) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + + return expectResultsCount(results, 1) +} diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 4d08c20f2..bb7ff8343 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -8,11 +8,9 @@ import ( "github.com/weppos/publicsuffix-go/publicsuffix" - "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/retryabledns" @@ -171,33 +169,13 @@ func (request *Request) Requests() int { } // Make returns the request to be sent for the protocol -func (request *Request) Make(host string) (*dns.Msg, error) { - isIP := iputil.IsIP(host) - switch { - case request.question == dns.TypePTR && isIP: - var err error - host, err = dns.ReverseAddr(host) - if err != nil { - return nil, err - } - default: - if isIP { - return nil, errors.New("cannot use IP address as DNS input") - } - host = dns.Fqdn(host) - } - +func (request *Request) Make(host string, vars map[string]interface{}) (*dns.Msg, error) { // Build a request on the specified URL req := new(dns.Msg) req.Id = dns.Id() req.RecursionDesired = *request.Recursion var q dns.Question - - vars := GenerateDNSVariables(host) - variablesMap := request.options.Variables.Evaluate(vars) - vars = generators.MergeMaps(variablesMap, vars) - final := replacer.Replace(request.Name, vars) q.Name = dns.Fqdn(final) diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index d5bfbdb15..c517df7c4 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -42,7 +42,7 @@ func TestDNSCompileMake(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") - req, err := request.Make("one.one.one.one") + req, err := request.Make("one.one.one.one", map[string]interface{}{}) require.Nil(t, err, "could not make dns request") require.Equal(t, "one.one.one.one.", req.Question[0].Name, "could not get correct dns question") } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 3045ddcaf..8233bfe25 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -34,7 +34,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 8bc7d3537..cab78eab1 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -4,12 +4,15 @@ import ( "encoding/hex" "net/url" + "github.com/miekg/dns" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" @@ -34,8 +37,17 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review domain = input } + var err error + domain, err = request.parseDNSInput(domain) + if err != nil { + return errors.Wrap(err, "could not build request") + } + vars := GenerateDNSVariables(domain) + variablesMap := request.options.Variables.Evaluate(vars) + vars = generators.MergeMaps(variablesMap, vars) + // Compile each request for the template based on the URL - compiledRequest, err := request.Make(domain) + compiledRequest, err := request.Make(domain, vars) if err != nil { request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) @@ -87,7 +99,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review for k, v := range previous { outputEvent[k] = v } - + for k, v := range vars { + outputEvent[k] = v + } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) // TODO: dynamic values are not supported yet @@ -100,6 +114,24 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } +func (request *Request) parseDNSInput(host string) (string, error) { + isIP := iputil.IsIP(host) + switch { + case request.question == dns.TypePTR && isIP: + var err error + host, err = dns.ReverseAddr(host) + if err != nil { + return "", err + } + default: + if isIP { + return "", errors.New("cannot use IP address as DNS input") + } + host = dns.Fqdn(host) + } + return host, nil +} + func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) { cliOptions := requestOptions.Options if cliOptions.Debug || cliOptions.DebugResponse { diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index b7ff03c37..88616d9c6 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -23,7 +23,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 7ce81a8a8..fbc46c65a 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -106,6 +106,9 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map for k, v := range out { outputEvent[k] = v } + for k, v := range payloads { + outputEvent[k] = v + } var event *output.InternalWrappedEvent if len(page.InteractshURLs) == 0 { diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ccba31a55..0be79b41d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -50,6 +50,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review for _, kv := range request.addresses { variables := generateNetworkVariables(address) + variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(variables, variables)) + variables = generators.MergeMaps(variablesMap, variables) actualAddress := replacer.Replace(kv.address, variables) if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { @@ -62,6 +64,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // executeAddress executes the request for an address func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address}) + payloads := generators.BuildPayloadFromOptions(request.options.Options) + if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) @@ -69,9 +74,6 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA return err } - variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address}) - payloads := generators.BuildPayloadFromOptions(request.options.Options) - if request.generator != nil { iterator := request.generator.NewIterator() @@ -101,9 +103,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac err error ) - variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(variables, payloads)) - payloads = generators.MergeMaps(variablesMap, payloads) - if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } @@ -126,6 +125,8 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} + interimValues := generators.MergeMaps(variables, payloads) + inputEvents := make(map[string]interface{}) for _, input := range request.Inputs { var data []byte @@ -149,7 +150,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac data = []byte(transformedData) } - finalData, dataErr := expressions.EvaluateByte(data, payloads) + finalData, dataErr := expressions.EvaluateByte(data, interimValues) if dataErr != nil { request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr) request.options.Progress.IncrementFailedRequestsBy(1) @@ -255,7 +256,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac for k, v := range previous { outputEvent[k] = v } - for k, v := range payloads { + for k, v := range interimValues { outputEvent[k] = v } for k, v := range inputEvents {