From 911045ae9a86d240732e9a760dedac2af9216893 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 21 Dec 2021 14:20:03 +0530 Subject: [PATCH 01/12] add stop at first match for interactsh matchers --- v2/internal/runner/runner.go | 2 +- v2/pkg/protocols/common/interactsh/interactsh.go | 14 ++++++++++++++ v2/pkg/protocols/http/build_request.go | 4 ++++ v2/pkg/protocols/network/request.go | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 56eb05360..2d9f28705 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -159,7 +159,7 @@ func New(options *types.Options) (*Runner, error) { opts.ColldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second opts.NoInteractsh = runner.options.NoInteractsh - + opts.StopAtFirstMatch = runner.options.StopAtFirstMatch interactshClient, err := interactsh.New(opts) if err != nil { gologger.Error().Msgf("Could not create interactsh client: %s", err) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 2781d8f52..fa83541c6 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -75,6 +75,8 @@ type Options struct { Debug bool NoInteractsh bool + + StopAtFirstMatch bool } const defaultMaxInteractionsCount = 5000 @@ -135,6 +137,9 @@ func (c *Client) firstTimeInitializeClient() error { c.interactsh = interactsh interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) { + if c.options.StopAtFirstMatch && c.matched { + return + } if c.options.Debug { debugPrintInteraction(interaction) } @@ -228,6 +233,11 @@ func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, [ return data, interactshURLs } +// SetStopAtFirstMatch sets StopAtFirstMatch for interactsh client options +func (c *Client) SetStopAtFirstMatch(stopAtFirstMatch bool) { + c.options.StopAtFirstMatch = stopAtFirstMatch +} + // MakeResultEventFunc is a result making function for nuclei type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent @@ -243,6 +253,9 @@ type RequestData struct { // RequestEvent is the event for a network request sent by nuclei. func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { for _, interactshURL := range interactshURLs { + if c.options.StopAtFirstMatch && c.matched { + break + } id := strings.TrimSuffix(interactshURL, c.dotHostname) interaction := c.interactions.Get(id) @@ -256,6 +269,7 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { for _, interaction := range interactions { if c.processInteractionForRequest(interaction, data) { c.interactions.Delete(id) + c.matched = true break } } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 70e212c70..e7882d905 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -61,6 +61,10 @@ func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues ma ctx := context.Background() if r.options.Interactsh != nil { + if r.options.StopAtFirstMatch || r.request.StopAtFirstMatch { + r.options.Interactsh.SetStopAtFirstMatch(true) + } + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) for payloadName, payloadValue := range payloads { payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index b3cb5237f..55315d6c9 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -142,6 +142,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac reqBuilder.Grow(len(input.Data)) if request.options.Interactsh != nil { + if request.options.StopAtFirstMatch { + request.options.Interactsh.SetStopAtFirstMatch(true) + } var transformedData string transformedData, interactshURLs = request.options.Interactsh.ReplaceMarkers(string(data), []string{}) data = []byte(transformedData) From c0f9c1da70afe24d20c6445803bdd07e154c7dea Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 21 Dec 2021 15:24:16 +0530 Subject: [PATCH 02/12] interactsh stopAtFirstMatch intergration test --- .../http/interactsh-stop-at-first-match.yaml | 23 ++++++++++++++++++ v2/cmd/integration-test/http.go | 24 +++++++++++++++++++ .../protocols/common/interactsh/interactsh.go | 6 ++--- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/network/request.go | 2 +- 5 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 integration_tests/http/interactsh-stop-at-first-match.yaml diff --git a/integration_tests/http/interactsh-stop-at-first-match.yaml b/integration_tests/http/interactsh-stop-at-first-match.yaml new file mode 100644 index 000000000..415a634ea --- /dev/null +++ b/integration_tests/http/interactsh-stop-at-first-match.yaml @@ -0,0 +1,23 @@ +id: interactsh-stop-at-first-match-integration-test + +info: + name: Interactsh StopAtFirstMatch Integration Test + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + - "{{BaseURL}}" + - "{{BaseURL}}" + headers: + url: 'http://{{interactsh-url}}' + + stop-at-first-match: true + + matchers: + - type: word + part: interactsh_protocol # Confirms the HTTP Interaction + words: + - "http" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index be640ec34..a95cea558 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, "http/interactsh.yaml": &httpInteractshRequest{}, + "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, "http/self-contained.yaml": &httpRequestSelContained{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, @@ -67,6 +68,29 @@ func (h *httpInteractshRequest) Execute(filePath string) error { return expectResultsCount(results, 1) } +type httpInteractshStopAtFirstMatchRequest struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + value := r.Header.Get("url") + if value != "" { + if resp, _ := http.DefaultClient.Get(value); resp != nil { + resp.Body.Close() + } + } + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index fa83541c6..6660151e2 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -233,9 +233,9 @@ func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, [ return data, interactshURLs } -// SetStopAtFirstMatch sets StopAtFirstMatch for interactsh client options -func (c *Client) SetStopAtFirstMatch(stopAtFirstMatch bool) { - c.options.StopAtFirstMatch = stopAtFirstMatch +// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options +func (c *Client) SetStopAtFirstMatch() { + c.options.StopAtFirstMatch = true } // MakeResultEventFunc is a result making function for nuclei diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index e7882d905..161357351 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -62,7 +62,7 @@ func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues ma if r.options.Interactsh != nil { if r.options.StopAtFirstMatch || r.request.StopAtFirstMatch { - r.options.Interactsh.SetStopAtFirstMatch(true) + r.options.Interactsh.SetStopAtFirstMatch() } data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 55315d6c9..450350911 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -143,7 +143,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac if request.options.Interactsh != nil { if request.options.StopAtFirstMatch { - request.options.Interactsh.SetStopAtFirstMatch(true) + request.options.Interactsh.SetStopAtFirstMatch() } var transformedData string transformedData, interactshURLs = request.options.Interactsh.ReplaceMarkers(string(data), []string{}) From 0edb4274b1206172ebbcfea97f3c0cc5664161e6 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 22 Dec 2021 21:42:21 +0530 Subject: [PATCH 03/12] add matchedTemplates to support template wise stop at first match --- .../protocols/common/interactsh/interactsh.go | 23 +++++++++++++++---- v2/pkg/protocols/http/build_request.go | 5 +--- v2/pkg/protocols/http/operators.go | 4 ++++ v2/pkg/protocols/network/request.go | 6 ++--- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 6660151e2..0ee5fefdd 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -38,9 +38,10 @@ type Client struct { pollDuration time.Duration cooldownDuration time.Duration - firstTimeGroup sync.Once - generated uint32 // decide to wait if we have a generated url - matched bool + firstTimeGroup sync.Once + generated uint32 // decide to wait if we have a generated url + matched bool + matchedTemplates []string } var ( @@ -160,6 +161,10 @@ func (c *Client) firstTimeInitializeClient() error { if !ok { return } + + if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok && contains(c.matchedTemplates, request.Event.InternalEvent["template-id"].(string)) { + return + } _ = c.processInteractionForRequest(interaction, request) }) return nil @@ -189,6 +194,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched = true + c.matchedTemplates = append(c.matchedTemplates, data.Event.InternalEvent["template-id"].(string)) } return true } @@ -269,7 +275,6 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { for _, interaction := range interactions { if c.processInteractionForRequest(interaction, data) { c.interactions.Delete(id) - c.matched = true break } } @@ -324,3 +329,13 @@ func debugPrintInteraction(interaction *server.Interaction) { } fmt.Fprint(os.Stderr, builder.String()) } + +// contains returns true if the given string is in the given array +func contains(array []string, str string) bool { + for _, s := range array { + if s == str { + return true + } + } + return false +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 161357351..ce6a49c47 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -61,11 +61,8 @@ func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues ma ctx := context.Background() if r.options.Interactsh != nil { - if r.options.StopAtFirstMatch || r.request.StopAtFirstMatch { - r.options.Interactsh.SetStopAtFirstMatch() - } - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, []string{}) for payloadName, payloadValue := range payloads { payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 70fd4d4c7..0f4726d8f 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -125,6 +125,10 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo data["template-path"] = request.options.TemplatePath + + if request.StopAtFirstMatch || request.options.StopAtFirstMatch { + data["stop-at-first-match"] = true + } return data } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 450350911..37bb71b8a 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -142,9 +142,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac reqBuilder.Grow(len(input.Data)) if request.options.Interactsh != nil { - if request.options.StopAtFirstMatch { - request.options.Interactsh.SetStopAtFirstMatch() - } var transformedData string transformedData, interactshURLs = request.options.Interactsh.ReplaceMarkers(string(data), []string{}) data = []byte(transformedData) @@ -250,6 +247,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac response := responseBuilder.String() outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) outputEvent["ip"] = request.dialer.GetDialedIP(hostname) + if request.options.StopAtFirstMatch { + outputEvent["stop-at-first-match"] = true + } for k, v := range previous { outputEvent[k] = v } From e2b39e403cd73b184050c62d2cb0c0ef4585aa1a Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 29 Dec 2021 12:28:18 +0530 Subject: [PATCH 04/12] use ccache to store matchedTemplates with template-id+host hash as key --- .../protocols/common/interactsh/interactsh.go | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 0ee5fefdd..ca644608e 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -2,6 +2,8 @@ package interactsh import ( "bytes" + "crypto/sha1" + "encoding/hex" "fmt" "net/url" "os" @@ -32,16 +34,17 @@ type Client struct { requests *ccache.Cache // interactions is a stored cache for interactsh-interaction->interactsh-url data interactions *ccache.Cache + // matchedTemplates is a stored cache to track matched templates + matchedTemplates *ccache.Cache options *Options eviction time.Duration pollDuration time.Duration cooldownDuration time.Duration - firstTimeGroup sync.Once - generated uint32 // decide to wait if we have a generated url - matched bool - matchedTemplates []string + firstTimeGroup sync.Once + generated uint32 // decide to wait if we have a generated url + matched bool } var ( @@ -97,9 +100,12 @@ func New(options *Options) (*Client, error) { interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount) interactionsCache := ccache.New(interactionsCfg) + matchedTemplateCache := ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)) + interactClient := &Client{ eviction: options.Eviction, interactions: interactionsCache, + matchedTemplates: matchedTemplateCache, dotHostname: "." + parsed.Host, options: options, requests: cache, @@ -162,14 +168,24 @@ func (c *Client) firstTimeInitializeClient() error { return } - if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok && contains(c.matchedTemplates, request.Event.InternalEvent["template-id"].(string)) { - return + if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok { + gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent["template-id"].(string) + request.Event.InternalEvent["host"].(string))) + if gotItem != nil { + return + } } + _ = c.processInteractionForRequest(interaction, request) }) return nil } +func hash(s string) string { + h := sha1.New() + h.Write([]byte(s)) + return hex.EncodeToString(h.Sum(nil)) +} + // processInteractionForRequest processes an interaction for a request func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool { data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol @@ -194,7 +210,9 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched = true - c.matchedTemplates = append(c.matchedTemplates, data.Event.InternalEvent["template-id"].(string)) + if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok { + c.matchedTemplates.Set(hash(data.Event.InternalEvent["template-id"].(string)+data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) + } } return true } From 5115c8b58c21673839c6856c9a960fb5d555af78 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 29 Dec 2021 12:33:54 +0530 Subject: [PATCH 05/12] remove redundant contains function --- .../protocols/common/interactsh/interactsh.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index ca644608e..91f61d97d 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -180,12 +180,6 @@ func (c *Client) firstTimeInitializeClient() error { return nil } -func hash(s string) string { - h := sha1.New() - h.Write([]byte(s)) - return hex.EncodeToString(h.Sum(nil)) -} - // processInteractionForRequest processes an interaction for a request func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool { data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol @@ -348,12 +342,8 @@ func debugPrintInteraction(interaction *server.Interaction) { fmt.Fprint(os.Stderr, builder.String()) } -// contains returns true if the given string is in the given array -func contains(array []string, str string) bool { - for _, s := range array { - if s == str { - return true - } - } - return false +func hash(s string) string { + h := sha1.New() + h.Write([]byte(s)) + return hex.EncodeToString(h.Sum(nil)) } From 5d699cdde056fce24ec2da89538a307a2491d1d8 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 29 Dec 2021 09:48:46 +0100 Subject: [PATCH 06/12] Adding support for full navigation history to headless matchers --- v2/pkg/protocols/headless/engine/page.go | 16 +++++++++++++ v2/pkg/protocols/headless/engine/rules.go | 29 ++++++++++++++++++++++- v2/pkg/protocols/headless/operators.go | 5 +++- v2/pkg/protocols/headless/request.go | 2 +- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index afd59fd29..ba83d3db4 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -2,6 +2,7 @@ package engine import ( "net/url" + "strings" "time" "github.com/go-rod/rod" @@ -14,6 +15,12 @@ type Page struct { rules []requestRule instance *Instance router *rod.HijackRouter + History []HistoryData +} + +type HistoryData struct { + RawRequest string + RawResponse string } // Run runs a list of actions by creating a new page in the browser. @@ -81,3 +88,12 @@ func (p *Page) URL() string { } return info.URL } + +func (p *Page) DumpHistory() string { + var historyDump strings.Builder + for _, historyData := range p.History { + historyDump.WriteString(historyData.RawRequest) + historyDump.WriteString(historyData.RawResponse) + } + return historyDump.String() +} diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 8dc206876..17b66ee42 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -2,6 +2,8 @@ package engine import ( "fmt" + "net/http/httputil" + "strings" "github.com/go-rod/rod" ) @@ -10,7 +12,6 @@ import ( func (p *Page) routingRuleHandler(ctx *rod.Hijack) { // usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) - for _, rule := range p.rules { if rule.Part != "request" { continue @@ -51,4 +52,30 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { ctx.Response.SetBody(rule.Args["body"]) } } + + // store history + req := ctx.Request.Req() + var rawReq string + if raw, err := httputil.DumpRequestOut(req, true); err == nil { + rawReq = string(raw) + } + + // attempts to rebuild the response + var rawResp strings.Builder + respPayloads := ctx.Response.Payload() + if respPayloads != nil { + rawResp.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\n", respPayloads.ResponseCode, respPayloads.ResponsePhrase)) + for _, header := range respPayloads.ResponseHeaders { + rawResp.WriteString(fmt.Sprintf("%s: %s\n", header.Name, header.Value)) + } + rawResp.WriteString("\n") + rawResp.WriteString(ctx.Response.Body()) + } + + // dump request + historyData := HistoryData{ + RawRequest: rawReq, + RawResponse: rawResp.String(), + } + p.History = append(p.History, historyData) } diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index b9f9b4ccc..5d48371cf 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -54,6 +54,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st switch part { case "body", "resp", "": part = "data" + case "history": + part = "history" } item, ok := data[part] @@ -66,12 +68,13 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st } // responseToDSLMap converts a headless response to a map for use in DSL matching -func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, "req": req, "data": resp, + "history": history, "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 639909c4e..41c268843 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -66,7 +66,7 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o if err == nil { responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory()) for k, v := range out { outputEvent[k] = v } From 0a6b84639be2c86440dba7750303f8003f52f411 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 29 Dec 2021 09:51:50 +0100 Subject: [PATCH 07/12] adding missing comments --- v2/pkg/protocols/headless/engine/page.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index ba83d3db4..7c7ad1cc4 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -18,6 +18,7 @@ type Page struct { History []HistoryData } +// HistoryData contains the page request/response pairs type HistoryData struct { RawRequest string RawResponse string @@ -89,6 +90,7 @@ func (p *Page) URL() string { return info.URL } +// DumpHistory returns the full page navigation history func (p *Page) DumpHistory() string { var historyDump strings.Builder for _, historyData := range p.History { From d252c3d6ea302c32cc78e78a6a3b94d9039f52aa Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 29 Dec 2021 11:06:33 +0100 Subject: [PATCH 08/12] Adding support for automatic http fallback retry for interactsh --- v2/go.mod | 8 ++++++-- v2/go.sum | 20 ++++++++++++++++--- .../protocols/common/interactsh/interactsh.go | 6 +++++- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index cdbaeed61..110d10a57 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,7 +21,7 @@ require ( github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.43 + github.com/miekg/dns v1.1.45 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 @@ -33,7 +33,7 @@ require ( github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v0.0.7 + github.com/projectdiscovery/interactsh v0.0.8-0.20211229094250-de6074f67bcc github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 @@ -135,9 +135,13 @@ require ( github.com/ysmood/goob v0.3.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect + goftp.io/server/v2 v2.0.0 // indirect golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect + golang.org/x/mod v0.4.2 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/v2/go.sum b/v2/go.sum index 5422ea0c1..1df0f4408 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -411,6 +411,7 @@ github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZn github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -508,8 +509,11 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.45 h1:g5fRIhm9nx7g8osrAvgb16QJfmyMsyOCb+J7LSv+Qzk= +github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -624,8 +628,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1: github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= -github.com/projectdiscovery/interactsh v0.0.7 h1:X/qvJZR+qz/G7M8oNkbDVNeTxAY4T7aRqFIsSoa5mfA= -github.com/projectdiscovery/interactsh v0.0.7/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= +github.com/projectdiscovery/interactsh v0.0.8-0.20211229094250-de6074f67bcc h1:lLIFYiLz769CM7GUL/HHHbbMJ7akRcpjND21XVxJDm8= +github.com/projectdiscovery/interactsh v0.0.8-0.20211229094250-de6074f67bcc/go.mod h1:CEQrIYT1VttJ0aOzYxRcC1K/QxvFCdUIxO3q+UIp20Y= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= @@ -713,6 +717,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= @@ -856,6 +861,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +goftp.io/server/v2 v2.0.0 h1:FF8JKXXKDxAeO1uXEZz7G+IZwCDhl19dpVIlDtp3QAg= +goftp.io/server/v2 v2.0.0/go.mod h1:7+H/EIq7tXdfo1Muu5p+l3oQ6rYkDZ8lY7IM5d5kVdQ= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -863,6 +870,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -908,6 +916,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -925,6 +934,7 @@ golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -957,6 +967,7 @@ golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1127,6 +1138,8 @@ golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1239,6 +1252,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 2781d8f52..8b5b518ed 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -73,7 +73,9 @@ type Options struct { Progress progress.Progress // Debug specifies whether debugging output should be shown for interactsh-client Debug bool - + // HttpFallback controls http retry in case of https failure for server url + HttpFallback bool + // NoInteractsh disables the engine NoInteractsh bool } @@ -117,6 +119,7 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre Output: output, IssuesClient: reporting, Progress: progress, + HttpFallback: true, } } @@ -128,6 +131,7 @@ func (c *Client) firstTimeInitializeClient() error { ServerURL: c.options.ServerURL, Token: c.options.Authorization, PersistentSession: false, + HTTPFallback: c.options.HttpFallback, }) if err != nil { return errors.Wrap(err, "could not create client") From d0040e440f91aab7953df69f19a7b9e78e58b2df Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 29 Dec 2021 11:16:54 +0100 Subject: [PATCH 09/12] enabling fallback in tests --- v2/pkg/protocols/http/build_request_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 8b8e3f612..4e4890db9 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -236,6 +236,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { Eviction: time.Duration(options.InteractionsEviction) * time.Second, ColldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, + HttpFallback: true, }) require.Nil(t, err, "could not create interactsh client") From 8799379e504da7fbac81edc910dbaa58e5f11501 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 29 Dec 2021 18:07:48 +0530 Subject: [PATCH 10/12] update hash function --- v2/pkg/protocols/common/interactsh/interactsh.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 91f61d97d..690291952 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -169,7 +169,7 @@ func (c *Client) firstTimeInitializeClient() error { } if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok { - gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent["template-id"].(string) + request.Event.InternalEvent["host"].(string))) + gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent["template-id"].(string), request.Event.InternalEvent["host"].(string))) if gotItem != nil { return } @@ -205,7 +205,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched = true if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok { - c.matchedTemplates.Set(hash(data.Event.InternalEvent["template-id"].(string)+data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) + c.matchedTemplates.Set(hash(data.Event.InternalEvent["template-id"].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) } } return true @@ -342,8 +342,9 @@ func debugPrintInteraction(interaction *server.Interaction) { fmt.Fprint(os.Stderr, builder.String()) } -func hash(s string) string { +func hash(templateID, host string) string { h := sha1.New() - h.Write([]byte(s)) + h.Write([]byte(templateID)) + h.Write([]byte(host)) return hex.EncodeToString(h.Sum(nil)) } From 7251a2ef602b12bbe87c80be2df5a5120832f837 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 30 Dec 2021 12:59:42 +0100 Subject: [PATCH 11/12] implementing requested changes --- v2/pkg/protocols/headless/engine/page.go | 23 ++++++++++++++++++----- v2/pkg/protocols/headless/engine/rules.go | 8 +++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 7c7ad1cc4..dbe9935fe 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -3,6 +3,7 @@ package engine import ( "net/url" "strings" + "sync" "time" "github.com/go-rod/rod" @@ -11,11 +12,12 @@ import ( // Page is a single page in an isolated browser instance type Page struct { - page *rod.Page - rules []requestRule - instance *Instance - router *rod.HijackRouter - History []HistoryData + page *rod.Page + rules []requestRule + instance *Instance + router *rod.HijackRouter + historyMutex sync.RWMutex + History []HistoryData } // HistoryData contains the page request/response pairs @@ -92,6 +94,9 @@ func (p *Page) URL() string { // DumpHistory returns the full page navigation history func (p *Page) DumpHistory() string { + p.historyMutex.RLock() + defer p.historyMutex.RUnlock() + var historyDump strings.Builder for _, historyData := range p.History { historyDump.WriteString(historyData.RawRequest) @@ -99,3 +104,11 @@ func (p *Page) DumpHistory() string { } return historyDump.String() } + +// addToHistory adds a request/response pair to the page history +func (p *Page) addToHistory(historyData HistoryData) { + p.historyMutex.Lock() + defer p.historyMutex.Unlock() + + p.History = append(p.History, historyData) +} diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 17b66ee42..5aa0c91dd 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -64,9 +64,11 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { var rawResp strings.Builder respPayloads := ctx.Response.Payload() if respPayloads != nil { - rawResp.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\n", respPayloads.ResponseCode, respPayloads.ResponsePhrase)) + rawResp.WriteString("HTTP/1.1 ") + rawResp.WriteString(fmt.Sprint(respPayloads.ResponseCode)) + rawResp.WriteString(" " + respPayloads.ResponsePhrase + "+\n") for _, header := range respPayloads.ResponseHeaders { - rawResp.WriteString(fmt.Sprintf("%s: %s\n", header.Name, header.Value)) + rawResp.WriteString(header.Name + ": " + header.Value + "\n") } rawResp.WriteString("\n") rawResp.WriteString(ctx.Response.Body()) @@ -77,5 +79,5 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { RawRequest: rawReq, RawResponse: rawResp.String(), } - p.History = append(p.History, historyData) + p.addToHistory(historyData) } From 1670bf874b390adf4f909b03a8a00631537075b8 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 30 Dec 2021 13:04:08 +0100 Subject: [PATCH 12/12] fixing lint errors --- v2/pkg/protocols/headless/engine/page.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index dbe9935fe..6543a8fb2 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -16,7 +16,7 @@ type Page struct { rules []requestRule instance *Instance router *rod.HijackRouter - historyMutex sync.RWMutex + historyMutex *sync.RWMutex History []HistoryData } @@ -40,7 +40,7 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio } } - createdPage := &Page{page: page, instance: i} + createdPage := &Page{page: page, instance: i, historyMutex: &sync.RWMutex{}} router := page.HijackRequests() if routerErr := router.Add("*", "", createdPage.routingRuleHandler); routerErr != nil { return nil, nil, routerErr