From 6f4b1ae48ad37061ee6dc908a5136ec1b100bf08 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 16 Apr 2023 19:49:35 +0200 Subject: [PATCH] Replacing ccache with generic gcache (#3523) * Replacing ccache with generic gcache * fixing lint issues * removing unecessary hashing + using errorutils * making test more tolerant * removing dead code + refactor * removing redundant code * removing race * maint * moving code * adding more iterations * note + typo * temporary fixing stop-at-first-match with interact * wrapping internal map with mux * sort before running integration test * fix deadlock in requestShouldStopAtFirstMatch * add timeout to integration_test workflow * attempting to remove outer lock * adds interactsh protocol tests in integration_test --------- Co-authored-by: Tarun Koyalwar --- .github/workflows/build-test.yml | 1 + .../http/interactsh-stop-at-first-match.yaml | 12 +- v2/cmd/integration-test/code.go | 2 +- v2/cmd/integration-test/http.go | 4 +- v2/cmd/integration-test/integration-test.go | 68 +++- v2/cmd/integration-test/interactsh.go | 10 + v2/examples/simple.go | 2 +- v2/go.mod | 3 +- v2/go.sum | 8 +- v2/internal/runner/runner.go | 4 +- v2/pkg/output/output.go | 7 + v2/pkg/protocols/common/interactsh/const.go | 18 ++ .../protocols/common/interactsh/interactsh.go | 305 +++++++----------- v2/pkg/protocols/common/interactsh/options.go | 67 ++++ v2/pkg/protocols/http/build_request_test.go | 2 +- v2/pkg/protocols/http/request.go | 42 +-- v2/pkg/utils/atomcache/atomcache.go | 62 ---- 17 files changed, 322 insertions(+), 295 deletions(-) create mode 100644 v2/cmd/integration-test/interactsh.go create mode 100644 v2/pkg/protocols/common/interactsh/const.go create mode 100644 v2/pkg/protocols/common/interactsh/options.go delete mode 100644 v2/pkg/utils/atomcache/atomcache.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ee969182e..644aa0674 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -42,6 +42,7 @@ jobs: working-directory: v2/ - name: Integration Tests + timeout-minutes: 50 env: GH_ACTION: true GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/integration_tests/http/interactsh-stop-at-first-match.yaml b/integration_tests/http/interactsh-stop-at-first-match.yaml index 415a634ea..b8063aadb 100644 --- a/integration_tests/http/interactsh-stop-at-first-match.yaml +++ b/integration_tests/http/interactsh-stop-at-first-match.yaml @@ -8,9 +8,15 @@ info: requests: - method: GET path: - - "{{BaseURL}}" - - "{{BaseURL}}" - - "{{BaseURL}}" + - "{{BaseURL}}/?a=1" + - "{{BaseURL}}/?a=2" + - "{{BaseURL}}/?a=3" + - "{{BaseURL}}/?a=4" + - "{{BaseURL}}/?a=5" + - "{{BaseURL}}/?a=6" + - "{{BaseURL}}/?a=7" + - "{{BaseURL}}/?a=8" + - "{{BaseURL}}/?a=9" headers: url: 'http://{{interactsh-url}}' diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index c6dd5f4f9..eb53be5c6 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -88,7 +88,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { defaultOpts.Templates = goflags.StringSlice{templatePath} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags - interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress) + interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) interactClient, err := interactsh.New(interactOpts) if err != nil { return nil, errors.Wrap(err, "could not create interact client") diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 545459cca..6a1997b1f 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -47,8 +47,6 @@ var httpTestcases = map[string]testutils.TestCase{ "http/http-paths.yaml": &httpPaths{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, - "http/interactsh.yaml": &httpInteractshRequest{}, - "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, "http/self-contained.yaml": &httpRequestSelfContained{}, "http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, @@ -71,7 +69,6 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-without-scheme.yaml": &httpGetWithoutScheme{}, "http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{}, "http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, - "http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{}, } type httpInteractshRequest struct{} @@ -164,6 +161,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error { if err != nil { return err } + // polling is asyncronous, so the interactions may be retrieved after the first request return expectResultsCount(results, 1) } diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 3d0411dec..73ee2a765 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "sort" "strings" "github.com/logrusorgru/aurora" @@ -22,6 +23,7 @@ var ( protocolTests = map[string]map[string]testutils.TestCase{ "http": httpTestcases, + "interactsh": interactshTestCases, "network": networkTestcases, "dns": dnsTestCases, "workflow": workflowTestcases, @@ -40,9 +42,10 @@ var ( } // For debug purposes - runProtocol = "" - runTemplate = "" - extraArgs = []string{} + runProtocol = "" + runTemplate = "" + extraArgs = []string{} + interactshRetryCount = 3 ) func main() { @@ -58,7 +61,6 @@ func main() { } if runProtocol != "" { - debug = true debugTests() os.Exit(1) } @@ -79,13 +81,36 @@ func main() { } } +// execute a testcase with retry and consider best of N +// intended for flaky tests like interactsh +func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) { + var err error + for i := 0; i < retryCount; i++ { + err = testCase.Execute(templatePath) + if err == nil { + fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath) + return "", nil + } + } + _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed after %v attempts : %s\n", failed, templatePath, retryCount, err) + return templatePath, err +} + func debugTests() { - for tpath, testcase := range protocolTests[runProtocol] { + keys := getMapKeys(protocolTests[runProtocol]) + for _, tpath := range keys { + testcase := protocolTests[runProtocol][tpath] if runTemplate != "" && !strings.Contains(tpath, runTemplate) { continue } - if err := testcase.Execute(tpath); err != nil { - fmt.Printf("\n%v", err.Error()) + if runProtocol == "interactsh" { + if _, err := executeWithRetry(testcase, tpath, interactshRetryCount); err != nil { + fmt.Printf("\n%v", err.Error()) + } + } else { + if _, err := execute(testcase, tpath); err != nil { + fmt.Printf("\n%v", err.Error()) + } } } } @@ -97,10 +122,19 @@ func runTests(customTemplatePaths []string) []string { if len(customTemplatePaths) == 0 { fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto)) } + keys := getMapKeys(testCases) - for templatePath, testCase := range testCases { + for _, templatePath := range keys { + testCase := testCases[templatePath] if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, templatePath) { - if failedTemplatePath, err := execute(testCase, templatePath); err != nil { + var failedTemplatePath string + var err error + if proto == "interactsh" { + failedTemplatePath, err = executeWithRetry(testCase, templatePath, interactshRetryCount) + } else { + failedTemplatePath, err = execute(testCase, templatePath) + } + if err != nil { failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath) } } @@ -120,9 +154,10 @@ func execute(testCase testutils.TestCase, templatePath string) (string, error) { return "", nil } -func expectResultsCount(results []string, expectedNumber int) error { - if len(results) != expectedNumber { - return fmt.Errorf("incorrect number of results: %d (actual) vs %d (expected) \nResults:\n\t%s\n", len(results), expectedNumber, strings.Join(results, "\n\t")) +func expectResultsCount(results []string, expectedNumbers ...int) error { + match := sliceutil.Contains(expectedNumbers, len(results)) + if !match { + return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) } return nil } @@ -132,3 +167,12 @@ func normalizeSplit(str string) []string { return r == ',' }) } + +func getMapKeys[T any](testcases map[string]T) []string { + keys := make([]string, 0, len(testcases)) + for k := range testcases { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/v2/cmd/integration-test/interactsh.go b/v2/cmd/integration-test/interactsh.go new file mode 100644 index 000000000..035f844d0 --- /dev/null +++ b/v2/cmd/integration-test/interactsh.go @@ -0,0 +1,10 @@ +package main + +import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + +// All Interactsh related testcases +var interactshTestCases = map[string]testutils.TestCase{ + "http/interactsh.yaml": &httpInteractshRequest{}, + "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, + "http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{}, +} diff --git a/v2/examples/simple.go b/v2/examples/simple.go index 129561fa3..5b80d2282 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -50,7 +50,7 @@ func main() { defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags - interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress) + interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) interactClient, err := interactsh.New(interactOpts) if err != nil { log.Fatalf("Could not create interact client: %s\n", err) diff --git a/v2/go.mod b/v2/go.mod index 8876bf025..4c0d3f6ea 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -18,7 +18,6 @@ require ( github.com/itchyny/gojq v0.12.11 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 - github.com/karlseguin/ccache v2.0.3+incompatible github.com/logrusorgru/aurora v2.0.3+incompatible github.com/miekg/dns v1.1.53 github.com/olekukonko/tablewriter v0.0.5 @@ -53,6 +52,7 @@ require ( require ( github.com/DataDog/gostackparse v0.6.0 + github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go-v2 v1.17.8 @@ -111,7 +111,6 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/karlseguin/expect v1.0.8 // indirect github.com/kataras/jwt v0.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect diff --git a/v2/go.sum b/v2/go.sum index 76e352aa8..e88a6ba17 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -10,6 +10,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo= @@ -290,11 +292,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= -github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= -github.com/karlseguin/expect v1.0.8 h1:Bb0H6IgBWQpadY25UDNkYPDB9ITqK1xnSoZfAq362fw= -github.com/karlseguin/expect v1.0.8/go.mod h1:lXdI8iGiQhmzpnnmU/EGA60vqKs8NbRNFnhhrJGoD5g= github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -582,8 +580,6 @@ github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2Z github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= -github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= -github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk= github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 2e339ed1a..ba467d159 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -262,14 +262,14 @@ func New(options *types.Options) (*Runner, error) { } runner.resumeCfg = resumeCfg - opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) + opts := interactsh.DefaultOptions(runner.output, runner.issuesClient, runner.progress) opts.Debug = runner.options.Debug opts.NoColor = runner.options.NoColor if options.InteractshURL != "" { opts.ServerURL = options.InteractshURL } opts.Authorization = options.InteractshToken - opts.CacheSize = int64(options.InteractionsCacheSize) + opts.CacheSize = options.InteractionsCacheSize opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second opts.CooldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index dba17992b..9d2895bc7 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -85,6 +85,13 @@ func (iwe *InternalWrappedEvent) HasOperatorResult() bool { return iwe.OperatorsResult != nil } +func (iwe *InternalWrappedEvent) HasResults() bool { + iwe.RLock() + defer iwe.RUnlock() + + return len(iwe.Results) > 0 +} + func (iwe *InternalWrappedEvent) SetOperatorResult(operatorResult *operators.Result) { iwe.Lock() defer iwe.Unlock() diff --git a/v2/pkg/protocols/common/interactsh/const.go b/v2/pkg/protocols/common/interactsh/const.go new file mode 100644 index 000000000..d574a121a --- /dev/null +++ b/v2/pkg/protocols/common/interactsh/const.go @@ -0,0 +1,18 @@ +package interactsh + +import ( + "regexp" + "time" +) + +var ( + defaultInteractionDuration = 60 * time.Second + interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`) +) + +const ( + stopAtFirstMatchAttribute = "stop-at-first-match" + templateIdAttribute = "template-id" + + defaultMaxInteractionsCount = 5000 +) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index d20d5c29e..94e559f93 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -2,8 +2,6 @@ package interactsh import ( "bytes" - "crypto/sha1" - "encoding/hex" "fmt" "os" "regexp" @@ -12,111 +10,56 @@ import ( "sync/atomic" "time" - "github.com/karlseguin/ccache" - "github.com/pkg/errors" + "errors" + + "github.com/Mzack9999/gcache" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/interactsh/pkg/client" "github.com/projectdiscovery/interactsh/pkg/server" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/projectdiscovery/nuclei/v2/pkg/utils/atomcache" - "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" ) // Client is a wrapped client for interactsh server. type Client struct { + sync.Once + sync.RWMutex + + options *Options + // interactsh is a client for interactsh server. interactsh *client.Client // requests is a stored cache for interactsh-url->request-event data. - requests *atomcache.Cache + requests gcache.Cache[string, *RequestData] // interactions is a stored cache for interactsh-interaction->interactsh-url data - interactions *atomcache.Cache + interactions gcache.Cache[string, []*server.Interaction] // matchedTemplates is a stored cache to track matched templates - matchedTemplates *atomcache.Cache - // interactshURLs is a stored cache to track track multiple interactsh markers - interactshURLs *atomcache.Cache + matchedTemplates gcache.Cache[string, bool] + // interactshURLs is a stored cache to track multiple interactsh markers + interactshURLs gcache.Cache[string, string] - options *Options eviction time.Duration pollDuration time.Duration cooldownDuration time.Duration - dataMutex *sync.RWMutex - hostname string - firstTimeGroup sync.Once - generated uint32 // decide to wait if we have a generated url - matched atomic.Bool + // determines if wait the cooldown period in case of generated URL + generated atomic.Bool + matched atomic.Bool } -var ( - defaultInteractionDuration = 60 * time.Second - interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`) -) - -const ( - stopAtFirstMatchAttribute = "stop-at-first-match" - templateIdAttribute = "template-id" -) - -// Options contains configuration options for interactsh nuclei integration. -type Options struct { - // ServerURL is the URL of the interactsh server. - ServerURL string - // Authorization is the Authorization header value - Authorization string - // CacheSize is the numbers of requests to keep track of at a time. - // Older items are discarded in LRU manner in favor of new requests. - CacheSize int64 - // Eviction is the period of time after which to automatically discard - // interaction requests. - Eviction time.Duration - // CooldownPeriod is additional time to wait for interactions after closing - // of the poller. - CooldownPeriod time.Duration - // PollDuration is the time to wait before each poll to the server for interactions. - PollDuration time.Duration - // Output is the output writer for nuclei - Output output.Writer - // IssuesClient is a client for issue exporting - IssuesClient reporting.Client - // Progress is the nuclei progress bar implementation. - Progress progress.Progress - // Debug specifies whether debugging output should be shown for interactsh-client - Debug bool - DebugRequest bool - DebugResponse bool - // DisableHttpFallback controls http retry in case of https failure for server url - DisableHttpFallback bool - // NoInteractsh disables the engine - NoInteractsh bool - // NoColor dissbles printing colors for matches - NoColor bool - - StopAtFirstMatch bool - HTTPClient *retryablehttp.Client -} - -const defaultMaxInteractionsCount = 5000 - // New returns a new interactsh server client func New(options *Options) (*Client, error) { - configure := ccache.Configure() - configure = configure.MaxSize(options.CacheSize) - cache := atomcache.NewWithCache(ccache.New(configure)) - - interactionsCfg := ccache.Configure() - interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount) - interactionsCache := atomcache.NewWithCache(ccache.New(interactionsCfg)) - - matchedTemplateCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))) - interactshURLCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))) + requestsCache := gcache.New[string, *RequestData](options.CacheSize).LRU().Build() + interactionsCache := gcache.New[string, []*server.Interaction](defaultMaxInteractionsCount).LRU().Build() + matchedTemplateCache := gcache.New[string, bool](defaultMaxInteractionsCount).LRU().Build() + interactshURLCache := gcache.New[string, string](defaultMaxInteractionsCount).LRU().Build() interactClient := &Client{ eviction: options.Eviction, @@ -124,31 +67,14 @@ func New(options *Options) (*Client, error) { matchedTemplates: matchedTemplateCache, interactshURLs: interactshURLCache, options: options, - requests: cache, + requests: requestsCache, pollDuration: options.PollDuration, cooldownDuration: options.CooldownPeriod, - dataMutex: &sync.RWMutex{}, } return interactClient, nil } -// NewDefaultOptions returns the default options for interactsh client -func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options { - return &Options{ - ServerURL: client.DefaultOptions.ServerURL, - CacheSize: 5000, - Eviction: 60 * time.Second, - CooldownPeriod: 5 * time.Second, - PollDuration: 5 * time.Second, - Output: output, - IssuesClient: reporting, - Progress: progress, - DisableHttpFallback: true, - NoColor: false, - } -} - -func (c *Client) firstTimeInitializeClient() error { +func (c *Client) poll() error { if c.options.NoInteractsh { return nil // do not init if disabled } @@ -159,40 +85,34 @@ func (c *Client) firstTimeInitializeClient() error { HTTPClient: c.options.HTTPClient, }) if err != nil { - return errors.Wrap(err, "could not create client") + return errorutil.NewWithErr(err).Msgf("could not create client") } + c.interactsh = interactsh interactURL := interactsh.URL() interactDomain := interactURL[strings.Index(interactURL, ".")+1:] gologger.Info().Msgf("Using Interactsh Server: %s", interactDomain) - c.dataMutex.Lock() - c.hostname = interactDomain - c.dataMutex.Unlock() + c.setHostname(interactDomain) err = interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) { - item := c.requests.Get(interaction.UniqueID) - if item == nil { + request, err := c.requests.Get(interaction.UniqueID) + if errors.Is(err, gcache.KeyNotFoundError) || request == nil { // If we don't have any request for this ID, add it to temporary // lru cache, so we can correlate when we get an add request. - gotItem := c.interactions.Get(interaction.UniqueID) - if gotItem == nil { - c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) - } else if items, ok := gotItem.Value().([]*server.Interaction); ok { + items, err := c.interactions.Get(interaction.UniqueID) + if errorutil.IsAny(err, gcache.KeyNotFoundError) || items == nil { + _ = c.interactions.SetWithExpire(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) + } else { items = append(items, interaction) - c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration) + _ = c.interactions.SetWithExpire(interaction.UniqueID, items, defaultInteractionDuration) } return } - request, ok := item.Value().(*RequestData) - if !ok { - return - } - if _, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { - gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent[templateIdAttribute].(string), request.Event.InternalEvent["host"].(string))) - if gotItem != nil { + if requestShouldStopAtFirstMatch(request) || c.options.StopAtFirstMatch { + if gotItem, err := c.matchedTemplates.Get(hash(request.Event.InternalEvent)); gotItem && err == nil { return } } @@ -201,23 +121,44 @@ func (c *Client) firstTimeInitializeClient() error { }) if err != nil { - return errors.Wrap(err, "could not perform instactsh polling") + return errorutil.NewWithErr(err).Msgf("could not perform interactsh polling") } return nil } +// requestShouldStopAtFirstmatch checks if furthur interactions should be stopped +// note: extra care should be taken while using this function since internalEvent is +// synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that +// we could use `TryLock()` but that may over complicate things and need to differentiate +// situations whether to block or skip +func requestShouldStopAtFirstMatch(request *RequestData) bool { + request.Event.RLock() + defer request.Event.RUnlock() + + if stop, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok { + if v, ok := stop.(bool); ok { + return v + } + } + return false +} + // processInteractionForRequest processes an interaction for a request func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool { + data.Event.Lock() data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress + data.Event.Unlock() result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, c.options.Debug || c.options.DebugRequest || c.options.DebugResponse) + + // if we don't match, return if !matched || result == nil { - return false // if we don't match, return + return false } - c.requests.Delete(interaction.UniqueID) + c.requests.Remove(interaction.UniqueID) if data.Event.OperatorsResult != nil { data.Event.OperatorsResult.Merge(result) @@ -225,10 +166,12 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d data.Event.SetOperatorResult(result) } + data.Event.Lock() data.Event.Results = data.MakeResultFunc(data.Event) for _, event := range data.Event.Results { event.Interaction = interaction } + data.Event.Unlock() if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse { c.debugPrintInteraction(interaction, data.Event.OperatorsResult) @@ -236,30 +179,39 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched.Store(true) - if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { - c.matchedTemplates.Set(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) + if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch { + _ = c.matchedTemplates.SetWithExpire(hash(data.Event.InternalEvent), true, defaultInteractionDuration) } } + return true } +func (c *Client) AlreadyMatched(data *RequestData) bool { + data.Event.RLock() + defer data.Event.RUnlock() + + return c.matchedTemplates.Has(hash(data.Event.InternalEvent)) +} + // URL returns a new URL that can be interacted with func (c *Client) URL() (string, error) { - c.firstTimeGroup.Do(func() { - if err := c.firstTimeInitializeClient(); err != nil { - gologger.Error().Msgf("Could not initialize interactsh client: %s", err) - } - }) if c.interactsh == nil { - return "", errors.New("interactsh client not initialized") + var err error + c.Do(func() { + err = c.poll() + }) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("interactsh client not initialized") + } } - atomic.CompareAndSwapUint32(&c.generated, 0, 1) + c.generated.Store(true) return c.interactsh.URL(), nil } -// Close closes the interactsh clients after waiting for cooldown period. +// Close the interactsh clients after waiting for cooldown period. func (c *Client) Close() bool { - if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 { + if c.cooldownDuration > 0 && c.generated.Load() { time.Sleep(c.cooldownDuration) } if c.interactsh != nil { @@ -267,15 +219,10 @@ func (c *Client) Close() bool { c.interactsh.Close() } - closeCache := func(cc *atomcache.Cache) { - if cc != nil { - cc.Stop() - } - } - closeCache(c.requests) - closeCache(c.interactions) - closeCache(c.matchedTemplates) - closeCache(c.interactshURLs) + c.requests.Purge() + c.interactions.Purge() + c.matchedTemplates.Purge() + c.interactshURLs.Purge() return c.matched.Load() } @@ -308,36 +255,29 @@ func (c *Client) NewURLWithData(data string) (string, error) { if url == "" { return "", errors.New("empty interactsh url") } - c.interactshURLs.Set(url, data, defaultInteractionDuration) + _ = c.interactshURLs.SetWithExpire(url, data, defaultInteractionDuration) return url, nil } // MakePlaceholders does placeholders for interact URLs and other data to a map func (c *Client) MakePlaceholders(urls []string, data map[string]interface{}) { - data["interactsh-server"] = c.getInteractServerHostname() + data["interactsh-server"] = c.getHostname() for _, url := range urls { - if interactshURLMarker := c.interactshURLs.Get(url); interactshURLMarker != nil { - if interactshURLMarker, ok := interactshURLMarker.Value().(string); ok { - interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}") + if interactshURLMarker, err := c.interactshURLs.Get(url); interactshURLMarker != "" && err == nil { + interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}") - c.interactshURLs.Delete(url) + c.interactshURLs.Remove(url) - data[interactshMarker] = url - urlIndex := strings.Index(url, ".") - if urlIndex == -1 { - continue - } - data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex] + data[interactshMarker] = url + urlIndex := strings.Index(url, ".") + if urlIndex == -1 { + continue } + data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex] } } } -// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options -func (c *Client) SetStopAtFirstMatch() { - c.options.StopAtFirstMatch = true -} - // MakeResultEventFunc is a result making function for nuclei type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent @@ -352,35 +292,26 @@ type RequestData struct { // RequestEvent is the event for a network request sent by nuclei. func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { - data.Event.Lock() - defer data.Event.Unlock() - for _, interactshURL := range interactshURLs { - id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.hostname), ".") + id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.getHostname()), ".") - if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { - gotItem := c.matchedTemplates.Get(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string))) - if gotItem != nil { + if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch { + gotItem, err := c.matchedTemplates.Get(hash(data.Event.InternalEvent)) + if gotItem && err == nil { break } } - interaction := c.interactions.Get(id) - if interaction != nil { - // If we have previous interactions, get them and process them. - interactions, ok := interaction.Value().([]*server.Interaction) - if !ok { - c.requests.Set(id, data, c.eviction) - return - } + interactions, err := c.interactions.Get(id) + if interactions != nil && err == nil { for _, interaction := range interactions { if c.processInteractionForRequest(interaction, data) { - c.interactions.Delete(id) + c.interactions.Remove(id) break } } } else { - c.requests.Set(id, data, c.eviction) + _ = c.requests.SetWithExpire(id, data, c.eviction) } } } @@ -397,16 +328,16 @@ func HasMatchers(op *operators.Operators) bool { for _, matcher := range op.Matchers { for _, dsl := range matcher.DSL { - if strings.Contains(dsl, "interactsh") { + if stringsutil.ContainsAnyI(dsl, "interactsh") { return true } } - if strings.HasPrefix(matcher.Part, "interactsh") { + if stringsutil.HasPrefixI(matcher.Part, "interactsh") { return true } } for _, matcher := range op.Extractors { - if strings.HasPrefix(matcher.Part, "interactsh") { + if stringsutil.HasPrefixI(matcher.Part, "interactsh") { return true } } @@ -461,16 +392,22 @@ func formatInteractionMessage(key, value string, event *operators.Result, noColo return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value) } -func hash(templateID, host string) string { - h := sha1.New() - h.Write([]byte(templateID)) - h.Write([]byte(host)) - return hex.EncodeToString(h.Sum(nil)) +func hash(internalEvent output.InternalEvent) string { + templateId := internalEvent[templateIdAttribute].(string) + host := internalEvent["host"].(string) + return fmt.Sprintf("%s:%s", templateId, host) } -func (c *Client) getInteractServerHostname() string { - c.dataMutex.RLock() - defer c.dataMutex.RUnlock() +func (c *Client) getHostname() string { + c.RLock() + defer c.RUnlock() return c.hostname } + +func (c *Client) setHostname(hostname string) { + c.Lock() + defer c.Unlock() + + c.hostname = hostname +} diff --git a/v2/pkg/protocols/common/interactsh/options.go b/v2/pkg/protocols/common/interactsh/options.go new file mode 100644 index 000000000..22a745d37 --- /dev/null +++ b/v2/pkg/protocols/common/interactsh/options.go @@ -0,0 +1,67 @@ +package interactsh + +import ( + "time" + + "github.com/projectdiscovery/interactsh/pkg/client" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/retryablehttp-go" +) + +// Options contains configuration options for interactsh nuclei integration. +type Options struct { + // ServerURL is the URL of the interactsh server. + ServerURL string + // Authorization is the Authorization header value + Authorization string + // CacheSize is the numbers of requests to keep track of at a time. + // Older items are discarded in LRU manner in favor of new requests. + CacheSize int + // Eviction is the period of time after which to automatically discard + // interaction requests. + Eviction time.Duration + // CooldownPeriod is additional time to wait for interactions after closing + // of the poller. + CooldownPeriod time.Duration + // PollDuration is the time to wait before each poll to the server for interactions. + PollDuration time.Duration + // Output is the output writer for nuclei + Output output.Writer + // IssuesClient is a client for issue exporting + IssuesClient reporting.Client + // Progress is the nuclei progress bar implementation. + Progress progress.Progress + // Debug specifies whether debugging output should be shown for interactsh-client + Debug bool + // DebugRequest outputs interaction request + DebugRequest bool + // DebugResponse outputs interaction response + DebugResponse bool + // DisableHttpFallback controls http retry in case of https failure for server url + DisableHttpFallback bool + // NoInteractsh disables the engine + NoInteractsh bool + // NoColor dissbles printing colors for matches + NoColor bool + + StopAtFirstMatch bool + HTTPClient *retryablehttp.Client +} + +// DefaultOptions returns the default options for interactsh client +func DefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options { + return &Options{ + ServerURL: client.DefaultOptions.ServerURL, + CacheSize: 5000, + Eviction: 60 * time.Second, + CooldownPeriod: 5 * time.Second, + PollDuration: 5 * time.Second, + Output: output, + IssuesClient: reporting, + Progress: progress, + DisableHttpFallback: true, + NoColor: false, + } +} diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index ed5f92b60..6d9a17aef 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -186,7 +186,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { generator.options.Interactsh, err = interactsh.New(&interactsh.Options{ ServerURL: options.InteractshURL, - CacheSize: int64(options.InteractionsCacheSize), + CacheSize: options.InteractionsCacheSize, Eviction: time.Duration(options.InteractionsEviction) * time.Second, CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0e80eca0c..4443ba987 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -248,24 +248,26 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous } var gotMatches bool requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) { - // Add the extracts to the dynamic values if any. - if event.OperatorsResult != nil { - gotMatches = event.OperatorsResult.Matched - } if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(gr.InteractURLs, &interactsh.RequestData{ + requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, MatchFunc: request.Match, ExtractFunc: request.Extract, - }) + } + request.options.Interactsh.RequestEvent(gr.InteractURLs, requestData) + gotMatches = request.options.Interactsh.AlreadyMatched(requestData) } else { callback(event) } + // Add the extracts to the dynamic values if any. + if event.OperatorsResult != nil { + gotMatches = event.OperatorsResult.Matched + } }, 0) // If a variable is unresolved, skip all further requests - if requestErr == errStopExecution { + if errors.Is(requestErr, errStopExecution) { return false } if requestErr != nil { @@ -276,7 +278,8 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous request.options.Progress.IncrementRequests() // If this was a match, and we want to stop at first match, skip all further requests. - if (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches { + shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch + if shouldStopAtFirstMatch && gotMatches { return false } return true @@ -374,19 +377,21 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } var gotMatches bool err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { - // Add the extracts to the dynamic values if any. - if event.OperatorsResult != nil { - gotMatches = event.OperatorsResult.Matched - gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues) - } if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{ + requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, MatchFunc: request.Match, ExtractFunc: request.Extract, - }) + } + request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, requestData) + gotMatches = request.options.Interactsh.AlreadyMatched(requestData) + } + // Add the extracts to the dynamic values if any. + if event.OperatorsResult != nil { + gotMatches = event.OperatorsResult.Matched + gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues) } // Note: This is a race condition prone zone i.e when request has interactsh_matchers // Interactsh.RequestEvent tries to access/update output.InternalWrappedEvent depending on logic @@ -397,7 +402,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa }, generator.currentIndex) // If a variable is unresolved, skip all further requests - if err == errStopExecution { + if errors.Is(err, errStopExecution) { return true, nil } if err != nil { @@ -409,7 +414,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.Progress.IncrementRequests() // If this was a match, and we want to stop at first match, skip all further requests. - if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches { + shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch + if shouldStopAtFirstMatch && gotMatches { return true, nil } return false, nil @@ -755,7 +761,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ callback(event) // Skip further responses if we have stop-at-first-match and a match - if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && len(event.Results) > 0 { + if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() { return nil } } diff --git a/v2/pkg/utils/atomcache/atomcache.go b/v2/pkg/utils/atomcache/atomcache.go deleted file mode 100644 index ee6cc84b7..000000000 --- a/v2/pkg/utils/atomcache/atomcache.go +++ /dev/null @@ -1,62 +0,0 @@ -package atomcache - -import ( - "sync" - "sync/atomic" - "time" - - "github.com/karlseguin/ccache" -) - -type Cache struct { - *ccache.Cache - Closed atomic.Bool - mu sync.RWMutex -} - -func NewWithCache(c *ccache.Cache) *Cache { - return &Cache{Cache: c} -} - -func (c *Cache) Get(key string) *ccache.Item { - if c.Closed.Load() { - return nil - } - c.mu.RLock() - defer c.mu.RUnlock() - - return c.Cache.Get(key) -} - -func (c *Cache) Set(key string, value interface{}, duration time.Duration) { - if c.Closed.Load() { - return - } - c.mu.Lock() - defer c.mu.Unlock() - - c.Cache.Set(key, value, duration) -} - -func (c *Cache) Delete(key string) bool { - if c.Closed.Load() { - return false - } - - c.mu.Lock() - defer c.mu.Unlock() - - return c.Cache.Delete(key) -} - -func (c *Cache) Stop() { - if c.Closed.Load() { - return - } - c.Closed.Store(true) - - c.mu.Lock() - defer c.mu.Unlock() - - c.Cache.Stop() -}