Replacing ccache with generic gcache (#3523)

* Replacing ccache with generic gcache

* fixing lint issues

* removing unecessary hashing + using errorutils

* making test more tolerant

* removing dead code + refactor

* removing redundant code

* removing race

* maint

* moving code

* adding more iterations

* note + typo

* temporary fixing stop-at-first-match with interact

* wrapping internal map with mux

* sort before running integration test

* fix deadlock in requestShouldStopAtFirstMatch

* add timeout to integration_test workflow

* attempting to remove outer lock

* adds interactsh protocol tests in integration_test

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

.github/workflows/build-test.yml

@@ -42,6 +42,7 @@ jobs:
         working-directory: v2/
 
       - name: Integration Tests
+        timeout-minutes: 50
         env:
           GH_ACTION: true
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

integration_tests/http/interactsh-stop-at-first-match.yaml

@@ -8,9 +8,15 @@ info:
 requests:
   - method: GET
     path:
-      - "{{BaseURL}}"
+      - "{{BaseURL}}/?a=1"
-      - "{{BaseURL}}"
+      - "{{BaseURL}}/?a=2"
-      - "{{BaseURL}}"
+      - "{{BaseURL}}/?a=3"
+      - "{{BaseURL}}/?a=4"
+      - "{{BaseURL}}/?a=5"
+      - "{{BaseURL}}/?a=6"
+      - "{{BaseURL}}/?a=7"
+      - "{{BaseURL}}/?a=8"
+      - "{{BaseURL}}/?a=9"
     headers:
       url: 'http://{{interactsh-url}}'
 

v2/cmd/integration-test/code.go

@@ -88,7 +88,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
 	defaultOpts.Templates = goflags.StringSlice{templatePath}
 	defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
 
-	interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
+	interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
 	interactClient, err := interactsh.New(interactOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create interact client")

v2/cmd/integration-test/http.go

@@ -47,8 +47,6 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/http-paths.yaml":                          &httpPaths{},
 	"http/request-condition.yaml":                   &httpRequestCondition{},
 	"http/request-condition-new.yaml":               &httpRequestCondition{},
-	"http/interactsh.yaml":                          &httpInteractshRequest{},
-	"http/interactsh-stop-at-first-match.yaml":      &httpInteractshStopAtFirstMatchRequest{},
 	"http/self-contained.yaml":                      &httpRequestSelfContained{},
 	"http/self-contained-file-input.yaml":           &httpRequestSelfContainedFileInput{},
 	"http/get-case-insensitive.yaml":                &httpGetCaseInsensitive{},
@@ -71,7 +69,6 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/get-without-scheme.yaml":                  &httpGetWithoutScheme{},
 	"http/cl-body-without-header.yaml":              &httpCLBodyWithoutHeader{},
 	"http/cl-body-with-header.yaml":                 &httpCLBodyWithHeader{},
-	"http/default-matcher-condition.yaml":           &httpDefaultMatcherCondition{},
 }
 
 type httpInteractshRequest struct{}
@@ -164,6 +161,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
+	// polling is asyncronous, so the interactions may be retrieved after the first request
 	return expectResultsCount(results, 1)
 }
 

v2/cmd/integration-test/integration-test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"sort"
 	"strings"
 
 	"github.com/logrusorgru/aurora"
@@ -22,6 +23,7 @@ var (
 
 	protocolTests = map[string]map[string]testutils.TestCase{
 		"http":            httpTestcases,
+		"interactsh":      interactshTestCases,
 		"network":         networkTestcases,
 		"dns":             dnsTestCases,
 		"workflow":        workflowTestcases,
@@ -43,6 +45,7 @@ var (
 	runProtocol          = ""
 	runTemplate          = ""
 	extraArgs            = []string{}
+	interactshRetryCount = 3
 )
 
 func main() {
@@ -58,7 +61,6 @@ func main() {
 	}
 
 	if runProtocol != "" {
-		debug = true
 		debugTests()
 		os.Exit(1)
 	}
@@ -79,14 +81,37 @@ func main() {
 	}
 }
 
+// execute a testcase with retry and consider best of N
+// intended for flaky tests like interactsh
+func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) {
+	var err error
+	for i := 0; i < retryCount; i++ {
+		err = testCase.Execute(templatePath)
+		if err == nil {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath)
+			return "", nil
+		}
+	}
+	_, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed after %v attempts : %s\n", failed, templatePath, retryCount, err)
+	return templatePath, err
+}
+
 func debugTests() {
-	for tpath, testcase := range protocolTests[runProtocol] {
+	keys := getMapKeys(protocolTests[runProtocol])
+	for _, tpath := range keys {
+		testcase := protocolTests[runProtocol][tpath]
 		if runTemplate != "" && !strings.Contains(tpath, runTemplate) {
 			continue
 		}
-		if err := testcase.Execute(tpath); err != nil {
+		if runProtocol == "interactsh" {
+			if _, err := executeWithRetry(testcase, tpath, interactshRetryCount); err != nil {
 				fmt.Printf("\n%v", err.Error())
 			}
+		} else {
+			if _, err := execute(testcase, tpath); err != nil {
+				fmt.Printf("\n%v", err.Error())
+			}
+		}
 	}
 }
 
@@ -97,10 +122,19 @@ func runTests(customTemplatePaths []string) []string {
 		if len(customTemplatePaths) == 0 {
 			fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto))
 		}
+		keys := getMapKeys(testCases)
 
-		for templatePath, testCase := range testCases {
+		for _, templatePath := range keys {
+			testCase := testCases[templatePath]
 			if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, templatePath) {
-				if failedTemplatePath, err := execute(testCase, templatePath); err != nil {
+				var failedTemplatePath string
+				var err error
+				if proto == "interactsh" {
+					failedTemplatePath, err = executeWithRetry(testCase, templatePath, interactshRetryCount)
+				} else {
+					failedTemplatePath, err = execute(testCase, templatePath)
+				}
+				if err != nil {
 					failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath)
 				}
 			}
@@ -120,9 +154,10 @@ func execute(testCase testutils.TestCase, templatePath string) (string, error) {
 	return "", nil
 }
 
-func expectResultsCount(results []string, expectedNumber int) error {
+func expectResultsCount(results []string, expectedNumbers ...int) error {
-	if len(results) != expectedNumber {
+	match := sliceutil.Contains(expectedNumbers, len(results))
-		return fmt.Errorf("incorrect number of results: %d (actual) vs %d (expected) \nResults:\n\t%s\n", len(results), expectedNumber, strings.Join(results, "\n\t"))
+	if !match {
+		return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t"))
 	}
 	return nil
 }
@@ -132,3 +167,12 @@ func normalizeSplit(str string) []string {
 		return r == ','
 	})
 }
+
+func getMapKeys[T any](testcases map[string]T) []string {
+	keys := make([]string, 0, len(testcases))
+	for k := range testcases {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}

v2/cmd/integration-test/interactsh.go

@@ -0,0 +1,10 @@
+package main
+
+import "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+
+// All Interactsh related testcases
+var interactshTestCases = map[string]testutils.TestCase{
+	"http/interactsh.yaml":                     &httpInteractshRequest{},
+	"http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{},
+	"http/default-matcher-condition.yaml":      &httpDefaultMatcherCondition{},
+}

v2/examples/simple.go

@@ -50,7 +50,7 @@ func main() {
 	defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"}
 	defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
 
-	interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
+	interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
 	interactClient, err := interactsh.New(interactOpts)
 	if err != nil {
 		log.Fatalf("Could not create interact client: %s\n", err)

v2/go.mod

@@ -18,7 +18,6 @@ require (
 	github.com/itchyny/gojq v0.12.11
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
-	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/miekg/dns v1.1.53
 	github.com/olekukonko/tablewriter v0.0.5
@@ -53,6 +52,7 @@ require (
 
 require (
 	github.com/DataDog/gostackparse v0.6.0
+	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
 	github.com/antchfx/xmlquery v1.3.15
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
 	github.com/aws/aws-sdk-go-v2 v1.17.8
@@ -111,7 +111,6 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/karlseguin/expect v1.0.8 // indirect
 	github.com/kataras/jwt v0.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect

v2/go.sum

@@ -10,6 +10,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE=
+github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
 github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo=
@@ -290,11 +292,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
-github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
 github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
-github.com/karlseguin/expect v1.0.8 h1:Bb0H6IgBWQpadY25UDNkYPDB9ITqK1xnSoZfAq362fw=
-github.com/karlseguin/expect v1.0.8/go.mod h1:lXdI8iGiQhmzpnnmU/EGA60vqKs8NbRNFnhhrJGoD5g=
 github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk=
 github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -582,8 +580,6 @@ github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2Z
 github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
 github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
 github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
 github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk=
 github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=

v2/internal/runner/runner.go

@@ -262,14 +262,14 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	runner.resumeCfg = resumeCfg
 
-	opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress)
+	opts := interactsh.DefaultOptions(runner.output, runner.issuesClient, runner.progress)
 	opts.Debug = runner.options.Debug
 	opts.NoColor = runner.options.NoColor
 	if options.InteractshURL != "" {
 		opts.ServerURL = options.InteractshURL
 	}
 	opts.Authorization = options.InteractshToken
-	opts.CacheSize = int64(options.InteractionsCacheSize)
+	opts.CacheSize = options.InteractionsCacheSize
 	opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second
 	opts.CooldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second
 	opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second

v2/pkg/output/output.go

@@ -85,6 +85,13 @@ func (iwe *InternalWrappedEvent) HasOperatorResult() bool {
 	return iwe.OperatorsResult != nil
 }
 
+func (iwe *InternalWrappedEvent) HasResults() bool {
+	iwe.RLock()
+	defer iwe.RUnlock()
+
+	return len(iwe.Results) > 0
+}
+
 func (iwe *InternalWrappedEvent) SetOperatorResult(operatorResult *operators.Result) {
 	iwe.Lock()
 	defer iwe.Unlock()

v2/pkg/protocols/common/interactsh/const.go

@@ -0,0 +1,18 @@
+package interactsh
+
+import (
+	"regexp"
+	"time"
+)
+
+var (
+	defaultInteractionDuration = 60 * time.Second
+	interactshURLMarkerRegex   = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`)
+)
+
+const (
+	stopAtFirstMatchAttribute = "stop-at-first-match"
+	templateIdAttribute       = "template-id"
+
+	defaultMaxInteractionsCount = 5000
+)

v2/pkg/protocols/common/interactsh/interactsh.go

@@ -2,8 +2,6 @@ package interactsh
 
 import (
 	"bytes"
-	"crypto/sha1"
-	"encoding/hex"
 	"fmt"
 	"os"
 	"regexp"
@@ -12,111 +10,56 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/karlseguin/ccache"
+	"errors"
-	"github.com/pkg/errors"
+
+	"github.com/Mzack9999/gcache"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/interactsh/pkg/client"
 	"github.com/projectdiscovery/interactsh/pkg/server"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	errorutil "github.com/projectdiscovery/utils/errors"
-	"github.com/projectdiscovery/nuclei/v2/pkg/utils/atomcache"
+	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/projectdiscovery/retryablehttp-go"
 )
 
 // Client is a wrapped client for interactsh server.
 type Client struct {
+	sync.Once
+	sync.RWMutex
+
+	options *Options
+
 	// interactsh is a client for interactsh server.
 	interactsh *client.Client
 	// requests is a stored cache for interactsh-url->request-event data.
-	requests *atomcache.Cache
+	requests gcache.Cache[string, *RequestData]
 	// interactions is a stored cache for interactsh-interaction->interactsh-url data
-	interactions *atomcache.Cache
+	interactions gcache.Cache[string, []*server.Interaction]
 	// matchedTemplates is a stored cache to track matched templates
-	matchedTemplates *atomcache.Cache
+	matchedTemplates gcache.Cache[string, bool]
-	// interactshURLs is a stored cache to track track multiple interactsh markers
+	// interactshURLs is a stored cache to track multiple interactsh markers
-	interactshURLs *atomcache.Cache
+	interactshURLs gcache.Cache[string, string]
 
-	options          *Options
 	eviction         time.Duration
 	pollDuration     time.Duration
 	cooldownDuration time.Duration
 
-	dataMutex *sync.RWMutex
-
 	hostname string
 
-	firstTimeGroup sync.Once
+	// determines if wait the cooldown period in case of generated URL
-	generated      uint32 // decide to wait if we have a generated url
+	generated atomic.Bool
 	matched   atomic.Bool
 }
 
-var (
-	defaultInteractionDuration = 60 * time.Second
-	interactshURLMarkerRegex   = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`)
-)
-
-const (
-	stopAtFirstMatchAttribute = "stop-at-first-match"
-	templateIdAttribute       = "template-id"
-)
-
-// Options contains configuration options for interactsh nuclei integration.
-type Options struct {
-	// ServerURL is the URL of the interactsh server.
-	ServerURL string
-	// Authorization is the Authorization header value
-	Authorization string
-	// CacheSize is the numbers of requests to keep track of at a time.
-	// Older items are discarded in LRU manner in favor of new requests.
-	CacheSize int64
-	// Eviction is the period of time after which to automatically discard
-	// interaction requests.
-	Eviction time.Duration
-	// CooldownPeriod is additional time to wait for interactions after closing
-	// of the poller.
-	CooldownPeriod time.Duration
-	// PollDuration is the time to wait before each poll to the server for interactions.
-	PollDuration time.Duration
-	// Output is the output writer for nuclei
-	Output output.Writer
-	// IssuesClient is a client for issue exporting
-	IssuesClient reporting.Client
-	// Progress is the nuclei progress bar implementation.
-	Progress progress.Progress
-	// Debug specifies whether debugging output should be shown for interactsh-client
-	Debug         bool
-	DebugRequest  bool
-	DebugResponse bool
-	// DisableHttpFallback controls http retry in case of https failure for server url
-	DisableHttpFallback bool
-	// NoInteractsh disables the engine
-	NoInteractsh bool
-	// NoColor dissbles printing colors for matches
-	NoColor bool
-
-	StopAtFirstMatch bool
-	HTTPClient       *retryablehttp.Client
-}
-
-const defaultMaxInteractionsCount = 5000
-
 // New returns a new interactsh server client
 func New(options *Options) (*Client, error) {
-	configure := ccache.Configure()
+	requestsCache := gcache.New[string, *RequestData](options.CacheSize).LRU().Build()
-	configure = configure.MaxSize(options.CacheSize)
+	interactionsCache := gcache.New[string, []*server.Interaction](defaultMaxInteractionsCount).LRU().Build()
-	cache := atomcache.NewWithCache(ccache.New(configure))
+	matchedTemplateCache := gcache.New[string, bool](defaultMaxInteractionsCount).LRU().Build()
-
+	interactshURLCache := gcache.New[string, string](defaultMaxInteractionsCount).LRU().Build()
-	interactionsCfg := ccache.Configure()
-	interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount)
-	interactionsCache := atomcache.NewWithCache(ccache.New(interactionsCfg))
-
-	matchedTemplateCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)))
-	interactshURLCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)))
 
 	interactClient := &Client{
 		eviction:         options.Eviction,
@@ -124,31 +67,14 @@ func New(options *Options) (*Client, error) {
 		matchedTemplates: matchedTemplateCache,
 		interactshURLs:   interactshURLCache,
 		options:          options,
-		requests:         cache,
+		requests:         requestsCache,
 		pollDuration:     options.PollDuration,
 		cooldownDuration: options.CooldownPeriod,
-		dataMutex:        &sync.RWMutex{},
 	}
 	return interactClient, nil
 }
 
-// NewDefaultOptions returns the default options for interactsh client
+func (c *Client) poll() error {
-func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options {
-	return &Options{
-		ServerURL:           client.DefaultOptions.ServerURL,
-		CacheSize:           5000,
-		Eviction:            60 * time.Second,
-		CooldownPeriod:      5 * time.Second,
-		PollDuration:        5 * time.Second,
-		Output:              output,
-		IssuesClient:        reporting,
-		Progress:            progress,
-		DisableHttpFallback: true,
-		NoColor:             false,
-	}
-}
-
-func (c *Client) firstTimeInitializeClient() error {
 	if c.options.NoInteractsh {
 		return nil // do not init if disabled
 	}
@@ -159,40 +85,34 @@ func (c *Client) firstTimeInitializeClient() error {
 		HTTPClient:          c.options.HTTPClient,
 	})
 	if err != nil {
-		return errors.Wrap(err, "could not create client")
+		return errorutil.NewWithErr(err).Msgf("could not create client")
 	}
+
 	c.interactsh = interactsh
 
 	interactURL := interactsh.URL()
 	interactDomain := interactURL[strings.Index(interactURL, ".")+1:]
 	gologger.Info().Msgf("Using Interactsh Server: %s", interactDomain)
 
-	c.dataMutex.Lock()
+	c.setHostname(interactDomain)
-	c.hostname = interactDomain
-	c.dataMutex.Unlock()
 
 	err = interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
-		item := c.requests.Get(interaction.UniqueID)
+		request, err := c.requests.Get(interaction.UniqueID)
-		if item == nil {
+		if errors.Is(err, gcache.KeyNotFoundError) || request == nil {
 			// If we don't have any request for this ID, add it to temporary
 			// lru cache, so we can correlate when we get an add request.
-			gotItem := c.interactions.Get(interaction.UniqueID)
+			items, err := c.interactions.Get(interaction.UniqueID)
-			if gotItem == nil {
+			if errorutil.IsAny(err, gcache.KeyNotFoundError) || items == nil {
-				c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
+				_ = c.interactions.SetWithExpire(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
-			} else if items, ok := gotItem.Value().([]*server.Interaction); ok {
+			} else {
 				items = append(items, interaction)
-				c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
+				_ = c.interactions.SetWithExpire(interaction.UniqueID, items, defaultInteractionDuration)
 			}
 			return
 		}
-		request, ok := item.Value().(*RequestData)
-		if !ok {
-			return
-		}
 
-		if _, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
+		if requestShouldStopAtFirstMatch(request) || c.options.StopAtFirstMatch {
-			gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent[templateIdAttribute].(string), request.Event.InternalEvent["host"].(string)))
+			if gotItem, err := c.matchedTemplates.Get(hash(request.Event.InternalEvent)); gotItem && err == nil {
-			if gotItem != nil {
 				return
 			}
 		}
@@ -201,23 +121,44 @@ func (c *Client) firstTimeInitializeClient() error {
 	})
 
 	if err != nil {
-		return errors.Wrap(err, "could not perform instactsh polling")
+		return errorutil.NewWithErr(err).Msgf("could not perform interactsh polling")
 	}
 	return nil
 }
 
+// requestShouldStopAtFirstmatch checks if furthur interactions should be stopped
+// note: extra care should be taken while using this function since internalEvent is
+// synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that
+// we could use `TryLock()` but that may over complicate things and need to differentiate
+// situations whether to block or skip
+func requestShouldStopAtFirstMatch(request *RequestData) bool {
+	request.Event.RLock()
+	defer request.Event.RUnlock()
+
+	if stop, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok {
+		if v, ok := stop.(bool); ok {
+			return v
+		}
+	}
+	return false
+}
+
 // processInteractionForRequest processes an interaction for a request
 func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool {
+	data.Event.Lock()
 	data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
 	data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
 	data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
 	data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress
+	data.Event.Unlock()
 
 	result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, c.options.Debug || c.options.DebugRequest || c.options.DebugResponse)
+
+	// if we don't match, return
 	if !matched || result == nil {
-		return false // if we don't match, return
+		return false
 	}
-	c.requests.Delete(interaction.UniqueID)
+	c.requests.Remove(interaction.UniqueID)
 
 	if data.Event.OperatorsResult != nil {
 		data.Event.OperatorsResult.Merge(result)
@@ -225,10 +166,12 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
 		data.Event.SetOperatorResult(result)
 	}
 
+	data.Event.Lock()
 	data.Event.Results = data.MakeResultFunc(data.Event)
 	for _, event := range data.Event.Results {
 		event.Interaction = interaction
 	}
+	data.Event.Unlock()
 
 	if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse {
 		c.debugPrintInteraction(interaction, data.Event.OperatorsResult)
@@ -236,30 +179,39 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
 
 	if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) {
 		c.matched.Store(true)
-		if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
+		if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch {
-			c.matchedTemplates.Set(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration)
+			_ = c.matchedTemplates.SetWithExpire(hash(data.Event.InternalEvent), true, defaultInteractionDuration)
 		}
 	}
+
 	return true
 }
 
+func (c *Client) AlreadyMatched(data *RequestData) bool {
+	data.Event.RLock()
+	defer data.Event.RUnlock()
+
+	return c.matchedTemplates.Has(hash(data.Event.InternalEvent))
+}
+
 // URL returns a new URL that can be interacted with
 func (c *Client) URL() (string, error) {
-	c.firstTimeGroup.Do(func() {
-		if err := c.firstTimeInitializeClient(); err != nil {
-			gologger.Error().Msgf("Could not initialize interactsh client: %s", err)
-		}
-	})
 	if c.interactsh == nil {
-		return "", errors.New("interactsh client not initialized")
+		var err error
+		c.Do(func() {
+			err = c.poll()
+		})
+		if err != nil {
+			return "", errorutil.NewWithErr(err).Msgf("interactsh client not initialized")
 		}
-	atomic.CompareAndSwapUint32(&c.generated, 0, 1)
+	}
+	c.generated.Store(true)
 	return c.interactsh.URL(), nil
 }
 
-// Close closes the interactsh clients after waiting for cooldown period.
+// Close the interactsh clients after waiting for cooldown period.
 func (c *Client) Close() bool {
-	if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 {
+	if c.cooldownDuration > 0 && c.generated.Load() {
 		time.Sleep(c.cooldownDuration)
 	}
 	if c.interactsh != nil {
@@ -267,15 +219,10 @@ func (c *Client) Close() bool {
 		c.interactsh.Close()
 	}
 
-	closeCache := func(cc *atomcache.Cache) {
+	c.requests.Purge()
-		if cc != nil {
+	c.interactions.Purge()
-			cc.Stop()
+	c.matchedTemplates.Purge()
-		}
+	c.interactshURLs.Purge()
-	}
-	closeCache(c.requests)
-	closeCache(c.interactions)
-	closeCache(c.matchedTemplates)
-	closeCache(c.interactshURLs)
 
 	return c.matched.Load()
 }
@@ -308,19 +255,18 @@ func (c *Client) NewURLWithData(data string) (string, error) {
 	if url == "" {
 		return "", errors.New("empty interactsh url")
 	}
-	c.interactshURLs.Set(url, data, defaultInteractionDuration)
+	_ = c.interactshURLs.SetWithExpire(url, data, defaultInteractionDuration)
 	return url, nil
 }
 
 // MakePlaceholders does placeholders for interact URLs and other data to a map
 func (c *Client) MakePlaceholders(urls []string, data map[string]interface{}) {
-	data["interactsh-server"] = c.getInteractServerHostname()
+	data["interactsh-server"] = c.getHostname()
 	for _, url := range urls {
-		if interactshURLMarker := c.interactshURLs.Get(url); interactshURLMarker != nil {
+		if interactshURLMarker, err := c.interactshURLs.Get(url); interactshURLMarker != "" && err == nil {
-			if interactshURLMarker, ok := interactshURLMarker.Value().(string); ok {
 			interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}")
 
-				c.interactshURLs.Delete(url)
+			c.interactshURLs.Remove(url)
 
 			data[interactshMarker] = url
 			urlIndex := strings.Index(url, ".")
@@ -330,12 +276,6 @@ func (c *Client) MakePlaceholders(urls []string, data map[string]interface{}) {
 			data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex]
 		}
 	}
-	}
-}
-
-// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options
-func (c *Client) SetStopAtFirstMatch() {
-	c.options.StopAtFirstMatch = true
 }
 
 // MakeResultEventFunc is a result making function for nuclei
@@ -352,35 +292,26 @@ type RequestData struct {
 
 // RequestEvent is the event for a network request sent by nuclei.
 func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) {
-	data.Event.Lock()
-	defer data.Event.Unlock()
-
 	for _, interactshURL := range interactshURLs {
-		id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.hostname), ".")
+		id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.getHostname()), ".")
 
-		if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
+		if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch {
-			gotItem := c.matchedTemplates.Get(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)))
+			gotItem, err := c.matchedTemplates.Get(hash(data.Event.InternalEvent))
-			if gotItem != nil {
+			if gotItem && err == nil {
 				break
 			}
 		}
 
-		interaction := c.interactions.Get(id)
+		interactions, err := c.interactions.Get(id)
-		if interaction != nil {
+		if interactions != nil && err == nil {
-			// If we have previous interactions, get them and process them.
-			interactions, ok := interaction.Value().([]*server.Interaction)
-			if !ok {
-				c.requests.Set(id, data, c.eviction)
-				return
-			}
 			for _, interaction := range interactions {
 				if c.processInteractionForRequest(interaction, data) {
-					c.interactions.Delete(id)
+					c.interactions.Remove(id)
 					break
 				}
 			}
 		} else {
-			c.requests.Set(id, data, c.eviction)
+			_ = c.requests.SetWithExpire(id, data, c.eviction)
 		}
 	}
 }
@@ -397,16 +328,16 @@ func HasMatchers(op *operators.Operators) bool {
 
 	for _, matcher := range op.Matchers {
 		for _, dsl := range matcher.DSL {
-			if strings.Contains(dsl, "interactsh") {
+			if stringsutil.ContainsAnyI(dsl, "interactsh") {
 				return true
 			}
 		}
-		if strings.HasPrefix(matcher.Part, "interactsh") {
+		if stringsutil.HasPrefixI(matcher.Part, "interactsh") {
 			return true
 		}
 	}
 	for _, matcher := range op.Extractors {
-		if strings.HasPrefix(matcher.Part, "interactsh") {
+		if stringsutil.HasPrefixI(matcher.Part, "interactsh") {
 			return true
 		}
 	}
@@ -461,16 +392,22 @@ func formatInteractionMessage(key, value string, event *operators.Result, noColo
 	return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value)
 }
 
-func hash(templateID, host string) string {
+func hash(internalEvent output.InternalEvent) string {
-	h := sha1.New()
+	templateId := internalEvent[templateIdAttribute].(string)
-	h.Write([]byte(templateID))
+	host := internalEvent["host"].(string)
-	h.Write([]byte(host))
+	return fmt.Sprintf("%s:%s", templateId, host)
-	return hex.EncodeToString(h.Sum(nil))
 }
 
-func (c *Client) getInteractServerHostname() string {
+func (c *Client) getHostname() string {
-	c.dataMutex.RLock()
+	c.RLock()
-	defer c.dataMutex.RUnlock()
+	defer c.RUnlock()
 
 	return c.hostname
 }
+
+func (c *Client) setHostname(hostname string) {
+	c.Lock()
+	defer c.Unlock()
+
+	c.hostname = hostname
+}

v2/pkg/protocols/common/interactsh/options.go

@@ -0,0 +1,67 @@
+package interactsh
+
+import (
+	"time"
+
+	"github.com/projectdiscovery/interactsh/pkg/client"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+// Options contains configuration options for interactsh nuclei integration.
+type Options struct {
+	// ServerURL is the URL of the interactsh server.
+	ServerURL string
+	// Authorization is the Authorization header value
+	Authorization string
+	// CacheSize is the numbers of requests to keep track of at a time.
+	// Older items are discarded in LRU manner in favor of new requests.
+	CacheSize int
+	// Eviction is the period of time after which to automatically discard
+	// interaction requests.
+	Eviction time.Duration
+	// CooldownPeriod is additional time to wait for interactions after closing
+	// of the poller.
+	CooldownPeriod time.Duration
+	// PollDuration is the time to wait before each poll to the server for interactions.
+	PollDuration time.Duration
+	// Output is the output writer for nuclei
+	Output output.Writer
+	// IssuesClient is a client for issue exporting
+	IssuesClient reporting.Client
+	// Progress is the nuclei progress bar implementation.
+	Progress progress.Progress
+	// Debug specifies whether debugging output should be shown for interactsh-client
+	Debug bool
+	// DebugRequest outputs interaction request
+	DebugRequest bool
+	// DebugResponse outputs interaction response
+	DebugResponse bool
+	// DisableHttpFallback controls http retry in case of https failure for server url
+	DisableHttpFallback bool
+	// NoInteractsh disables the engine
+	NoInteractsh bool
+	// NoColor dissbles printing colors for matches
+	NoColor bool
+
+	StopAtFirstMatch bool
+	HTTPClient       *retryablehttp.Client
+}
+
+// DefaultOptions returns the default options for interactsh client
+func DefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options {
+	return &Options{
+		ServerURL:           client.DefaultOptions.ServerURL,
+		CacheSize:           5000,
+		Eviction:            60 * time.Second,
+		CooldownPeriod:      5 * time.Second,
+		PollDuration:        5 * time.Second,
+		Output:              output,
+		IssuesClient:        reporting,
+		Progress:            progress,
+		DisableHttpFallback: true,
+		NoColor:             false,
+	}
+}

v2/pkg/protocols/http/build_request_test.go

@@ -186,7 +186,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 
 	generator.options.Interactsh, err = interactsh.New(&interactsh.Options{
 		ServerURL:           options.InteractshURL,
-		CacheSize:           int64(options.InteractionsCacheSize),
+		CacheSize:           options.InteractionsCacheSize,
 		Eviction:            time.Duration(options.InteractionsEviction) * time.Second,
 		CooldownPeriod:      time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
 		PollDuration:        time.Duration(options.InteractionsPollDuration) * time.Second,

v2/pkg/protocols/http/request.go

@@ -248,24 +248,26 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 		}
 		var gotMatches bool
 		requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
-			// Add the extracts to the dynamic values if any.
-			if event.OperatorsResult != nil {
-				gotMatches = event.OperatorsResult.Matched
-			}
 			if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
-				request.options.Interactsh.RequestEvent(gr.InteractURLs, &interactsh.RequestData{
+				requestData := &interactsh.RequestData{
 					MakeResultFunc: request.MakeResultEvent,
 					Event:          event,
 					Operators:      request.CompiledOperators,
 					MatchFunc:      request.Match,
 					ExtractFunc:    request.Extract,
-				})
+				}
+				request.options.Interactsh.RequestEvent(gr.InteractURLs, requestData)
+				gotMatches = request.options.Interactsh.AlreadyMatched(requestData)
 			} else {
 				callback(event)
 			}
+			// Add the extracts to the dynamic values if any.
+			if event.OperatorsResult != nil {
+				gotMatches = event.OperatorsResult.Matched
+			}
 		}, 0)
 		// If a variable is unresolved, skip all further requests
-		if requestErr == errStopExecution {
+		if errors.Is(requestErr, errStopExecution) {
 			return false
 		}
 		if requestErr != nil {
@@ -276,7 +278,8 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 		request.options.Progress.IncrementRequests()
 
 		// If this was a match, and we want to stop at first match, skip all further requests.
-		if (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches {
+		shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
+		if shouldStopAtFirstMatch && gotMatches {
 			return false
 		}
 		return true
@@ -374,19 +377,21 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			}
 			var gotMatches bool
 			err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
-				// Add the extracts to the dynamic values if any.
-				if event.OperatorsResult != nil {
-					gotMatches = event.OperatorsResult.Matched
-					gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues)
-				}
 				if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
-					request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{
+					requestData := &interactsh.RequestData{
 						MakeResultFunc: request.MakeResultEvent,
 						Event:          event,
 						Operators:      request.CompiledOperators,
 						MatchFunc:      request.Match,
 						ExtractFunc:    request.Extract,
-					})
+					}
+					request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, requestData)
+					gotMatches = request.options.Interactsh.AlreadyMatched(requestData)
+				}
+				// Add the extracts to the dynamic values if any.
+				if event.OperatorsResult != nil {
+					gotMatches = event.OperatorsResult.Matched
+					gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues)
 				}
 				// Note: This is a race condition prone zone i.e when request has interactsh_matchers
 				// Interactsh.RequestEvent tries to access/update output.InternalWrappedEvent depending on logic
@@ -397,7 +402,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			}, generator.currentIndex)
 
 			// If a variable is unresolved, skip all further requests
-			if err == errStopExecution {
+			if errors.Is(err, errStopExecution) {
 				return true, nil
 			}
 			if err != nil {
@@ -409,7 +414,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			request.options.Progress.IncrementRequests()
 
 			// If this was a match, and we want to stop at first match, skip all further requests.
-			if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches {
+			shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch
+			if shouldStopAtFirstMatch && gotMatches {
 				return true, nil
 			}
 			return false, nil
@@ -755,7 +761,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		callback(event)
 
 		// Skip further responses if we have stop-at-first-match and a match
-		if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && len(event.Results) > 0 {
+		if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() {
 			return nil
 		}
 	}

v2/pkg/utils/atomcache/atomcache.go

@@ -1,62 +0,0 @@
-package atomcache
-
-import (
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/karlseguin/ccache"
-)
-
-type Cache struct {
-	*ccache.Cache
-	Closed atomic.Bool
-	mu     sync.RWMutex
-}
-
-func NewWithCache(c *ccache.Cache) *Cache {
-	return &Cache{Cache: c}
-}
-
-func (c *Cache) Get(key string) *ccache.Item {
-	if c.Closed.Load() {
-		return nil
-	}
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	return c.Cache.Get(key)
-}
-
-func (c *Cache) Set(key string, value interface{}, duration time.Duration) {
-	if c.Closed.Load() {
-		return
-	}
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.Cache.Set(key, value, duration)
-}
-
-func (c *Cache) Delete(key string) bool {
-	if c.Closed.Load() {
-		return false
-	}
-
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	return c.Cache.Delete(key)
-}
-
-func (c *Cache) Stop() {
-	if c.Closed.Load() {
-		return
-	}
-	c.Closed.Store(true)
-
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.Cache.Stop()
-}