Replacing ccache with generic gcache (#3523)

* Replacing ccache with generic gcache

* fixing lint issues

* removing unecessary hashing + using errorutils

* making test more tolerant

* removing dead code + refactor

* removing redundant code

* removing race

* maint

* moving code

* adding more iterations

* note + typo

* temporary fixing stop-at-first-match with interact

* wrapping internal map with mux

* sort before running integration test

* fix deadlock in requestShouldStopAtFirstMatch

* add timeout to integration_test workflow

* attempting to remove outer lock

* adds interactsh protocol tests in integration_test

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
This commit is contained in:
Mzack9999 2023-04-16 19:49:35 +02:00 committed by GitHub
parent 307085ef4c
commit 6f4b1ae48a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 322 additions and 295 deletions

View File

@ -42,6 +42,7 @@ jobs:
working-directory: v2/
- name: Integration Tests
timeout-minutes: 50
env:
GH_ACTION: true
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

View File

@ -8,9 +8,15 @@ info:
requests:
- method: GET
path:
- "{{BaseURL}}"
- "{{BaseURL}}"
- "{{BaseURL}}"
- "{{BaseURL}}/?a=1"
- "{{BaseURL}}/?a=2"
- "{{BaseURL}}/?a=3"
- "{{BaseURL}}/?a=4"
- "{{BaseURL}}/?a=5"
- "{{BaseURL}}/?a=6"
- "{{BaseURL}}/?a=7"
- "{{BaseURL}}/?a=8"
- "{{BaseURL}}/?a=9"
headers:
url: 'http://{{interactsh-url}}'

View File

@ -88,7 +88,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
defaultOpts.Templates = goflags.StringSlice{templatePath}
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
interactClient, err := interactsh.New(interactOpts)
if err != nil {
return nil, errors.Wrap(err, "could not create interact client")

View File

@ -47,8 +47,6 @@ var httpTestcases = map[string]testutils.TestCase{
"http/http-paths.yaml": &httpPaths{},
"http/request-condition.yaml": &httpRequestCondition{},
"http/request-condition-new.yaml": &httpRequestCondition{},
"http/interactsh.yaml": &httpInteractshRequest{},
"http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{},
"http/self-contained.yaml": &httpRequestSelfContained{},
"http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{},
"http/get-case-insensitive.yaml": &httpGetCaseInsensitive{},
@ -71,7 +69,6 @@ var httpTestcases = map[string]testutils.TestCase{
"http/get-without-scheme.yaml": &httpGetWithoutScheme{},
"http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{},
"http/cl-body-with-header.yaml": &httpCLBodyWithHeader{},
"http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{},
}
type httpInteractshRequest struct{}
@ -164,6 +161,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error {
if err != nil {
return err
}
// polling is asyncronous, so the interactions may be retrieved after the first request
return expectResultsCount(results, 1)
}

View File

@ -4,6 +4,7 @@ import (
"flag"
"fmt"
"os"
"sort"
"strings"
"github.com/logrusorgru/aurora"
@ -22,6 +23,7 @@ var (
protocolTests = map[string]map[string]testutils.TestCase{
"http": httpTestcases,
"interactsh": interactshTestCases,
"network": networkTestcases,
"dns": dnsTestCases,
"workflow": workflowTestcases,
@ -40,9 +42,10 @@ var (
}
// For debug purposes
runProtocol = ""
runTemplate = ""
extraArgs = []string{}
runProtocol = ""
runTemplate = ""
extraArgs = []string{}
interactshRetryCount = 3
)
func main() {
@ -58,7 +61,6 @@ func main() {
}
if runProtocol != "" {
debug = true
debugTests()
os.Exit(1)
}
@ -79,13 +81,36 @@ func main() {
}
}
// execute a testcase with retry and consider best of N
// intended for flaky tests like interactsh
func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) {
var err error
for i := 0; i < retryCount; i++ {
err = testCase.Execute(templatePath)
if err == nil {
fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath)
return "", nil
}
}
_, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed after %v attempts : %s\n", failed, templatePath, retryCount, err)
return templatePath, err
}
func debugTests() {
for tpath, testcase := range protocolTests[runProtocol] {
keys := getMapKeys(protocolTests[runProtocol])
for _, tpath := range keys {
testcase := protocolTests[runProtocol][tpath]
if runTemplate != "" && !strings.Contains(tpath, runTemplate) {
continue
}
if err := testcase.Execute(tpath); err != nil {
fmt.Printf("\n%v", err.Error())
if runProtocol == "interactsh" {
if _, err := executeWithRetry(testcase, tpath, interactshRetryCount); err != nil {
fmt.Printf("\n%v", err.Error())
}
} else {
if _, err := execute(testcase, tpath); err != nil {
fmt.Printf("\n%v", err.Error())
}
}
}
}
@ -97,10 +122,19 @@ func runTests(customTemplatePaths []string) []string {
if len(customTemplatePaths) == 0 {
fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto))
}
keys := getMapKeys(testCases)
for templatePath, testCase := range testCases {
for _, templatePath := range keys {
testCase := testCases[templatePath]
if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, templatePath) {
if failedTemplatePath, err := execute(testCase, templatePath); err != nil {
var failedTemplatePath string
var err error
if proto == "interactsh" {
failedTemplatePath, err = executeWithRetry(testCase, templatePath, interactshRetryCount)
} else {
failedTemplatePath, err = execute(testCase, templatePath)
}
if err != nil {
failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath)
}
}
@ -120,9 +154,10 @@ func execute(testCase testutils.TestCase, templatePath string) (string, error) {
return "", nil
}
func expectResultsCount(results []string, expectedNumber int) error {
if len(results) != expectedNumber {
return fmt.Errorf("incorrect number of results: %d (actual) vs %d (expected) \nResults:\n\t%s\n", len(results), expectedNumber, strings.Join(results, "\n\t"))
func expectResultsCount(results []string, expectedNumbers ...int) error {
match := sliceutil.Contains(expectedNumbers, len(results))
if !match {
return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t"))
}
return nil
}
@ -132,3 +167,12 @@ func normalizeSplit(str string) []string {
return r == ','
})
}
func getMapKeys[T any](testcases map[string]T) []string {
keys := make([]string, 0, len(testcases))
for k := range testcases {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}

View File

@ -0,0 +1,10 @@
package main
import "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
// All Interactsh related testcases
var interactshTestCases = map[string]testutils.TestCase{
"http/interactsh.yaml": &httpInteractshRequest{},
"http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{},
"http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{},
}

View File

@ -50,7 +50,7 @@ func main() {
defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"}
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
interactClient, err := interactsh.New(interactOpts)
if err != nil {
log.Fatalf("Could not create interact client: %s\n", err)

View File

@ -18,7 +18,6 @@ require (
github.com/itchyny/gojq v0.12.11
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/miekg/dns v1.1.53
github.com/olekukonko/tablewriter v0.0.5
@ -53,6 +52,7 @@ require (
require (
github.com/DataDog/gostackparse v0.6.0
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
github.com/antchfx/xmlquery v1.3.15
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
github.com/aws/aws-sdk-go-v2 v1.17.8
@ -111,7 +111,6 @@ require (
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/karlseguin/expect v1.0.8 // indirect
github.com/kataras/jwt v0.1.8 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mackerelio/go-osstat v0.2.4 // indirect

View File

@ -10,6 +10,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y
github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE=
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo=
@ -290,11 +292,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
github.com/karlseguin/expect v1.0.8 h1:Bb0H6IgBWQpadY25UDNkYPDB9ITqK1xnSoZfAq362fw=
github.com/karlseguin/expect v1.0.8/go.mod h1:lXdI8iGiQhmzpnnmU/EGA60vqKs8NbRNFnhhrJGoD5g=
github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk=
github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@ -582,8 +580,6 @@ github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2Z
github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk=
github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo=
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=

View File

@ -262,14 +262,14 @@ func New(options *types.Options) (*Runner, error) {
}
runner.resumeCfg = resumeCfg
opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress)
opts := interactsh.DefaultOptions(runner.output, runner.issuesClient, runner.progress)
opts.Debug = runner.options.Debug
opts.NoColor = runner.options.NoColor
if options.InteractshURL != "" {
opts.ServerURL = options.InteractshURL
}
opts.Authorization = options.InteractshToken
opts.CacheSize = int64(options.InteractionsCacheSize)
opts.CacheSize = options.InteractionsCacheSize
opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second
opts.CooldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second
opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second

View File

@ -85,6 +85,13 @@ func (iwe *InternalWrappedEvent) HasOperatorResult() bool {
return iwe.OperatorsResult != nil
}
func (iwe *InternalWrappedEvent) HasResults() bool {
iwe.RLock()
defer iwe.RUnlock()
return len(iwe.Results) > 0
}
func (iwe *InternalWrappedEvent) SetOperatorResult(operatorResult *operators.Result) {
iwe.Lock()
defer iwe.Unlock()

View File

@ -0,0 +1,18 @@
package interactsh
import (
"regexp"
"time"
)
var (
defaultInteractionDuration = 60 * time.Second
interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`)
)
const (
stopAtFirstMatchAttribute = "stop-at-first-match"
templateIdAttribute = "template-id"
defaultMaxInteractionsCount = 5000
)

View File

@ -2,8 +2,6 @@ package interactsh
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"fmt"
"os"
"regexp"
@ -12,111 +10,56 @@ import (
"sync/atomic"
"time"
"github.com/karlseguin/ccache"
"github.com/pkg/errors"
"errors"
"github.com/Mzack9999/gcache"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/interactsh/pkg/client"
"github.com/projectdiscovery/interactsh/pkg/server"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/utils/atomcache"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
)
// Client is a wrapped client for interactsh server.
type Client struct {
sync.Once
sync.RWMutex
options *Options
// interactsh is a client for interactsh server.
interactsh *client.Client
// requests is a stored cache for interactsh-url->request-event data.
requests *atomcache.Cache
requests gcache.Cache[string, *RequestData]
// interactions is a stored cache for interactsh-interaction->interactsh-url data
interactions *atomcache.Cache
interactions gcache.Cache[string, []*server.Interaction]
// matchedTemplates is a stored cache to track matched templates
matchedTemplates *atomcache.Cache
// interactshURLs is a stored cache to track track multiple interactsh markers
interactshURLs *atomcache.Cache
matchedTemplates gcache.Cache[string, bool]
// interactshURLs is a stored cache to track multiple interactsh markers
interactshURLs gcache.Cache[string, string]
options *Options
eviction time.Duration
pollDuration time.Duration
cooldownDuration time.Duration
dataMutex *sync.RWMutex
hostname string
firstTimeGroup sync.Once
generated uint32 // decide to wait if we have a generated url
matched atomic.Bool
// determines if wait the cooldown period in case of generated URL
generated atomic.Bool
matched atomic.Bool
}
var (
defaultInteractionDuration = 60 * time.Second
interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`)
)
const (
stopAtFirstMatchAttribute = "stop-at-first-match"
templateIdAttribute = "template-id"
)
// Options contains configuration options for interactsh nuclei integration.
type Options struct {
// ServerURL is the URL of the interactsh server.
ServerURL string
// Authorization is the Authorization header value
Authorization string
// CacheSize is the numbers of requests to keep track of at a time.
// Older items are discarded in LRU manner in favor of new requests.
CacheSize int64
// Eviction is the period of time after which to automatically discard
// interaction requests.
Eviction time.Duration
// CooldownPeriod is additional time to wait for interactions after closing
// of the poller.
CooldownPeriod time.Duration
// PollDuration is the time to wait before each poll to the server for interactions.
PollDuration time.Duration
// Output is the output writer for nuclei
Output output.Writer
// IssuesClient is a client for issue exporting
IssuesClient reporting.Client
// Progress is the nuclei progress bar implementation.
Progress progress.Progress
// Debug specifies whether debugging output should be shown for interactsh-client
Debug bool
DebugRequest bool
DebugResponse bool
// DisableHttpFallback controls http retry in case of https failure for server url
DisableHttpFallback bool
// NoInteractsh disables the engine
NoInteractsh bool
// NoColor dissbles printing colors for matches
NoColor bool
StopAtFirstMatch bool
HTTPClient *retryablehttp.Client
}
const defaultMaxInteractionsCount = 5000
// New returns a new interactsh server client
func New(options *Options) (*Client, error) {
configure := ccache.Configure()
configure = configure.MaxSize(options.CacheSize)
cache := atomcache.NewWithCache(ccache.New(configure))
interactionsCfg := ccache.Configure()
interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount)
interactionsCache := atomcache.NewWithCache(ccache.New(interactionsCfg))
matchedTemplateCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)))
interactshURLCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)))
requestsCache := gcache.New[string, *RequestData](options.CacheSize).LRU().Build()
interactionsCache := gcache.New[string, []*server.Interaction](defaultMaxInteractionsCount).LRU().Build()
matchedTemplateCache := gcache.New[string, bool](defaultMaxInteractionsCount).LRU().Build()
interactshURLCache := gcache.New[string, string](defaultMaxInteractionsCount).LRU().Build()
interactClient := &Client{
eviction: options.Eviction,
@ -124,31 +67,14 @@ func New(options *Options) (*Client, error) {
matchedTemplates: matchedTemplateCache,
interactshURLs: interactshURLCache,
options: options,
requests: cache,
requests: requestsCache,
pollDuration: options.PollDuration,
cooldownDuration: options.CooldownPeriod,
dataMutex: &sync.RWMutex{},
}
return interactClient, nil
}
// NewDefaultOptions returns the default options for interactsh client
func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options {
return &Options{
ServerURL: client.DefaultOptions.ServerURL,
CacheSize: 5000,
Eviction: 60 * time.Second,
CooldownPeriod: 5 * time.Second,
PollDuration: 5 * time.Second,
Output: output,
IssuesClient: reporting,
Progress: progress,
DisableHttpFallback: true,
NoColor: false,
}
}
func (c *Client) firstTimeInitializeClient() error {
func (c *Client) poll() error {
if c.options.NoInteractsh {
return nil // do not init if disabled
}
@ -159,40 +85,34 @@ func (c *Client) firstTimeInitializeClient() error {
HTTPClient: c.options.HTTPClient,
})
if err != nil {
return errors.Wrap(err, "could not create client")
return errorutil.NewWithErr(err).Msgf("could not create client")
}
c.interactsh = interactsh
interactURL := interactsh.URL()
interactDomain := interactURL[strings.Index(interactURL, ".")+1:]
gologger.Info().Msgf("Using Interactsh Server: %s", interactDomain)
c.dataMutex.Lock()
c.hostname = interactDomain
c.dataMutex.Unlock()
c.setHostname(interactDomain)
err = interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
item := c.requests.Get(interaction.UniqueID)
if item == nil {
request, err := c.requests.Get(interaction.UniqueID)
if errors.Is(err, gcache.KeyNotFoundError) || request == nil {
// If we don't have any request for this ID, add it to temporary
// lru cache, so we can correlate when we get an add request.
gotItem := c.interactions.Get(interaction.UniqueID)
if gotItem == nil {
c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
} else if items, ok := gotItem.Value().([]*server.Interaction); ok {
items, err := c.interactions.Get(interaction.UniqueID)
if errorutil.IsAny(err, gcache.KeyNotFoundError) || items == nil {
_ = c.interactions.SetWithExpire(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
} else {
items = append(items, interaction)
c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
_ = c.interactions.SetWithExpire(interaction.UniqueID, items, defaultInteractionDuration)
}
return
}
request, ok := item.Value().(*RequestData)
if !ok {
return
}
if _, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent[templateIdAttribute].(string), request.Event.InternalEvent["host"].(string)))
if gotItem != nil {
if requestShouldStopAtFirstMatch(request) || c.options.StopAtFirstMatch {
if gotItem, err := c.matchedTemplates.Get(hash(request.Event.InternalEvent)); gotItem && err == nil {
return
}
}
@ -201,23 +121,44 @@ func (c *Client) firstTimeInitializeClient() error {
})
if err != nil {
return errors.Wrap(err, "could not perform instactsh polling")
return errorutil.NewWithErr(err).Msgf("could not perform interactsh polling")
}
return nil
}
// requestShouldStopAtFirstmatch checks if furthur interactions should be stopped
// note: extra care should be taken while using this function since internalEvent is
// synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that
// we could use `TryLock()` but that may over complicate things and need to differentiate
// situations whether to block or skip
func requestShouldStopAtFirstMatch(request *RequestData) bool {
request.Event.RLock()
defer request.Event.RUnlock()
if stop, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok {
if v, ok := stop.(bool); ok {
return v
}
}
return false
}
// processInteractionForRequest processes an interaction for a request
func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool {
data.Event.Lock()
data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress
data.Event.Unlock()
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, c.options.Debug || c.options.DebugRequest || c.options.DebugResponse)
// if we don't match, return
if !matched || result == nil {
return false // if we don't match, return
return false
}
c.requests.Delete(interaction.UniqueID)
c.requests.Remove(interaction.UniqueID)
if data.Event.OperatorsResult != nil {
data.Event.OperatorsResult.Merge(result)
@ -225,10 +166,12 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.SetOperatorResult(result)
}
data.Event.Lock()
data.Event.Results = data.MakeResultFunc(data.Event)
for _, event := range data.Event.Results {
event.Interaction = interaction
}
data.Event.Unlock()
if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse {
c.debugPrintInteraction(interaction, data.Event.OperatorsResult)
@ -236,30 +179,39 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) {
c.matched.Store(true)
if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
c.matchedTemplates.Set(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration)
if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch {
_ = c.matchedTemplates.SetWithExpire(hash(data.Event.InternalEvent), true, defaultInteractionDuration)
}
}
return true
}
func (c *Client) AlreadyMatched(data *RequestData) bool {
data.Event.RLock()
defer data.Event.RUnlock()
return c.matchedTemplates.Has(hash(data.Event.InternalEvent))
}
// URL returns a new URL that can be interacted with
func (c *Client) URL() (string, error) {
c.firstTimeGroup.Do(func() {
if err := c.firstTimeInitializeClient(); err != nil {
gologger.Error().Msgf("Could not initialize interactsh client: %s", err)
}
})
if c.interactsh == nil {
return "", errors.New("interactsh client not initialized")
var err error
c.Do(func() {
err = c.poll()
})
if err != nil {
return "", errorutil.NewWithErr(err).Msgf("interactsh client not initialized")
}
}
atomic.CompareAndSwapUint32(&c.generated, 0, 1)
c.generated.Store(true)
return c.interactsh.URL(), nil
}
// Close closes the interactsh clients after waiting for cooldown period.
// Close the interactsh clients after waiting for cooldown period.
func (c *Client) Close() bool {
if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 {
if c.cooldownDuration > 0 && c.generated.Load() {
time.Sleep(c.cooldownDuration)
}
if c.interactsh != nil {
@ -267,15 +219,10 @@ func (c *Client) Close() bool {
c.interactsh.Close()
}
closeCache := func(cc *atomcache.Cache) {
if cc != nil {
cc.Stop()
}
}
closeCache(c.requests)
closeCache(c.interactions)
closeCache(c.matchedTemplates)
closeCache(c.interactshURLs)
c.requests.Purge()
c.interactions.Purge()
c.matchedTemplates.Purge()
c.interactshURLs.Purge()
return c.matched.Load()
}
@ -308,36 +255,29 @@ func (c *Client) NewURLWithData(data string) (string, error) {
if url == "" {
return "", errors.New("empty interactsh url")
}
c.interactshURLs.Set(url, data, defaultInteractionDuration)
_ = c.interactshURLs.SetWithExpire(url, data, defaultInteractionDuration)
return url, nil
}
// MakePlaceholders does placeholders for interact URLs and other data to a map
func (c *Client) MakePlaceholders(urls []string, data map[string]interface{}) {
data["interactsh-server"] = c.getInteractServerHostname()
data["interactsh-server"] = c.getHostname()
for _, url := range urls {
if interactshURLMarker := c.interactshURLs.Get(url); interactshURLMarker != nil {
if interactshURLMarker, ok := interactshURLMarker.Value().(string); ok {
interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}")
if interactshURLMarker, err := c.interactshURLs.Get(url); interactshURLMarker != "" && err == nil {
interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}")
c.interactshURLs.Delete(url)
c.interactshURLs.Remove(url)
data[interactshMarker] = url
urlIndex := strings.Index(url, ".")
if urlIndex == -1 {
continue
}
data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex]
data[interactshMarker] = url
urlIndex := strings.Index(url, ".")
if urlIndex == -1 {
continue
}
data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex]
}
}
}
// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options
func (c *Client) SetStopAtFirstMatch() {
c.options.StopAtFirstMatch = true
}
// MakeResultEventFunc is a result making function for nuclei
type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
@ -352,35 +292,26 @@ type RequestData struct {
// RequestEvent is the event for a network request sent by nuclei.
func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) {
data.Event.Lock()
defer data.Event.Unlock()
for _, interactshURL := range interactshURLs {
id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.hostname), ".")
id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.getHostname()), ".")
if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
gotItem := c.matchedTemplates.Get(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)))
if gotItem != nil {
if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch {
gotItem, err := c.matchedTemplates.Get(hash(data.Event.InternalEvent))
if gotItem && err == nil {
break
}
}
interaction := c.interactions.Get(id)
if interaction != nil {
// If we have previous interactions, get them and process them.
interactions, ok := interaction.Value().([]*server.Interaction)
if !ok {
c.requests.Set(id, data, c.eviction)
return
}
interactions, err := c.interactions.Get(id)
if interactions != nil && err == nil {
for _, interaction := range interactions {
if c.processInteractionForRequest(interaction, data) {
c.interactions.Delete(id)
c.interactions.Remove(id)
break
}
}
} else {
c.requests.Set(id, data, c.eviction)
_ = c.requests.SetWithExpire(id, data, c.eviction)
}
}
}
@ -397,16 +328,16 @@ func HasMatchers(op *operators.Operators) bool {
for _, matcher := range op.Matchers {
for _, dsl := range matcher.DSL {
if strings.Contains(dsl, "interactsh") {
if stringsutil.ContainsAnyI(dsl, "interactsh") {
return true
}
}
if strings.HasPrefix(matcher.Part, "interactsh") {
if stringsutil.HasPrefixI(matcher.Part, "interactsh") {
return true
}
}
for _, matcher := range op.Extractors {
if strings.HasPrefix(matcher.Part, "interactsh") {
if stringsutil.HasPrefixI(matcher.Part, "interactsh") {
return true
}
}
@ -461,16 +392,22 @@ func formatInteractionMessage(key, value string, event *operators.Result, noColo
return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value)
}
func hash(templateID, host string) string {
h := sha1.New()
h.Write([]byte(templateID))
h.Write([]byte(host))
return hex.EncodeToString(h.Sum(nil))
func hash(internalEvent output.InternalEvent) string {
templateId := internalEvent[templateIdAttribute].(string)
host := internalEvent["host"].(string)
return fmt.Sprintf("%s:%s", templateId, host)
}
func (c *Client) getInteractServerHostname() string {
c.dataMutex.RLock()
defer c.dataMutex.RUnlock()
func (c *Client) getHostname() string {
c.RLock()
defer c.RUnlock()
return c.hostname
}
func (c *Client) setHostname(hostname string) {
c.Lock()
defer c.Unlock()
c.hostname = hostname
}

View File

@ -0,0 +1,67 @@
package interactsh
import (
"time"
"github.com/projectdiscovery/interactsh/pkg/client"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/retryablehttp-go"
)
// Options contains configuration options for interactsh nuclei integration.
type Options struct {
// ServerURL is the URL of the interactsh server.
ServerURL string
// Authorization is the Authorization header value
Authorization string
// CacheSize is the numbers of requests to keep track of at a time.
// Older items are discarded in LRU manner in favor of new requests.
CacheSize int
// Eviction is the period of time after which to automatically discard
// interaction requests.
Eviction time.Duration
// CooldownPeriod is additional time to wait for interactions after closing
// of the poller.
CooldownPeriod time.Duration
// PollDuration is the time to wait before each poll to the server for interactions.
PollDuration time.Duration
// Output is the output writer for nuclei
Output output.Writer
// IssuesClient is a client for issue exporting
IssuesClient reporting.Client
// Progress is the nuclei progress bar implementation.
Progress progress.Progress
// Debug specifies whether debugging output should be shown for interactsh-client
Debug bool
// DebugRequest outputs interaction request
DebugRequest bool
// DebugResponse outputs interaction response
DebugResponse bool
// DisableHttpFallback controls http retry in case of https failure for server url
DisableHttpFallback bool
// NoInteractsh disables the engine
NoInteractsh bool
// NoColor dissbles printing colors for matches
NoColor bool
StopAtFirstMatch bool
HTTPClient *retryablehttp.Client
}
// DefaultOptions returns the default options for interactsh client
func DefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options {
return &Options{
ServerURL: client.DefaultOptions.ServerURL,
CacheSize: 5000,
Eviction: 60 * time.Second,
CooldownPeriod: 5 * time.Second,
PollDuration: 5 * time.Second,
Output: output,
IssuesClient: reporting,
Progress: progress,
DisableHttpFallback: true,
NoColor: false,
}
}

View File

@ -186,7 +186,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
generator.options.Interactsh, err = interactsh.New(&interactsh.Options{
ServerURL: options.InteractshURL,
CacheSize: int64(options.InteractionsCacheSize),
CacheSize: options.InteractionsCacheSize,
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,

View File

@ -248,24 +248,26 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
}
var gotMatches bool
requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
gotMatches = event.OperatorsResult.Matched
}
if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
request.options.Interactsh.RequestEvent(gr.InteractURLs, &interactsh.RequestData{
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
})
}
request.options.Interactsh.RequestEvent(gr.InteractURLs, requestData)
gotMatches = request.options.Interactsh.AlreadyMatched(requestData)
} else {
callback(event)
}
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
gotMatches = event.OperatorsResult.Matched
}
}, 0)
// If a variable is unresolved, skip all further requests
if requestErr == errStopExecution {
if errors.Is(requestErr, errStopExecution) {
return false
}
if requestErr != nil {
@ -276,7 +278,8 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
request.options.Progress.IncrementRequests()
// If this was a match, and we want to stop at first match, skip all further requests.
if (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches {
shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
if shouldStopAtFirstMatch && gotMatches {
return false
}
return true
@ -374,19 +377,21 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}
var gotMatches bool
err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
gotMatches = event.OperatorsResult.Matched
gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues)
}
if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
})
}
request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, requestData)
gotMatches = request.options.Interactsh.AlreadyMatched(requestData)
}
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
gotMatches = event.OperatorsResult.Matched
gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues)
}
// Note: This is a race condition prone zone i.e when request has interactsh_matchers
// Interactsh.RequestEvent tries to access/update output.InternalWrappedEvent depending on logic
@ -397,7 +402,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}, generator.currentIndex)
// If a variable is unresolved, skip all further requests
if err == errStopExecution {
if errors.Is(err, errStopExecution) {
return true, nil
}
if err != nil {
@ -409,7 +414,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
request.options.Progress.IncrementRequests()
// If this was a match, and we want to stop at first match, skip all further requests.
if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches {
shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch
if shouldStopAtFirstMatch && gotMatches {
return true, nil
}
return false, nil
@ -755,7 +761,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
callback(event)
// Skip further responses if we have stop-at-first-match and a match
if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && len(event.Results) > 0 {
if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() {
return nil
}
}

View File

@ -1,62 +0,0 @@
package atomcache
import (
"sync"
"sync/atomic"
"time"
"github.com/karlseguin/ccache"
)
type Cache struct {
*ccache.Cache
Closed atomic.Bool
mu sync.RWMutex
}
func NewWithCache(c *ccache.Cache) *Cache {
return &Cache{Cache: c}
}
func (c *Cache) Get(key string) *ccache.Item {
if c.Closed.Load() {
return nil
}
c.mu.RLock()
defer c.mu.RUnlock()
return c.Cache.Get(key)
}
func (c *Cache) Set(key string, value interface{}, duration time.Duration) {
if c.Closed.Load() {
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.Cache.Set(key, value, duration)
}
func (c *Cache) Delete(key string) bool {
if c.Closed.Load() {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
return c.Cache.Delete(key)
}
func (c *Cache) Stop() {
if c.Closed.Load() {
return
}
c.Closed.Store(true)
c.mu.Lock()
defer c.mu.Unlock()
c.Cache.Stop()
}