fix: data race at protocolstate, contextargs & some outdated test cases (#5820)

* test(flow): update outdated test cases

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test(multiproto): update outdated test cases

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat: fixed failing tests

* fixed data race

* fixed memgaurdian race conditiong

* test(customtemplates): use test repo

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(customtemplates): add more `{Clone,Pull}Options`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(customtemplates): validate `{Clone,Pull}Options`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* bugfix: fixed failing integration tests for flow and multi

* chore: either 1 or 2 results in interactsh

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>
Co-authored-by: Ice3man <nizamulrana@gmail.com>

cmd/integration-test/flow.go

@@ -22,7 +22,7 @@ var flowTestcases = []TestCaseInfo{
 type conditionalFlow struct{}
 
 func (t *conditionalFlow) Execute(filePath string) error {
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "cloud.projectdiscovery.io", debug)
 	if err != nil {
 		return err
 	}

cmd/integration-test/http.go

@@ -155,7 +155,7 @@ func (h *httpInteractshRequest) Execute(filePath string) error {
 		return err
 	}
 
-	return expectResultsCount(results, 1)
+	return expectResultsCount(results, 1, 2)
 }
 
 type httpDefaultMatcherCondition struct{}

cmd/integration-test/multi.go

@@ -14,7 +14,7 @@ type multiProtoDynamicExtractor struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *multiProtoDynamicExtractor) Execute(templatePath string) error {
-	results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "blog.projectdiscovery.io", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "docs.projectdiscovery.io", debug)
 	if err != nil {
 		return err
 	}

integration_tests/flow/conditional-flow.yaml

@@ -14,7 +14,7 @@ dns:
     matchers:
       - type: word
         words:
-          - "ghost.io"
+          - ".vercel-dns.com"
         internal: true
 
 http:
@@ -25,4 +25,4 @@ http:
     matchers:
       - type: word
         words:
-          - "ghost.io"
+          - "html>"

integration_tests/protocols/code/pre-condition.yaml

@@ -23,4 +23,4 @@ code:
       - type: dsl
         dsl:
           - true
-# digest: 4a0a0047304502200307590191cb7c766b6c21e5777d345bdddf7adf9d6da8f7d336d585d9ac4a8b022100fd30fb0c7722778eb3d861d60e721d805925b8d8df2b979ef2104c35ec57d5cb:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a00463044022048c083c338c0195f5012122d40c1009d2e2030c583e56558e0d6249a41e6f3f4022070656adf748f4874018d7a01fce116db10a3acd1f9b03e12a83906fb625b5c50:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a0047304502203fe1d7d52bc2a41886d576a90c82c3be42078baaa4b46e1f3d8519665d6f88b202210081feb82c41150c5b218e226fc4f299ded19f42ba01ef34ba60b0634b4ea6ee12:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a00483046022100cbbdb7214f669d111b671d271110872dc8af2ab41cf5c312b6e4f64126f55337022100a60547952a0c2bea58388f2d2effe8ad73cd6b6fc92e73eb3c8f88beab6105ec:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a00483046022100afb5ebff14a40e7f9b679ffc4d93ce7849e33eb398ebb47f2e757cd24831f9dd02210089ffa21b2763e99ebce95dfc5b91e1e62da4ccdc9d2ad5c48584fa350ba335af:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022032b81e8bb7475abf27639b0ced71355497166d664698021f26498e7031d62a23022100e99ccde578bfc0b658f16427ae9a3d18922849d3ba3e022032ea0d2a8e77fadb:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4b0a00483046022100939f83e74d43932a5bd792b1fd2c100eec2df60f2b2a8dd56b5c8ef5faa92b17022100f93031b0de373af7d78e623968ea5a2d67c4561ef70e3e6da15aef7e5c853115:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a0047304502201a5dd0eddfab4f02588a5a8ac1947a5fa41fed80b59d698ad5cc00456296efb6022100fe6e608e38c060964800f5f863a7cdc93f686f2d0f4b52854f73948b808b4511:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-snippet.yaml

@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a00473045022100b8e676ce0c57b60c233a0203539dec20457bbb5f1790d351a5d45405b6668b2602204b1f2fa18e7db099f05329009597ceb2d9b7337562c1a676e8d50ea2f1c6fcbe:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a00483046022100ced1702728cc68f906c4c7d2c4d05ed071bfabee1e36eec7ebecbeca795a170c022100d20fd41796f130a8f9c4972fee85386d67d61eb5fc1119b1afe2a851eb2f3e65:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/multi/dynamic-values.yaml

@@ -13,7 +13,7 @@ dns:
       - type: dsl
         name: blogid
         dsl:
-          - trim_suffix(cname,'.ghost.io')
+          - trim_suffix(cname,'.vercel-dns.com')
         internal: true
 
 http:
@@ -24,6 +24,6 @@ http:
     matchers:
       - type: dsl
         dsl:
-          - contains(body,'ProjectDiscovery.io') # check for http string
-          - blogid == 'projectdiscovery' # check for cname (extracted information from dns response)
+          - contains(body,'introduction') # check for http string
+          - blogid == 'cname' # check for cname (extracted information from dns response)
         condition: and

integration_tests/protocols/multi/evaluate-variables.yaml

@@ -7,7 +7,7 @@ info:
 
 
 variables:
-  cname_filtered: '{{trim_suffix(dns_cname,".ghost.io")}}'
+  cname_filtered: '{{trim_suffix(dns_cname,".vercel-dns.com")}}'
 
 dns:
   - name: "{{FQDN}}" # DNS Request
@@ -24,7 +24,7 @@ http:
     matchers:
       - type: dsl
         dsl:
-          - contains(http_body,'ProjectDiscovery.io') # check for http string
-          - cname_filtered == 'projectdiscovery' # check for cname (extracted information from dns response)
-          - ssl_subject_cn == 'blog.projectdiscovery.io'
+          - contains(http_body,'introduction') # check for http string
+          - cname_filtered == 'cname' # check for cname (extracted information from dns response)
+          - ssl_subject_cn == 'docs.projectdiscovery.io'
         condition: and

integration_tests/protocols/multi/exported-response-vars.yaml

@@ -20,7 +20,7 @@ http:
     matchers:
       - type: dsl
         dsl:
-          - contains(http_body,'ProjectDiscovery.io') # check for http string
-          - trim_suffix(dns_cname,'.ghost.io') == 'projectdiscovery' # check for cname (extracted information from dns response)
-          - ssl_subject_cn == 'blog.projectdiscovery.io'
+          - contains(http_body,'introduction') # check for http string
+          - trim_suffix(dns_cname,'.vercel-dns.com') == 'cname' # check for cname (extracted information from dns response)
+          - ssl_subject_cn == 'docs.projectdiscovery.io'
         condition: and

integration_tests/workflow/code-template-1.yaml

@@ -19,4 +19,4 @@ code:
         regex:
           - 'hello from (.*)'
         group: 1
-# digest: 490a00463044022050da011362cf08c2cb81e812c7f86d7282afe0562d4bf00d390f1300d19bc910022029e9d305da69e941ac18797645aecb217abde6557f891e141301b48e89a3c0cd:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a0046304402206b3648e8d393ac4df82c7d59b1a6ee3731c66c249dbd4d9bf31f0b7f176b37ec02203184d36373e516757c7d708b5799bc16edb1cebc0a64f3442d13ded4b33c42fb:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-template-2.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from first"
-# digest: 4b0a00483046022100b3b8759c0df028455eb59b1433ac240e5d4604b011bb0c63680bd3cc159ac6f0022100f44aa11b640d11ad0e2902897f4eb51666ab3cd83c31dfd2590f6e43391e39b0:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a0046304402204cbb1bdf8370e49bb930b17460fb35e15f285a3b48b165736ac0e7ba2f9bc0fb022067c134790c4a2cf646b195aa4488e2c222266436e6bda47931908a28807bdb81:4a3eb6b4988d95847d4203be25ed1d46

pkg/external/customtemplates/github.go

@@ -137,33 +137,59 @@ getRepo:
 
 // download the git repo to a given path
 func (ctr *customTemplateGitHubRepo) cloneRepo(clonePath, githubToken string) error {
-	r, err := git.PlainClone(clonePath, false, &git.CloneOptions{
+	cloneOpts := &git.CloneOptions{
 		URL:          ctr.gitCloneURL,
 		Auth:         getAuth(ctr.owner, githubToken),
-	})
+		SingleBranch: true,
+		Depth:        1,
+	}
+
+	err := cloneOpts.Validate()
+	if err != nil {
+		return err
+	}
+
+	r, err := git.PlainClone(clonePath, false, cloneOpts)
 	if err != nil {
 		return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
 	}
+
 	// Add the user as well in the config. By default, user is not set
 	config, _ := r.Storer.Config()
 	config.User.Name = ctr.owner
+
 	return r.SetConfig(config)
 }
 
 // performs the git pull on given repo
 func (ctr *customTemplateGitHubRepo) pullChanges(repoPath, githubToken string) error {
+	pullOpts := &git.PullOptions{
+		RemoteName:   "origin",
+		Auth:         getAuth(ctr.owner, githubToken),
+		SingleBranch: true,
+		Depth:        1,
+	}
+
+	err := pullOpts.Validate()
+	if err != nil {
+		return err
+	}
+
 	r, err := git.PlainOpen(repoPath)
 	if err != nil {
 		return err
 	}
+
 	w, err := r.Worktree()
 	if err != nil {
 		return err
 	}
-	err = w.Pull(&git.PullOptions{RemoteName: "origin", Auth: getAuth(ctr.owner, githubToken)})
+
+	err = w.Pull(pullOpts)
 	if err != nil {
 		return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
 	}
+
 	return nil
 }
 

pkg/external/customtemplates/github_test.go

@@ -22,14 +22,12 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) {
 	config.DefaultConfig.SetTemplatesDir(templatesDirectory)
 
 	options := testutils.DefaultOptions
-	options.GitHubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"}
-	options.GitHubToken = os.Getenv("GITHUB_TOKEN")
+	options.GitHubTemplateRepo = []string{"projectdiscovery/nuclei-templates-test"}
 
 	ctm, err := NewCustomTemplatesManager(options)
 	require.Nil(t, err, "could not create custom templates manager")
 
 	ctm.Download(context.Background())
 
-	require.DirExists(t, filepath.Join(templatesDirectory, "github", "projectdiscovery", "nuclei-templates"), "cloned directory does not exists")
-	require.DirExists(t, filepath.Join(templatesDirectory, "github", "ehsandeep", "nuclei-templates"), "cloned directory does not exists")
+	require.DirExists(t, filepath.Join(templatesDirectory, "github", "projectdiscovery", "nuclei-templates-test"), "cloned directory does not exists")
 }

pkg/protocols/common/contextargs/metainput.go

@@ -142,6 +142,9 @@ func (metaInput *MetaInput) Unmarshal(data string) error {
 }
 
 func (metaInput *MetaInput) Clone() *MetaInput {
+	metaInput.mu.Lock()
+	defer metaInput.mu.Unlock()
+
 	input := NewMetaInput()
 	input.Input = metaInput.Input
 	input.CustomIP = metaInput.CustomIP

pkg/protocols/common/protocolstate/memguardian.go

@@ -19,7 +19,7 @@ var (
 )
 
 func StartActiveMemGuardian(ctx context.Context) {
-	if memguardian.DefaultMemGuardian == nil {
+	if memguardian.DefaultMemGuardian == nil || memTimer != nil {
 		return
 	}
 

pkg/tmplexec/flow/flow_executor_test.go

@@ -137,7 +137,7 @@ func TestFlowWithConditionPositive(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	input := contextargs.NewWithInput(context.Background(), "cloud.projectdiscovery.io")
 	ctx := scan.NewScanContext(context.Background(), input)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err := Template.Executer.Execute(ctx)
@@ -150,7 +150,22 @@ func TestFlowWithNoMatchers(t *testing.T) {
 	// when using conditional flow with no matchers at all
 	// we implicitly assume that request was successful and internally changed the result to true (for scope of condition only)
 
-	// testcase-1 : no matchers but contains extractor
+	Template, err := templates.Parse("testcases/condition-flow-no-operators.yaml", nil, executerOpts)
+	require.Nil(t, err, "could not parse template")
+
+	require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
+
+	err = Template.Executer.Compile()
+	require.Nil(t, err, "could not compile template")
+
+	anotherInput := contextargs.NewWithInput(context.Background(), "cloud.projectdiscovery.io")
+	anotherCtx := scan.NewScanContext(context.Background(), anotherInput)
+	// positive match . expect results also verify that both dns() and http() were executed
+	gotresults, err := Template.Executer.Execute(anotherCtx)
+	require.Nil(t, err, "could not execute template")
+	require.True(t, gotresults)
+
+	t.Run("Contains Extractor", func(t *testing.T) {
 		Template, err := templates.Parse("testcases/condition-flow-extractors.yaml", nil, executerOpts)
 		require.Nil(t, err, "could not parse template")
 
@@ -159,27 +174,11 @@ func TestFlowWithNoMatchers(t *testing.T) {
 		err = Template.Executer.Compile()
 		require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+		input := contextargs.NewWithInput(context.Background(), "scanme.sh")
 		ctx := scan.NewScanContext(context.Background(), input)
 		// positive match . expect results also verify that both dns() and http() were executed
 		gotresults, err := Template.Executer.Execute(ctx)
 		require.Nil(t, err, "could not execute template")
 		require.True(t, gotresults)
-
-	// testcase-2 : no matchers and no extractors
-	Template, err = templates.Parse("testcases/condition-flow-no-operators.yaml", nil, executerOpts)
-	require.Nil(t, err, "could not parse template")
-
-	require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
-
-	err = Template.Executer.Compile()
-	require.Nil(t, err, "could not compile template")
-
-	anotherInput := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
-	anotherCtx := scan.NewScanContext(context.Background(), anotherInput)
-	// positive match . expect results also verify that both dns() and http() were executed
-	gotresults, err = Template.Executer.Execute(anotherCtx)
-	require.Nil(t, err, "could not execute template")
-	require.True(t, gotresults)
-
+	})
 }

pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml

@@ -1,29 +1,28 @@
-id: ghost-blog-detection
+id: condition-flow-extractors
 info:
-  name: Ghost blog detection
+  name: Condition Flow Extractors
   author: pdteam
   severity: info
 
-
 flow: dns() && http()
 
 dns:
   - name: "{{FQDN}}"
-    type: CNAME
+    type: A
 
     extractors:
       - type: dsl
-        name: cname
+        name: a
         internal: true
         dsl:
-            - cname
+          - a
 
 http:
   - method: GET
     path:
-      - "{{BaseURL}}?ref={{cname}}"
+      - "{{BaseURL}}/?ref={{a}}"
 
     matchers:
       - type: word
         words:
-          - "ghost.io"
+          - "ok"

pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml

@@ -1,13 +1,11 @@
-id: ghost-blog-detection
+id: condition-flow-no-operators
 info:
-  name: Ghost blog detection
+  name: Condition Flow No Operators
   author: pdteam
   severity: info
 
-
 flow: dns() && http()
 
-
 dns:
   - name: "{{FQDN}}"
     type: CNAME
@@ -15,9 +13,9 @@ dns:
 http:
   - method: GET
     path:
-      - "{{BaseURL}}?ref={{dns_cname}}"
+      - "{{BaseURL}}/?ref={{dns_cname}}"
 
     matchers:
       - type: word
         words:
-          - "ghost.io"
+          - "html>"

pkg/tmplexec/flow/testcases/condition-flow.yaml

@@ -1,6 +1,6 @@
-id: ghost-blog-detection
+id: vercel-hosted-detection
 info:
-  name: Ghost blog detection
+  name: Vercel-hosted detection
   author: pdteam
   severity: info
 
@@ -14,14 +14,14 @@ dns:
     matchers:
       - type: word
         words:
-          - "ghost.io"
+          - "vercel-dns"
 
 http:
   - method: GET
     path:
-      - "{{BaseURL}}"
+      - "{{dns_cname}}"
 
     matchers:
       - type: word
         words:
-          - "ghost.io"
+          - "DEPLOYMENT_NOT_FOUND"

pkg/tmplexec/multiproto/multi_test.go

@@ -56,7 +56,7 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	input := contextargs.NewWithInput(context.Background(), "http://scanme.sh")
 	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -72,7 +72,7 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	input := contextargs.NewWithInput(context.Background(), "https://cloud.projectdiscovery.io/sign-in")
 	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")

pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml

@@ -7,15 +7,7 @@ info:
 
 dns:
   - name: "{{FQDN}}" # DNS Request
-    type: cname
-
-    extractors:
-      - type: dsl
-        name: blogid
-        dsl:
-          - trim_suffix(cname,'.ghost.io')
-        internal: true
-
+    type: a
 
 http:
   - method: GET # http request
@@ -25,6 +17,6 @@ http:
     matchers:
       - type: dsl
         dsl:
-          - contains(body,'ProjectDiscovery.io') # check for http string
-          - blogid == 'projectdiscovery' # check for cname (extracted information from dns response)
+          - body == "ok"
+          - dns_a == '128.199.158.128' # check for A record (extracted information from dns response)
         condition: and

pkg/tmplexec/multiproto/testcases/multiprotowithprefix.yaml

@@ -20,7 +20,7 @@ http:
     matchers:
       - type: dsl
         dsl:
-          - contains(http_body,'ProjectDiscovery.io') # check for http string
-          - trim_suffix(dns_cname,'.ghost.io') == 'projectdiscovery' # check for cname (extracted information from dns response)
-          - ssl_subject_cn == 'blog.projectdiscovery.io'
+          - contains(http_body, 'ProjectDiscovery Cloud Platform') # check for http string
+          - dns_cname == 'cname.vercel-dns.com' # check for cname (extracted information from dns response)
+          - ssl_subject_cn == 'cloud.projectdiscovery.io'
         condition: and