diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index e55cf986a..94e14315f 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -2506,6 +2506,8 @@ Enum Values:
- TXT
- AAAA
+
+ - CAA
diff --git a/integration_tests/dns/caa.yaml b/integration_tests/dns/caa.yaml
new file mode 100644
index 000000000..9a2ffc987
--- /dev/null
+++ b/integration_tests/dns/caa.yaml
@@ -0,0 +1,22 @@
+id: caa-fingerprinting
+
+info:
+ name: CAA Fingerprint
+ author: pdteam
+ severity: info
+ tags: dns,caa
+
+dns:
+ - name: "{{FQDN}}"
+ type: CAA
+
+ matchers:
+ - type: word
+ words:
+ - "IN\tCAA"
+
+ extractors:
+ - type: regex
+ group: 1
+ regex:
+ - "IN\tCAA\t(.+)"
\ No newline at end of file
diff --git a/integration_tests/http/stop-at-first-match-with-extractors.yaml b/integration_tests/http/stop-at-first-match-with-extractors.yaml
new file mode 100644
index 000000000..b22e9f4ec
--- /dev/null
+++ b/integration_tests/http/stop-at-first-match-with-extractors.yaml
@@ -0,0 +1,18 @@
+id: stop-at-first-match-with-extractors
+
+info:
+ name: Stop at first match Request with extractors
+ author: pdteam
+ severity: info
+
+requests:
+ - method: GET
+ path:
+ - "{{BaseURL}}?a=1"
+ - "{{BaseURL}}?a=2"
+ stop-at-first-match: true
+ extractors:
+ - type: kval
+ part: header
+ kval:
+ - "date"
\ No newline at end of file
diff --git a/integration_tests/http/stop-at-first-match.yaml b/integration_tests/http/stop-at-first-match.yaml
new file mode 100644
index 000000000..a5a06da78
--- /dev/null
+++ b/integration_tests/http/stop-at-first-match.yaml
@@ -0,0 +1,17 @@
+id: stop-at-first-match
+
+info:
+ name: Stop at first match Request
+ author: pdteam
+ severity: info
+
+requests:
+ - method: GET
+ path:
+ - "{{BaseURL}}?a=1"
+ - "{{BaseURL}}?a=2"
+ matchers:
+ - type: word
+ words:
+ - "This is test"
+ stop-at-first-match: true
\ No newline at end of file
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 6e4148745..a0bed62d9 100755
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -338,7 +338,8 @@
"PTR",
"MX",
"TXT",
- "AAAA"
+ "AAAA",
+ "CAA"
],
"type": "string",
"title": "type of DNS request to make",
diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go
index 8e3b7213a..e0068fa90 100644
--- a/v2/cmd/integration-test/dns.go
+++ b/v2/cmd/integration-test/dns.go
@@ -7,6 +7,7 @@ import (
var dnsTestCases = map[string]testutils.TestCase{
"dns/basic.yaml": &dnsBasic{},
"dns/ptr.yaml": &dnsPtr{},
+ "dns/caa.yaml": &dnsCAA{},
}
type dnsBasic struct{}
@@ -40,3 +41,19 @@ func (h *dnsPtr) Execute(filePath string) error {
}
return expectResultsCount(results, 1)
}
+
+type dnsCAA struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *dnsCAA) Execute(filePath string) error {
+ var routerErr error
+
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug)
+ if err != nil {
+ return err
+ }
+ if routerErr != nil {
+ return routerErr
+ }
+ return expectResultsCount(results, 1)
+}
diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go
index 90a776ba0..19a6c25c3 100644
--- a/v2/cmd/integration-test/http.go
+++ b/v2/cmd/integration-test/http.go
@@ -18,32 +18,34 @@ import (
)
var httpTestcases = map[string]testutils.TestCase{
- "http/get-headers.yaml": &httpGetHeaders{},
- "http/get-query-string.yaml": &httpGetQueryString{},
- "http/get-redirects.yaml": &httpGetRedirects{},
- "http/get.yaml": &httpGet{},
- "http/post-body.yaml": &httpPostBody{},
- "http/post-json-body.yaml": &httpPostJSONBody{},
- "http/post-multipart-body.yaml": &httpPostMultipartBody{},
- "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{},
- "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{},
- "http/raw-get-query.yaml": &httpRawGetQuery{},
- "http/raw-get.yaml": &httpRawGet{},
- "http/raw-payload.yaml": &httpRawPayload{},
- "http/raw-post-body.yaml": &httpRawPostBody{},
- "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{},
- "http/request-condition.yaml": &httpRequestCondition{},
- "http/request-condition-new.yaml": &httpRequestCondition{},
- "http/interactsh.yaml": &httpInteractshRequest{},
- "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{},
- "http/self-contained.yaml": &httpRequestSelContained{},
- "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{},
- "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
- "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{},
- "http/dsl-matcher-variable.yaml": &httpDSLVariable{},
- "http/dsl-functions.yaml": &httpDSLFunctions{},
- "http/race-simple.yaml": &httpRaceSimple{},
- "http/race-multiple.yaml": &httpRaceMultiple{},
+ "http/get-headers.yaml": &httpGetHeaders{},
+ "http/get-query-string.yaml": &httpGetQueryString{},
+ "http/get-redirects.yaml": &httpGetRedirects{},
+ "http/get.yaml": &httpGet{},
+ "http/post-body.yaml": &httpPostBody{},
+ "http/post-json-body.yaml": &httpPostJSONBody{},
+ "http/post-multipart-body.yaml": &httpPostMultipartBody{},
+ "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{},
+ "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{},
+ "http/raw-get-query.yaml": &httpRawGetQuery{},
+ "http/raw-get.yaml": &httpRawGet{},
+ "http/raw-payload.yaml": &httpRawPayload{},
+ "http/raw-post-body.yaml": &httpRawPostBody{},
+ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{},
+ "http/request-condition.yaml": &httpRequestCondition{},
+ "http/request-condition-new.yaml": &httpRequestCondition{},
+ "http/interactsh.yaml": &httpInteractshRequest{},
+ "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{},
+ "http/self-contained.yaml": &httpRequestSelContained{},
+ "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{},
+ "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
+ "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{},
+ "http/dsl-matcher-variable.yaml": &httpDSLVariable{},
+ "http/dsl-functions.yaml": &httpDSLFunctions{},
+ "http/race-simple.yaml": &httpRaceSimple{},
+ "http/race-multiple.yaml": &httpRaceMultiple{},
+ "http/stop-at-first-match.yaml": &httpStopAtFirstMatch{},
+ "http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{},
}
type httpInteractshRequest struct{}
@@ -727,3 +729,41 @@ func (h *httpRaceMultiple) Execute(filePath string) error {
}
return expectResultsCount(results, 5)
}
+
+type httpStopAtFirstMatch struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpStopAtFirstMatch) Execute(filePath string) error {
+ router := httprouter.New()
+ router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ fmt.Fprintf(w, "This is test")
+ })
+ ts := httptest.NewServer(router)
+ defer ts.Close()
+
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+ if err != nil {
+ return err
+ }
+
+ return expectResultsCount(results, 1)
+}
+
+type httpStopAtFirstMatchWithExtractors struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpStopAtFirstMatchWithExtractors) Execute(filePath string) error {
+ router := httprouter.New()
+ router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ fmt.Fprintf(w, "This is test")
+ })
+ ts := httptest.NewServer(router)
+ defer ts.Close()
+
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+ if err != nil {
+ return err
+ }
+
+ return expectResultsCount(results, 2)
+}
diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go
index d118a6411..9c0439c91 100644
--- a/v2/cmd/integration-test/loader.go
+++ b/v2/cmd/integration-test/loader.go
@@ -2,6 +2,7 @@ package main
import (
"fmt"
+ "io/ioutil"
"net/http"
"net/http/httptest"
"os"
@@ -17,6 +18,7 @@ var loaderTestcases = map[string]testutils.TestCase{
"loader/workflow-list.yaml": &remoteWorkflowList{},
"loader/nonexistent-template-list.yaml": &nonExistentTemplateList{},
"loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{},
+ "loader/template-list-not-allowed.yaml": &remoteTemplateListNotAllowed{},
}
type remoteTemplateList struct{}
@@ -45,7 +47,14 @@ func (h *remoteTemplateList) Execute(templateList string) error {
ts := httptest.NewServer(router)
defer ts.Close()
- results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+ configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+ err := ioutil.WriteFile("test-config.yaml", []byte(configFileData), os.ModePerm)
+ if err != nil {
+ return err
+ }
+ defer os.Remove("test-config.yaml")
+
+ results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml")
if err != nil {
return err
}
@@ -53,6 +62,41 @@ func (h *remoteTemplateList) Execute(templateList string) error {
return expectResultsCount(results, 2)
}
+type remoteTemplateListNotAllowed struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
+ router := httprouter.New()
+
+ router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ fmt.Fprintf(w, "This is test matcher text")
+ if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+ fmt.Fprintf(w, "This is test headers matcher text")
+ }
+ })
+
+ router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ file, err := os.ReadFile(templateList)
+ if err != nil {
+ w.WriteHeader(500)
+ }
+ _, err = w.Write(file)
+ if err != nil {
+ w.WriteHeader(500)
+ }
+ })
+ ts := httptest.NewServer(router)
+ defer ts.Close()
+
+ _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+ if err == nil {
+ return fmt.Errorf("expected error for not allowed remote template list url")
+ }
+
+ return nil
+
+}
+
type remoteWorkflowList struct{}
// Execute executes a test case and returns an error if occurred
@@ -79,7 +123,14 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
ts := httptest.NewServer(router)
defer ts.Close()
- results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list")
+ configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+ err := ioutil.WriteFile("test-config.yaml", []byte(configFileData), os.ModePerm)
+ if err != nil {
+ return err
+ }
+ defer os.Remove("test-config.yaml")
+
+ results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml")
if err != nil {
return err
}
diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index 8a63e5a65..d46129579 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -91,6 +91,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", []string{}, "URL containing list of workflows to run"),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+ flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"),
)
createGroup(flagSet, "filters", "Filtering",
diff --git a/v2/go.mod b/v2/go.mod
index 4c4538ec1..e60c388da 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -30,7 +30,7 @@ require (
github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5
- github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a
+ github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
github.com/projectdiscovery/interactsh v0.0.8-0.20220112083504-b0b3b2f359a5
@@ -38,12 +38,12 @@ require (
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660
github.com/projectdiscovery/retryablehttp-go v1.0.2
- github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
+ github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.3.0 // indirect
github.com/segmentio/ksuid v1.0.4
- github.com/shirou/gopsutil/v3 v3.21.12
+ github.com/shirou/gopsutil/v3 v3.22.1
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.4.1
github.com/syndtr/goleveldb v1.0.0
@@ -63,7 +63,7 @@ require (
moul.io/http2curl v1.0.0
)
-require github.com/aws/aws-sdk-go v1.42.37
+require github.com/aws/aws-sdk-go v1.42.45
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
@@ -148,7 +148,7 @@ require (
goftp.io/server/v2 v2.0.0 // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/mod v0.4.2 // indirect
- golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+ golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
diff --git a/v2/go.sum b/v2/go.sum
index 529551016..4be422b3f 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -82,8 +82,8 @@ github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3st
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.42.37 h1:EIziSq3REaoi1LgUBgxoQr29DQS7GYHnBbZPajtJmXM=
-github.com/aws/aws-sdk-go v1.42.37/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
+github.com/aws/aws-sdk-go v1.42.45 h1:rzYlmOX2EqdsYKvo0WBBffuff3BuckL1UB2KyzWhXyQ=
+github.com/aws/aws-sdk-go v1.42.45/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -221,8 +221,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -421,8 +422,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h
github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e h1:RJJuYyuwskYtzZi2gziy6SE/b7saWEzyskaA252E0VY=
github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
-github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk=
-github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
+github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0 h1:KtCp/dCsxXNdT8m0yyWc/4ou4YaKWVakAr3G03TjQCk=
+github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
@@ -460,8 +461,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB
github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
-github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk=
+github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4=
@@ -484,8 +486,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
-github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA=
-github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
+github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
+github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -790,10 +792,10 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
+golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go
index 5650efa3c..b95813033 100644
--- a/v2/internal/runner/options.go
+++ b/v2/internal/runner/options.go
@@ -2,6 +2,8 @@ package runner
import (
"bufio"
+ "io"
+ "log"
"os"
"path/filepath"
"strings"
@@ -152,7 +154,7 @@ func configureOutput(options *types.Options) {
if options.Verbose || options.Validate {
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
}
- if options.Debug {
+ if options.Debug || options.DebugRequests || options.DebugResponse {
gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
}
if options.NoColor {
@@ -161,6 +163,10 @@ func configureOutput(options *types.Options) {
if options.Silent {
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
}
+
+ // disable standard logger (ref: https://github.com/golang/go/issues/19895)
+ log.SetFlags(0)
+ log.SetOutput(io.Discard)
}
// loadResolvers loads resolvers from both user provided flag and file
diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index 4d8c01482..8302ccad9 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -183,6 +183,9 @@ func New(options *types.Options) (*Runner, error) {
opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
opts.NoInteractsh = runner.options.NoInteractsh
opts.StopAtFirstMatch = runner.options.StopAtFirstMatch
+ opts.Debug = runner.options.Debug
+ opts.DebugRequest = runner.options.DebugRequests
+ opts.DebugResponse = runner.options.DebugResponse
interactshClient, err := interactsh.New(opts)
if err != nil {
gologger.Error().Msgf("Could not create interactsh client: %s", err)
diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go
index 7b3ffc6b4..348c2841d 100644
--- a/v2/pkg/catalog/find.go
+++ b/v2/pkg/catalog/find.go
@@ -18,14 +18,21 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
allTemplates := []string{}
for _, t := range definitions {
- paths, err := c.GetTemplatePath(t)
- if err != nil {
- gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
- }
- for _, path := range paths {
- if _, ok := processed[path]; !ok {
- processed[path] = true
- allTemplates = append(allTemplates, path)
+ if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) {
+ if _, ok := processed[t]; !ok {
+ processed[t] = true
+ allTemplates = append(allTemplates, t)
+ }
+ } else {
+ paths, err := c.GetTemplatePath(t)
+ if err != nil {
+ gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+ }
+ for _, path := range paths {
+ if _, ok := processed[path]; !ok {
+ processed[path] = true
+ allTemplates = append(allTemplates, path)
+ }
}
}
}
diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go
index 2276445d8..6323b8326 100644
--- a/v2/pkg/catalog/loader/loader.go
+++ b/v2/pkg/catalog/loader/loader.go
@@ -18,12 +18,13 @@ import (
// Config contains the configuration options for the loader
type Config struct {
- Templates []string
- TemplateURLs []string
- Workflows []string
- WorkflowURLs []string
- ExcludeTemplates []string
- IncludeTemplates []string
+ Templates []string
+ TemplateURLs []string
+ Workflows []string
+ WorkflowURLs []string
+ ExcludeTemplates []string
+ IncludeTemplates []string
+ RemoteTemplateDomainList []string
Tags []string
ExcludeTags []string
@@ -58,25 +59,26 @@ type Store struct {
// NewConfig returns a new loader config
func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
loaderConfig := Config{
- Templates: options.Templates,
- Workflows: options.Workflows,
- TemplateURLs: options.TemplateURLs,
- WorkflowURLs: options.WorkflowURLs,
- ExcludeTemplates: options.ExcludedTemplates,
- Tags: options.Tags,
- ExcludeTags: options.ExcludeTags,
- IncludeTemplates: options.IncludeTemplates,
- Authors: options.Authors,
- Severities: options.Severities,
- ExcludeSeverities: options.ExcludeSeverities,
- IncludeTags: options.IncludeTags,
- IncludeIds: options.IncludeIds,
- ExcludeIds: options.ExcludeIds,
- TemplatesDirectory: options.TemplatesDirectory,
- Protocols: options.Protocols,
- ExcludeProtocols: options.ExcludeProtocols,
- Catalog: catalog,
- ExecutorOptions: executerOpts,
+ Templates: options.Templates,
+ Workflows: options.Workflows,
+ RemoteTemplateDomainList: options.RemoteTemplateDomainList,
+ TemplateURLs: options.TemplateURLs,
+ WorkflowURLs: options.WorkflowURLs,
+ ExcludeTemplates: options.ExcludedTemplates,
+ Tags: options.Tags,
+ ExcludeTags: options.ExcludeTags,
+ IncludeTemplates: options.IncludeTemplates,
+ Authors: options.Authors,
+ Severities: options.Severities,
+ ExcludeSeverities: options.ExcludeSeverities,
+ IncludeTags: options.IncludeTags,
+ IncludeIds: options.IncludeIds,
+ ExcludeIds: options.ExcludeIds,
+ TemplatesDirectory: options.TemplatesDirectory,
+ Protocols: options.Protocols,
+ ExcludeProtocols: options.ExcludeProtocols,
+ Catalog: catalog,
+ ExecutorOptions: executerOpts,
}
return &loaderConfig
}
@@ -108,7 +110,7 @@ func New(config *Config) (*Store, error) {
urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0
if urlBasedTemplatesProvided {
- remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs)
+ remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs, config.RemoteTemplateDomainList)
if err != nil {
return store, err
}
diff --git a/v2/pkg/catalog/loader/loader_test.go b/v2/pkg/catalog/loader/loader_test.go
index b9f6ab43e..ce8c77822 100644
--- a/v2/pkg/catalog/loader/loader_test.go
+++ b/v2/pkg/catalog/loader/loader_test.go
@@ -1,6 +1,7 @@
package loader
import (
+ "reflect"
"testing"
"github.com/stretchr/testify/require"
@@ -38,3 +39,55 @@ func TestLoadTemplates(t *testing.T) {
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
}
+
+func TestRemoteTemplates(t *testing.T) {
+ var nilStringSlice []string
+ type args struct {
+ config *Config
+ }
+ tests := []struct {
+ name string
+ args args
+ want *Store
+ wantErr bool
+ }{
+ {
+ name: "remote-templates-positive",
+ args: args{
+ config: &Config{
+ TemplateURLs: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
+ RemoteTemplateDomainList: []string{"localhost","raw.githubusercontent.com"},
+ },
+ },
+ want: &Store{
+ finalTemplates: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
+ },
+ wantErr: false,
+ },
+ {
+ name: "remote-templates-negative",
+ args: args{
+ config: &Config{
+ TemplateURLs: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
+ RemoteTemplateDomainList: []string{"localhost"},
+ },
+ },
+ want: &Store{
+ finalTemplates: nilStringSlice,
+ },
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := New(tt.args.config)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !reflect.DeepEqual(got.finalTemplates, tt.want.finalTemplates) {
+ t.Errorf("New() = %v, want %v", got.finalTemplates, tt.want.finalTemplates)
+ }
+ })
+ }
+}
diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go
index c787e9601..ac2164b33 100644
--- a/v2/pkg/catalog/loader/remote_loader.go
+++ b/v2/pkg/catalog/loader/remote_loader.go
@@ -4,9 +4,11 @@ import (
"bufio"
"fmt"
"net/http"
+ "net/url"
"strings"
"github.com/pkg/errors"
+ "github.com/projectdiscovery/nuclei/v2/pkg/utils"
)
type ContentType string
@@ -16,38 +18,38 @@ const (
Workflow ContentType = "Workflow"
)
-type RemoteContentError struct {
+type RemoteContent struct {
Content []string
Type ContentType
Error error
}
-func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) {
- remoteContentErrorChannel := make(chan RemoteContentError)
+func getRemoteTemplatesAndWorkflows(templateURLs, workflowURLs, remoteTemplateDomainList []string) ([]string, []string, error) {
+ remoteContentChannel := make(chan RemoteContent)
for _, templateURL := range templateURLs {
- go getRemoteContent(templateURL, remoteContentErrorChannel, Template)
+ go getRemoteContent(templateURL, remoteTemplateDomainList, remoteContentChannel, Template)
}
for _, workflowURL := range workflowURLs {
- go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow)
+ go getRemoteContent(workflowURL, remoteTemplateDomainList, remoteContentChannel, Workflow)
}
var remoteTemplateList []string
var remoteWorkFlowList []string
var err error
for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ {
- remoteContentError := <-remoteContentErrorChannel
- if remoteContentError.Error != nil {
+ remoteContent := <-remoteContentChannel
+ if remoteContent.Error != nil {
if err != nil {
- err = errors.New(remoteContentError.Error.Error() + ": " + err.Error())
+ err = errors.New(remoteContent.Error.Error() + ": " + err.Error())
} else {
- err = remoteContentError.Error
+ err = remoteContent.Error
}
} else {
- if remoteContentError.Type == Template {
- remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...)
- } else if remoteContentError.Type == Workflow {
- remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...)
+ if remoteContent.Type == Template {
+ remoteTemplateList = append(remoteTemplateList, remoteContent.Content...)
+ } else if remoteContent.Type == Workflow {
+ remoteWorkFlowList = append(remoteWorkFlowList, remoteContent.Content...)
}
}
}
@@ -55,17 +57,30 @@ func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string
return remoteTemplateList, remoteWorkFlowList, err
}
-func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) {
+func getRemoteContent(URL string, remoteTemplateDomainList []string, remoteContentChannel chan<- RemoteContent, contentType ContentType) {
+ if err := validateRemoteRemplateURL(URL, remoteTemplateDomainList); err != nil {
+ remoteContentChannel <- RemoteContent{
+ Error: err,
+ }
+ return
+ }
+ if strings.HasPrefix(URL, "http") && (strings.HasSuffix(URL, ".yaml") || strings.HasSuffix(URL, ".yml")) {
+ remoteContentChannel <- RemoteContent{
+ Content: []string{URL},
+ Type: contentType,
+ }
+ return
+ }
response, err := http.Get(URL)
if err != nil {
- w <- RemoteContentError{
+ remoteContentChannel <- RemoteContent{
Error: err,
}
return
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode > 299 {
- w <- RemoteContentError{
+ remoteContentChannel <- RemoteContent{
Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode),
}
return
@@ -78,18 +93,37 @@ func getRemoteContent(URL string, w chan<- RemoteContentError, contentType Conte
if text == "" {
continue
}
+ if utils.IsURL(text) {
+ if err := validateRemoteRemplateURL(text, remoteTemplateDomainList); err != nil {
+ remoteContentChannel <- RemoteContent{
+ Error: err,
+ }
+ return
+ }
+ }
templateList = append(templateList, text)
}
if err := scanner.Err(); err != nil {
- w <- RemoteContentError{
+ remoteContentChannel <- RemoteContent{
Error: errors.Wrap(err, "get \"%s\""),
}
return
}
- w <- RemoteContentError{
+ remoteContentChannel <- RemoteContent{
Content: templateList,
Type: contentType,
}
}
+
+func validateRemoteRemplateURL(inputURL string, remoteTemplateDomainList []string) error {
+ parsedURL, err := url.Parse(inputURL)
+ if err != nil {
+ return err
+ }
+ if !utils.StringSliceContains(remoteTemplateDomainList, parsedURL.Host) {
+ return errors.Errorf("Remote template URL host (%s) is not present in the `remote-template-domain` list in nuclei config", parsedURL.Host)
+ }
+ return nil
+}
diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go
index f8fa75925..39bc405e3 100644
--- a/v2/pkg/parsers/parser.go
+++ b/v2/pkg/parsers/parser.go
@@ -2,8 +2,6 @@ package parsers
import (
"fmt"
- "io/ioutil"
- "os"
"regexp"
"strings"
@@ -129,14 +127,7 @@ func ParseTemplate(templatePath string) (*templates.Template, error) {
if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
return value.(*templates.Template), err
}
-
- f, err := os.Open(templatePath)
- if err != nil {
- return nil, err
- }
- defer f.Close()
-
- data, err := ioutil.ReadAll(f)
+ data, err := utils.ReadFromPathOrURL(templatePath)
if err != nil {
return nil, err
}
diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go
index 742acc916..0eed8976c 100644
--- a/v2/pkg/protocols/common/expressions/expressions.go
+++ b/v2/pkg/protocols/common/expressions/expressions.go
@@ -6,9 +6,9 @@ import (
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
- "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+ "github.com/projectdiscovery/stringsutil"
)
// Evaluate checks if the match contains a dynamic variable, for each
@@ -33,13 +33,20 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
}
func evaluate(data string, base map[string]interface{}) (string, error) {
+ // replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value
data = replacer.Replace(data, base)
+ // expressions can be:
+ // - simple: containing base values keys (variables)
+ // - complex: containing helper functions [ + variables]
+ // literals like {{2+2}} are not considered expressions
+ expressions := findExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base)))
dynamicValues := make(map[string]interface{})
- for _, match := range findMatches(data) {
- expr := generators.TrimDelimiters(match)
-
- compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
+ for _, expression := range expressions {
+ // replace variable placeholders with base values
+ expression = replacer.Replace(expression, base)
+ // turns expressions (either helper functions+base values or base values)
+ compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions())
if err != nil {
continue
}
@@ -47,19 +54,104 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
if err != nil {
continue
}
- dynamicValues[expr] = result
+ dynamicValues[expression] = result
}
- // Replacer dynamic values if any in raw request and parse it
+ // Replacer dynamic values if any in raw request and parse it
return replacer.Replace(data, dynamicValues), nil
}
-func findMatches(data string) []string {
- var matches []string
- for _, token := range strings.Split(data, marker.ParenthesisOpen) {
- closingToken := strings.LastIndex(token, marker.ParenthesisClose)
- if closingToken > 0 {
- matches = append(matches, token[:closingToken])
+// maxIterations to avoid infinite loop
+const maxIterations = 250
+
+func findExpressions(data, OpenMarker, CloseMarker string, functions map[string]govaluate.ExpressionFunction) []string {
+ var (
+ iterations int
+ exps []string
+ )
+ for {
+ // check if we reached the maximum number of iterations
+ if iterations > maxIterations {
+ break
+ }
+ iterations++
+ // attempt to find open markers
+ indexOpenMarker := strings.Index(data, OpenMarker)
+ // exits if not found
+ if indexOpenMarker < 0 {
+ break
+ }
+
+ indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)
+
+ shouldSearchCloseMarker := true
+ closeMarkerFound := false
+ innerData := data
+ var potentialMatch string
+ var indexCloseMarker, indexCloseMarkerOffset int
+ skip := indexOpenMarkerOffset
+ for shouldSearchCloseMarker {
+ // attempt to find close marker
+ indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)
+ // if no close markers are found exit
+ if indexCloseMarker < 0 {
+ shouldSearchCloseMarker = false
+ continue
+ }
+ indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)
+
+ potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
+ if isExpression(potentialMatch, functions) {
+ closeMarkerFound = true
+ shouldSearchCloseMarker = false
+ exps = append(exps, potentialMatch)
+ } else {
+ skip = indexCloseMarkerOffset
+ }
+ }
+
+ if closeMarkerFound {
+ // move after the close marker
+ data = data[indexCloseMarkerOffset:]
+ } else {
+ // move after the open marker
+ data = data[indexOpenMarkerOffset:]
}
}
- return matches
+ return exps
+}
+
+func isExpression(data string, functions map[string]govaluate.ExpressionFunction) bool {
+ if _, err := govaluate.NewEvaluableExpression(data); err == nil {
+ return stringsutil.ContainsAny(data, getFunctionsNames(functions)...)
+ }
+
+ // check if it's a complex expression
+ _, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions())
+ return err == nil
+}
+
+func mapToFunctions(vars map[string]interface{}) map[string]govaluate.ExpressionFunction {
+ f := make(map[string]govaluate.ExpressionFunction)
+ for k := range vars {
+ f[k] = nil
+ }
+ return f
+}
+
+func mergeFunctions(m ...map[string]govaluate.ExpressionFunction) map[string]govaluate.ExpressionFunction {
+ o := make(map[string]govaluate.ExpressionFunction)
+ for _, mm := range m {
+ for k, v := range mm {
+ o[k] = v
+ }
+ }
+ return o
+}
+
+func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string {
+ var keys []string
+ for k := range m {
+ keys = append(keys, k)
+ }
+ return keys
}
diff --git a/v2/pkg/protocols/common/expressions/expressions_test.go b/v2/pkg/protocols/common/expressions/expressions_test.go
index 668b3b455..a154eba77 100644
--- a/v2/pkg/protocols/common/expressions/expressions_test.go
+++ b/v2/pkg/protocols/common/expressions/expressions_test.go
@@ -14,19 +14,19 @@ func TestEvaluate(t *testing.T) {
}{
{input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}},
{input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}},
- // TODO #1501
- //{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
- //{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
- //{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
- //{input: `{{"{{" + '123*123' + "}}"}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+ {input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
+ {input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+ {input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+ {input: `{{"{{" + '123*123' + "}}"}}`, expected: `{{"{{" + '123*123' + "}}"}}`, extra: map[string]interface{}{}},
+ {input: `{{a + '123*123' + b}}`, expected: `aa123*123bb`, extra: map[string]interface{}{"a": "aa", "b": "bb"}},
{input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}},
- {input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}},
- {input: `{{"1"+"1"}}`, expected: "11", extra: map[string]interface{}{}},
- {input: `{{"1" + '*' + "1"}}`, expected: "1*1", extra: map[string]interface{}{}},
- {input: `{{"a" + 'b' + "c"}}`, expected: "abc", extra: map[string]interface{}{}},
- {input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}},
- {input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}},
- {input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}},
+ {input: `{{1+1}}`, expected: "{{1+1}}", extra: map[string]interface{}{}},
+ {input: `{{"1"+"1"}}`, expected: `{{"1"+"1"}}`, extra: map[string]interface{}{}},
+ {input: `{{"1" + '*' + "1"}}`, expected: `{{"1" + '*' + "1"}}`, extra: map[string]interface{}{}},
+ {input: `{{"a" + 'b' + "c"}}`, expected: `{{"a" + 'b' + "c"}}`, extra: map[string]interface{}{}},
+ {input: `{{10*2}}`, expected: `{{10*2}}`, extra: map[string]interface{}{}},
+ {input: `{{10/2}}`, expected: `{{10/2}}`, extra: map[string]interface{}{}},
+ {input: `{{10-2}}`, expected: `{{10-2}}`, extra: map[string]interface{}{}},
{input: "test", expected: "test", extra: map[string]interface{}{}},
{input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}},
{input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}},
diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go
index a61ec4cc1..fe3f7f218 100644
--- a/v2/pkg/protocols/common/interactsh/interactsh.go
+++ b/v2/pkg/protocols/common/interactsh/interactsh.go
@@ -75,7 +75,9 @@ type Options struct {
// Progress is the nuclei progress bar implementation.
Progress progress.Progress
// Debug specifies whether debugging output should be shown for interactsh-client
- Debug bool
+ Debug bool
+ DebugRequest bool
+ DebugResponse bool
// DisableHttpFallback controls http retry in case of https failure for server url
DisableHttpFallback bool
// NoInteractsh disables the engine
@@ -146,8 +148,8 @@ func (c *Client) firstTimeInitializeClient() error {
c.hostname = interactDomain
interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
- if c.options.Debug {
- debugPrintInteraction(interaction)
+ if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse {
+ c.debugPrintInteraction(interaction)
}
item := c.requests.Get(interaction.UniqueID)
@@ -343,26 +345,53 @@ func HasMatchers(op *operators.Operators) bool {
return false
}
-func debugPrintInteraction(interaction *server.Interaction) {
+// HasMarkers checks if the text contains interactsh markers
+func HasMarkers(data string) bool {
+ return strings.Contains(data, interactshURLMarker)
+}
+
+func (c *Client) debugPrintInteraction(interaction *server.Interaction) {
builder := &bytes.Buffer{}
switch interaction.Protocol {
case "dns":
- builder.WriteString(fmt.Sprintf("[%s] Received DNS interaction (%s) from %s at %s", interaction.FullId, interaction.QType, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
- builder.WriteString(fmt.Sprintf("\n-----------\nDNS Request\n-----------\n\n%s\n\n------------\nDNS Response\n------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
+ builder.WriteString(formatInteractionHeader("DNS", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+ if c.options.DebugRequest || c.options.Debug {
+ builder.WriteString(formatInteractionMessage("DNS Request", interaction.RawRequest))
+ }
+ if c.options.DebugResponse || c.options.Debug {
+ builder.WriteString(formatInteractionMessage("DNS Response", interaction.RawResponse))
+ }
case "http":
- builder.WriteString(fmt.Sprintf("[%s] Received HTTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
- builder.WriteString(fmt.Sprintf("\n------------\nHTTP Request\n------------\n\n%s\n\n-------------\nHTTP Response\n-------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
+ builder.WriteString(formatInteractionHeader("HTTP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+ if c.options.DebugRequest || c.options.Debug {
+ builder.WriteString(formatInteractionMessage("HTTP Request", interaction.RawRequest))
+ }
+ if c.options.DebugResponse || c.options.Debug {
+ builder.WriteString(formatInteractionMessage("HTTP Response", interaction.RawResponse))
+ }
case "smtp":
- builder.WriteString(fmt.Sprintf("[%s] Received SMTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
- builder.WriteString(fmt.Sprintf("\n------------\nSMTP Interaction\n------------\n\n%s\n\n", interaction.RawRequest))
+ builder.WriteString(formatInteractionHeader("SMTP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+ if c.options.DebugRequest || c.options.Debug || c.options.DebugResponse {
+ builder.WriteString(formatInteractionMessage("SMTP Interaction", interaction.RawRequest))
+ }
case "ldap":
- builder.WriteString(fmt.Sprintf("[%s] Received LDAP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
- builder.WriteString(fmt.Sprintf("\n------------\nLDAP Interaction\n------------\n\n%s\n\n", interaction.RawRequest))
+ builder.WriteString(formatInteractionHeader("LDAP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+ if c.options.DebugRequest || c.options.Debug || c.options.DebugResponse {
+ builder.WriteString(formatInteractionMessage("LDAP Interaction", interaction.RawRequest))
+ }
}
fmt.Fprint(os.Stderr, builder.String())
}
+func formatInteractionHeader(protocol, ID, address string, at time.Time) string {
+ return fmt.Sprintf("[%s] Received %s interaction from %s at %s", ID, protocol, address, at.Format("2006-01-02 15:04:05"))
+}
+
+func formatInteractionMessage(key, value string) string {
+ return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value)
+}
+
func hash(templateID, host string) string {
h := sha1.New()
h.Write([]byte(templateID))
diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go
index 31adee9e7..279e6eea6 100644
--- a/v2/pkg/protocols/dns/dns.go
+++ b/v2/pkg/protocols/dns/dns.go
@@ -234,6 +234,8 @@ func questionTypeToInt(questionType string) uint16 {
question = dns.TypeDS
case "AAAA":
question = dns.TypeAAAA
+ case "CAA":
+ question = dns.TypeCAA
}
return question
}
diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go
index dc0a22d2c..34c8b7e16 100644
--- a/v2/pkg/protocols/dns/dns_types.go
+++ b/v2/pkg/protocols/dns/dns_types.go
@@ -31,6 +31,8 @@ const (
TXT
// name:AAAA
AAAA
+ // name:CAA
+ CAA
limit
)
@@ -45,6 +47,7 @@ var DNSRequestTypeMapping = map[DNSRequestType]string{
MX: "MX",
TXT: "TXT",
AAAA: "AAAA",
+ CAA: "CAA",
}
// GetSupportedDNSRequestTypes returns list of supported types
diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go
index 1b6516b85..8bc7d3537 100644
--- a/v2/pkg/protocols/dns/request.go
+++ b/v2/pkg/protocols/dns/request.go
@@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+ "github.com/projectdiscovery/nuclei/v2/pkg/utils"
"github.com/projectdiscovery/retryabledns"
)
@@ -27,7 +28,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Parse the URL and return domain if URL.
var domain string
- if isURL(input) {
+ if utils.IsURL(input) {
domain = extractDomain(input)
} else {
domain = input
@@ -125,18 +126,6 @@ func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols
}
}
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
- if _, err := url.ParseRequestURI(toTest); err != nil {
- return false
- }
- u, err := url.Parse(toTest)
- if err != nil || u.Scheme == "" || u.Host == "" {
- return false
- }
- return true
-}
-
// extractDomain extracts the domain name of a URL
func extractDomain(theURL string) string {
u, err := url.Parse(theURL)
diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go
index fe6831e41..10dc7ef8e 100644
--- a/v2/pkg/protocols/headless/engine/rules.go
+++ b/v2/pkg/protocols/headless/engine/rules.go
@@ -64,9 +64,7 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
var rawResp strings.Builder
respPayloads := ctx.Response.Payload()
if respPayloads != nil {
- rawResp.WriteString("HTTP/1.1 ")
- rawResp.WriteString(fmt.Sprint(respPayloads.ResponseCode))
- rawResp.WriteString(" " + respPayloads.ResponsePhrase + "\n")
+ rawResp.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\n", respPayloads.ResponseCode, respPayloads.ResponsePhrase))
for _, header := range respPayloads.ResponseHeaders {
rawResp.WriteString(header.Name + ": " + header.Value + "\n")
}
diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go
index c4dccaf9f..29f8ca692 100644
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@@ -239,8 +239,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
for {
// returns two values, error and skip, which skips the execution for the request instance.
executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
- hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
-
+ hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
generatedHttpRequest, err := generator.Make(reqURL, data, payloads, dynamicValue)
if err != nil {
if err == io.EOF {
@@ -249,6 +248,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return true, err
}
+ hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
if reqURL == "" {
reqURL = generatedHttpRequest.URL()
}
@@ -256,16 +256,16 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) {
return true, nil
}
- var gotOutput bool
+ var gotMatches bool
request.options.RateLimiter.Take()
- err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) {
+ err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
- gotOutput = true
+ gotMatches = event.OperatorsResult.Matched
gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues)
}
- if hasInteractMarkers && request.options.Interactsh != nil {
+ if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
@@ -292,7 +292,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
request.options.Progress.IncrementRequests()
// If this was a match, and we want to stop at first match, skip all further requests.
- if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput {
+ if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches {
return true, nil
}
return false, nil
@@ -329,7 +329,7 @@ const drainReqSize = int64(8 * 1024)
var errStopExecution = errors.New("stop execution due to unresolved variables")
// executeRequest executes the actual generated request and returns error if occurred
-func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
+func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error {
request.setCustomHeaders(generatedRequest)
var (
@@ -434,7 +434,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
// If we have interactsh markers and request times out, still send
// a callback event so in case we receive an interaction, correlation is possible.
- if hasInteractMarkers {
+ if hasInteractMatchers {
outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i]
@@ -558,7 +558,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})
- if hasInteractMarkers {
+ if hasInteractMatchers {
event.UsesInteractsh = true
}
diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go
index 6ed64dc6d..24e334edd 100644
--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@@ -2,8 +2,6 @@ package templates
import (
"fmt"
- "io/ioutil"
- "os"
"reflect"
"strings"
@@ -38,13 +36,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
template := &Template{}
- f, err := os.Open(filePath)
- if err != nil {
- return nil, err
- }
- defer f.Close()
-
- data, err := ioutil.ReadAll(f)
+ data, err := utils.ReadFromPathOrURL(filePath)
if err != nil {
return nil, err
}
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go
index 024408d52..8cffb6778 100644
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@@ -1110,6 +1110,7 @@ func init() {
"MX",
"TXT",
"AAAA",
+ "CAA",
}
FILERequestDoc.Type = "file.Request"
diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
index fcccd8822..2be9af680 100644
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@@ -23,6 +23,8 @@ type Options struct {
Templates goflags.StringSlice
// TemplateURLs specifies URLs to a list of templates to use
TemplateURLs goflags.StringSlice
+ // RemoteTemplates specifies list of allowed URLs to load remote templates from
+ RemoteTemplateDomainList goflags.StringSlice
// ExcludedTemplates specifies the template/templates to exclude
ExcludedTemplates goflags.StringSlice
// CustomHeaders is the list of custom global headers to send with each request.
diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go
index ba4bf87cb..e7a1413f5 100644
--- a/v2/pkg/utils/utils.go
+++ b/v2/pkg/utils/utils.go
@@ -2,6 +2,10 @@ package utils
import (
"errors"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "os"
"strings"
"github.com/projectdiscovery/fileutil"
@@ -37,3 +41,54 @@ func LoadFile(filename string) ([]string, error) {
}
return items, nil
}
+
+// IsURL tests a string to determine if it is a well-structured url or not.
+func IsURL(input string) bool {
+ _, err := url.ParseRequestURI(input)
+ if err != nil {
+ return false
+ }
+
+ u, err := url.Parse(input)
+ if err != nil || u.Scheme == "" || u.Host == "" {
+ return false
+ }
+
+ return true
+}
+
+// ReadFromPathOrURL reads and returns the contents of a file or url.
+func ReadFromPathOrURL(templatePath string) (data []byte, err error) {
+ if IsURL(templatePath) {
+ resp, err := http.Get(templatePath)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ data, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ f, err := os.Open(templatePath)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ data, err = ioutil.ReadAll(f)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return
+}
+
+// StringSliceContains checks if a string slice contains a string.
+func StringSliceContains(slice []string, item string) bool {
+ for _, i := range slice {
+ if strings.EqualFold(i, item) {
+ return true
+ }
+ }
+ return false
+}