External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:25:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

add flow support in template (i.e javascript scripting) (#4015)

Browse Source 
* add flow logic

* progress

* working POC

* fix string slice normalization issue in variables

* update

* fix nil panic

* remove poll()

* load file with sandbox and more

* fix failing integration tests

* JS: log: print in vardump format

* fix missing id in protocols

* fix proto prefix in template context

* flow: add unit tests

* conditional flow support using flow

* fix proto callbacks + more unit tests

* adds integration test

* conditional flow: check if req has any matchers

* fix lint error

* deprecate iterate-all+ missing multi-proto implementation

* fix ip input in raw request

* JS: feat dedupe object+ more builtin funcs

* feat: hide protocol result using hide

* feat: async execution

* complete async execution support

* fix condition-flow without any matchers

* refactor: template executer package (tmplexec)

* flow executor working

* fix data race in templateCtx

* templateCtx redesign

* fix failing unit test

* add multiprotocol support to deprecated syntax

* fix race condition in utils & tlsx

* add documentation in flow package

* remove regions.txt file

* fix minor issue with self contained templates

* fix typos of copilot

* dep + misc update

* fix reqID: use req.Type instead of template.Type

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
This commit is contained in:
Tarun Koyalwar 2023-08-31 18:03:01 +05:30 committed by GitHub
parent 8125b6805c
commit f7fe99f806
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
78 changed files with 2577 additions and 640 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
integration_tests/flow/conditional-flow-negative.yaml Normal file
View File

@ -0,0 +1,27 @@
id: ghost-blog-detection
info:
name: Ghost blog detection
author: pdteam
severity: info
flow: dns() && http()
dns:
- name: "{{FQDN}}"
type: CNAME
matchers:
- type: word
words:
- "ghost.io"
http:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "ghost.io"

27
integration_tests/flow/conditional-flow.yaml Normal file
View File

@ -0,0 +1,27 @@
id: ghost-blog-detection
info:
name: Ghost blog detection
author: pdteam
severity: info
flow: dns() && http()
dns:
- name: "{{FQDN}}"
type: CNAME
matchers:
- type: word
words:
- "ghost.io"
http:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "ghost.io"

42
integration_tests/flow/dns-ns-probe.yaml Normal file
View File

@ -0,0 +1,42 @@
id: dns-ns-probe
info:
name: Nuclei flow dns ns probe
author: pdteam
severity: info
description: Description of the Template
reference: https://example-reference-link
flow: |
dns("fetch-ns");
for(let ns of template["nameservers"]) {
set("nameserver",ns);
dns("probe-ns");
};
dns:
- id: "fetch-ns"
name: "{{FQDN}}"
type: NS
matchers:
- type: word
words:
- "IN\tNS"
extractors:
- type: regex
internal: true
name: "nameservers"
group: 1
regex:
- "IN\tNS\t(.+)"
- id: "probe-ns"
name: "{{nameserver}}"
type: A
class: inet
retries: 3
recursion: true
extractors:
- type: dsl
dsl:
- "a"

35
integration_tests/flow/iterate-values-flow.yaml Normal file
View File

@ -0,0 +1,35 @@
id: extract-emails
info:
name: Extract Email IDs from Response
author: pdteam
severity: info
flow: |
http(0)
for(let email of template["emails"]) {
set("email",email);
http(1);
}
http:
- method: GET
path:
- "{{BaseURL}}"
extractors:
- type: regex
name: emails
internal: true
regex:
- '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
- method: GET
path:
- "{{BaseURL}}/user/{{base64(email)}}"
matchers:
- type: word
words:
- "Welcome"

2
integration_tests/protocols/code/py-env-var.yaml
View File

@ -20,4 +20,4 @@ code:
- type: word
words:
- "hello from input baz"
# digest: 4b0a00483046022100cd2b9d34169cdb716caee25976fed763880435f2f1e2979c9d7c9d2bd7b8e409022100dd0ba8bd3fa6a6be5f964ca3b0ce8bdbb20d865553133cf494ef64fbeebff345
# digest: 4a0a00473045022100e17c7a809fd64419baf401b5331edab3a68a4c182f7777614beb1862eb6ea8b7022011b95fc0e22d7f82e08e01b56ce87afdbe03027c238ba290a058d695226173ae

2
integration_tests/protocols/code/py-file.yaml
View File

@ -18,4 +18,4 @@ code:
- type: word
words:
- "hello from input"
# digest: 4b0a00483046022100f663e5afaf5c118b21b9c5918cba12d7cc83edc2a3ee0f338c07e3cd1fe40e20022100b46193e3275c490a4ad3897c6e2ca51ce09f408538b17d041e0063d40f4df833
# digest: 490a004630440220241d7faae14ab5760dbe7acf621a3923d0650148bc14a52a9be06ba76e8e0abf02201885fcc432d354d3c99ea97b964838719451bc97f148229f187f61eee7525eb6

2
integration_tests/protocols/code/py-interactsh.yaml
View File

@ -26,4 +26,4 @@ code:
part: interactsh_protocol
words:
- "http"
# digest: 4b0a00483046022100c45cd27b9d49879663e1ea3c877dc362d06b8a0aea64b1ab06be3af5aa9a32ee0221008f5ee347245a2c1e04c46528e4c70a5a851f95c6ba49d2834ef7c3784bca47a9
# digest: 490a004630440220427cb7100f0b7d95224f490a6f4f626186782cb26c69f2551d6aefcdbc7c17d20220206161ad3a98afe8fcef9dd06d9a6dd5f34c5f7e3cd3ab7f81328f033dcd2b48

2
integration_tests/protocols/code/py-snippet.yaml
View File

@ -20,4 +20,4 @@ code:
- type: word
words:
- "hello from input"
# digest: 4a0a00473045022100df57bf446d6d8e73ff9424b1055faebcea9038e5d5934834ed8e619b77bdfd5e02201754c1cebe9f65883315b3830755a0689999f33db7102cd8d5469e4c01cc6a66
# digest: 4a0a00473045022056092462597e85139626656d37df123094cb3861bdf583450c38814bac8df9cb022100e83a8c552f8f8a098f6b7ec8a32c6b448b995e000884beadb50cb0f2720117de

2
integration_tests/protocols/http/matcher-status.yaml
View File

@ -1,4 +1,4 @@
id: matchet-status
id: matcher-status
info:
name: Test Matcher Status

84
v2/cmd/integration-test/flow.go Normal file
View File

@ -0,0 +1,84 @@
package main
import (
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var flowTestcases = []TestCaseInfo{
{Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}},
{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}},
{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}},
{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}},
}
type conditionalFlow struct{}
func (t *conditionalFlow) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug)
if err != nil {
return err
}
return expectResultsCount(results, 2)
}
type conditionalFlowNegative struct{}
func (t *conditionalFlowNegative) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}
type iterateValuesFlow struct{}
func (t *iterateValuesFlow) Execute(filePath string) error {
router := httprouter.New()
testemails := []string{
"secrets@scanme.sh",
"superadmin@scanme.sh",
}
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(fmt.Sprint(testemails)))
})
router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
})
router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 2)
}
type dnsNsProbe struct{}
func (t *dnsNsProbe) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug)
if err != nil {
return err
}
return expectResultsCount(results, 3)
}
func getBase64(input string) string {
return base64.StdEncoding.EncodeToString([]byte(input))
}

1
v2/cmd/integration-test/integration-test.go
View File

@ -49,6 +49,7 @@ var (
"multi": multiProtoTestcases,
"generic": genericTestcases,
"dsl": dslTestcases,
"flow": flowTestcases,
}
// For debug purposes

4
v2/cmd/integration-test/multi.go
View File

@ -1,6 +1,8 @@
package main
import "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
import (
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var multiProtoTestcases = []TestCaseInfo{
{Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}},

2
v2/cmd/integration-test/template-path.go
View File

@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{
//template folder path issue
{Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}},
//cwd
{Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
{Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
//relative path
{Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}},
//absolute path

39
v2/go.mod
View File

@ -21,17 +21,17 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.19
github.com/projectdiscovery/fastdialer v0.0.35
github.com/projectdiscovery/hmap v0.0.13
github.com/projectdiscovery/fastdialer v0.0.36
github.com/projectdiscovery/hmap v0.0.15
github.com/projectdiscovery/interactsh v1.1.4
github.com/projectdiscovery/rawhttp v0.1.18
github.com/projectdiscovery/retryabledns v1.0.32
github.com/projectdiscovery/retryablehttp-go v1.0.20
github.com/projectdiscovery/retryabledns v1.0.35
github.com/projectdiscovery/retryablehttp-go v1.0.24
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
github.com/segmentio/ksuid v1.0.4
github.com/shirou/gopsutil/v3 v3.23.6 // indirect
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cast v1.5.1
github.com/syndtr/goleveldb v1.0.0
@ -39,9 +39,9 @@ require (
github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db
github.com/xanzy/go-gitlab v0.84.0
go.uber.org/multierr v1.11.0
golang.org/x/net v0.12.0
golang.org/x/oauth2 v0.10.0
golang.org/x/text v0.11.0
golang.org/x/net v0.14.0
golang.org/x/oauth2 v0.11.0
golang.org/x/text v0.12.0
gopkg.in/yaml.v2 v2.4.0
moul.io/http2curl v1.0.0
)
@ -60,6 +60,7 @@ require (
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72
github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
github.com/docker/go-units v0.5.0
github.com/dop251/goja v0.0.0-20230812105242-81d76064690d
github.com/fatih/structs v1.1.0
github.com/go-git/go-git/v5 v5.7.0
github.com/h2non/filetype v1.1.3
@ -68,7 +69,7 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/projectdiscovery/dsl v0.0.16
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/goflags v0.1.12
github.com/projectdiscovery/goflags v0.1.18
github.com/projectdiscovery/gologger v1.1.11
github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6
github.com/projectdiscovery/httpx v1.3.4
@ -76,9 +77,9 @@ require (
github.com/projectdiscovery/ratelimit v0.0.9
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.1
github.com/projectdiscovery/tlsx v1.1.4
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1
github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40
github.com/projectdiscovery/utils v0.0.51
github.com/projectdiscovery/wappalyzergo v0.0.107
github.com/stretchr/testify v1.8.4
gopkg.in/src-d/go-git.v4 v4.13.1
@ -87,7 +88,7 @@ require (
require (
aead.dev/minisign v0.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect
@ -105,14 +106,16 @@ require (
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
github.com/cloudflare/cfssl v1.6.4 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/dlclark/regexp2 v1.8.1 // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gaukas/godicttls v0.0.3 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/google/certificate-transparency-go v1.1.4 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kataras/jwt v0.1.8 // indirect
@ -128,11 +131,13 @@ require (
github.com/projectdiscovery/asnmap v1.0.4 // indirect
github.com/projectdiscovery/cdncheck v1.0.9 // indirect
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/refraction-networking/utls v1.3.2 // indirect
github.com/quic-go/quic-go v0.37.0 // indirect
github.com/refraction-networking/utls v1.4.2 // indirect
github.com/sashabaranov/go-openai v1.14.1 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/skeema/knownhosts v1.1.1 // indirect
github.com/smartystreets/assertions v1.0.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/tidwall/btree v1.6.0 // indirect
github.com/tidwall/buntdb v1.3.0 // indirect
github.com/tidwall/gjson v1.14.4 // indirect
@ -216,10 +221,10 @@ require (
go.etcd.io/bbolt v1.3.7 // indirect
go.uber.org/zap v1.24.0 // indirect
goftp.io/server/v2 v2.0.0 // indirect
golang.org/x/crypto v0.11.0
golang.org/x/crypto v0.12.0
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
golang.org/x/mod v0.12.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.11.0 // indirect
google.golang.org/appengine v1.6.7 // indirect

101
v2/go.sum
View File

@ -3,8 +3,8 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
@ -128,6 +128,9 @@ github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM2
github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo=
github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8=
github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
@ -145,11 +148,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0=
github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja v0.0.0-20230812105242-81d76064690d h1:9aaGwVf4q+kknu+mROAXUApJ1DoOwhE8dGj/XLBYzWg=
github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
@ -179,6 +189,7 @@ github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw4
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
@ -190,6 +201,9 @@ github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+j
github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y=
github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
@ -234,6 +248,9 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ=
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@ -250,8 +267,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU=
github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A=
@ -259,6 +276,7 @@ github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDj
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU=
@ -295,6 +313,7 @@ github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
@ -371,8 +390,9 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
@ -398,20 +418,20 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z
github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c=
github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4=
github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU=
github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g=
github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo=
github.com/projectdiscovery/fastdialer v0.0.36 h1:Ac/CRLryJB2mA8erDwAHoCJGFvjCDIPUznxWl9kJPW8=
github.com/projectdiscovery/fastdialer v0.0.36/go.mod h1:jxX9iQJdTwlD6u0Q9Dj9/AmatHPW2GRl3V6XTAvKtHY=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0=
github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws=
github.com/projectdiscovery/goflags v0.1.18 h1:L4nwDBNJcZhbmhI3GhQ1GJwz7xVWFL3BumJ+TIDBi5E=
github.com/projectdiscovery/goflags v0.1.18/go.mod h1:cZut0Q98yksNVtM73RPSm22N/eDkAMFT9t6mwu6S5pY=
github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8=
github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY=
github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA=
github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6/go.mod h1:jCpXNvLUCPMzm5AhJv8wtnUt/7rz0TY2SsqvKQ8tn2E=
github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI=
github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg=
github.com/projectdiscovery/hmap v0.0.15 h1:iTRXV94bNIuR5obYBxOhvs3yEYXdNdJJnrXnxv4uLHc=
github.com/projectdiscovery/hmap v0.0.15/go.mod h1:oybodscUwBbL4GnhBPPTemazPXyMErqL+dE+0ZtJ6lg=
github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg=
github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY=
github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI=
@ -426,31 +446,34 @@ github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt3
github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo=
github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c=
github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE=
github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI=
github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU=
github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c=
github.com/projectdiscovery/retryablehttp-go v1.0.24 h1:1In7vIUnNvEdHhnA5KmUVf+D+tVZgITaJUZxFYgKCdo=
github.com/projectdiscovery/retryablehttp-go v1.0.24/go.mod h1:S2KiViUrjvRua/mifKEj+6Gs8TJjhFsrZwOXKJAZzSA=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8=
github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY=
github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoWslEpIw=
github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI=
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak=
github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8=
github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 h1:bgTXdrA/yFhFGfjhMIsczVNhnsMEHFidgS/FD2Tq5Js=
github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q=
github.com/projectdiscovery/utils v0.0.51 h1:WZV8kP4VW/I1ZkXDjyoudhhfVVHpAKKnW+Re0LVNMbc=
github.com/projectdiscovery/utils v0.0.51/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc=
github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8=
github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8=
github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E=
github.com/quic-go/quic-go v0.37.0 h1:wf/Ym2yeWi98oQn4ahiBSqdnaXVxNQGj2oBQFgiVChc=
github.com/quic-go/quic-go v0.37.0/go.mod h1:XtCUOCALTTWbPyd0IxFfHf6h0sEMubRFvEYHl3QxKw8=
github.com/refraction-networking/utls v1.4.2 h1:7N+928mSM1pEyAJb8x2Y1FbEwTIftGwn2IFykosSzwc=
github.com/refraction-networking/utls v1.4.2/go.mod h1:JkUIj+Pc8eyFB0z+A4RJRZmoT43ajjFZWVMXuZQ8BEQ=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
@ -463,8 +486,8 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08=
github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU=
github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4=
github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@ -478,8 +501,9 @@ github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2Iqp
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
@ -619,8 +643,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -653,12 +677,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -693,6 +717,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -706,9 +731,9 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -717,19 +742,20 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -767,6 +793,7 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nR
gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o=
gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=

2
v2/pkg/catalog/config/constants.go
View File

@ -17,7 +17,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
Version = `v3.0.0`
Version = `v3.0.0-dev`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"

15
v2/pkg/protocols/code/code.go
View File

@ -32,6 +32,8 @@ type Request struct {
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"`
// description: |
// Engine type
Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine,enum=python,enum=powershell,enum=command"`
@ -96,7 +98,7 @@ func (request *Request) Requests() int {
// GetID returns the ID for the request if any.
func (request *Request) GetID() string {
return ""
return request.ID
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
@ -115,10 +117,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
// inject all template context values as gozero env variables
variables := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
// add template context values
variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll())
// optionvars are vars passed from CLI or env variables
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
variablesMap := request.options.Variables.Evaluate(variables)
variables = generators.MergeMaps(variablesMap, variables, optionVars)
variables = generators.MergeMaps(variablesMap, variables, optionVars, request.options.Constants)
for name, value := range variables {
v := fmt.Sprint(value)
v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs)
@ -151,6 +155,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
// expose response variables in proto_var format
// this is no-op if the template is not a multi protocol template
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data)
// add variables from template context before matching/extraction
data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(interactshURLs, data)
}

32
v2/pkg/protocols/common/contextargs/contextargs.go
View File

@ -50,6 +50,36 @@ func (ctx *Context) hasArgs() bool {
return !ctx.args.IsEmpty()
}
// Merge the key-value pairs
func (ctx *Context) Merge(args map[string]interface{}) {
_ = ctx.args.Merge(args)
}
// Add the specific key-value pair
func (ctx *Context) Add(key string, v interface{}) {
values, ok := ctx.args.Get(key)
if !ok {
ctx.Set(key, v)
}
// If the key exists, append the value to the existing value
switch v := v.(type) {
case []string:
if values, ok := values.([]string); ok {
values = append(values, v...)
ctx.Set(key, values)
}
case string:
if values, ok := values.(string); ok {
tmp := []string{values, v}
ctx.Set(key, tmp)
}
default:
values, _ := ctx.Get(key)
ctx.Set(key, []interface{}{values, v})
}
}
// Get the value with specific key if exists
func (ctx *Context) Get(key string) (interface{}, bool) {
if !ctx.hasArgs() {
@ -86,7 +116,7 @@ func (ctx *Context) HasArgs() bool {
func (ctx *Context) Clone() *Context {
newCtx := &Context{
MetaInput: ctx.MetaInput.Clone(),
args: ctx.args,
args: ctx.args.Clone(),
CookieJar: ctx.CookieJar,
}
return newCtx

19
v2/pkg/protocols/common/contextargs/metainput.go
View File

@ -2,6 +2,7 @@ package contextargs
import (
"bytes"
"crypto/md5"
"fmt"
"strings"
@ -14,6 +15,8 @@ type MetaInput struct {
Input string `json:"input,omitempty"`
// CustomIP to use for connection
CustomIP string `json:"customIP,omitempty"`
// hash of the input
hash string `json:"-"`
}
func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) {
@ -67,3 +70,19 @@ func (metaInput *MetaInput) PrettyPrint() string {
}
return metaInput.Input
}
// GetScanHash returns a unique hash that represents a scan by hashing (metainput + templateId)
func (metaInput *MetaInput) GetScanHash(templateId string) string {
// there may be some cases where metainput is changed ex: while executing self-contained template etc
// but that totally changes the scanID/hash so to avoid that we compute hash only once
// and reuse it for all subsequent calls
if metaInput.hash == "" {
metaInput.hash = getMd5Hash(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP)
}
return metaInput.hash
}
func getMd5Hash(data string) string {
bin := md5.Sum([]byte(data))
return string(bin[:])
}

189
v2/pkg/protocols/common/executer/executer.go
View File

@ -1,189 +0,0 @@
package executer
import (
"fmt"
"strings"
"sync/atomic"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
)
// Executer executes a group of requests for a protocol
type Executer struct {
requests []protocols.Request
options *protocols.ExecutorOptions
}
var _ protocols.Executer = &Executer{}
// NewExecuter creates a new request executer for list of requests
func NewExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *Executer {
return &Executer{requests: requests, options: options}
}
// Compile compiles the execution generators preparing any requests possible.
func (e *Executer) Compile() error {
cliOptions := e.options.Options
for _, request := range e.requests {
if err := request.Compile(e.options); err != nil {
var dslCompilationError *dsl.CompilationError
if errors.As(err, &dslCompilationError) {
if cliOptions.Verbose {
rawErrorMessage := dslCompilationError.Error()
formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "."
gologger.Warning().Msgf(formattedErrorMessage)
gologger.Info().Msgf("The available custom DSL functions are:")
fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor))
}
}
return err
}
}
return nil
}
// Requests returns the total number of requests the rule will perform
func (e *Executer) Requests() int {
var count int
for _, request := range e.requests {
count += request.Requests()
}
return count
}
// Execute executes the protocol group and returns true or false if results were found.
func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
results := &atomic.Bool{}
dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
dynamicValues[key] = value
})
}
previous := make(map[string]interface{})
var lastMatcherEvent *output.InternalWrappedEvent
writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) {
if !results.Load() && matcherStatus {
if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil {
gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
}
results.CompareAndSwap(false, true)
}
}
for _, req := range e.requests {
inputItem := input.Clone()
if e.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.MetaInput.Input = e.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
return false, nil
}
}
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
if event == nil {
// ideally this should never happen since protocol exits on error and callback is not called
return
}
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
for k, v := range event.InternalEvent {
builder.WriteString(ID)
builder.WriteString("_")
builder.WriteString(k)
previous[builder.String()] = v
builder.Reset()
}
}
// If no results were found, and also interactsh is not being used
// in that case we can skip it, otherwise we've to show failure in
// case of matcher-status flag.
if !event.HasOperatorResult() && !event.UsesInteractsh {
lastMatcherEvent = event
} else {
if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) {
results.CompareAndSwap(false, true)
} else {
lastMatcherEvent = event
}
}
})
if err != nil {
if e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err)
}
// If a match was found and stop at first match is set, break out of the loop and return
if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {
break
}
}
if lastMatcherEvent != nil {
writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus)
}
return results.Load(), nil
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
dynamicValues[key] = value
})
}
previous := make(map[string]interface{})
results := &atomic.Bool{}
for _, req := range e.requests {
req := req
inputItem := input.Clone()
if e.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" {
return nil
}
}
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
for k, v := range event.InternalEvent {
builder.WriteString(ID)
builder.WriteString("_")
builder.WriteString(k)
previous[builder.String()] = v
builder.Reset()
}
}
if event.OperatorsResult == nil {
return
}
results.CompareAndSwap(false, true)
callback(event)
})
if err != nil {
if e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err)
}
// If a match was found and stop at first match is set, break out of the loop and return
if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {
break
}
}
return nil
}

9
v2/pkg/protocols/common/generators/generators.go
View File

@ -6,7 +6,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// PayloadGenerator is the generator struct for generating payloads
@ -14,10 +14,11 @@ type PayloadGenerator struct {
Type AttackType
catalog catalog.Catalog
payloads map[string][]string
options *types.Options
}
// New creates a new generator structure for payload generation
func New(payloads map[string]interface{}, attackType AttackType, templatePath string, allowLocalFileAccess bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) {
func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog, customAttackType string, opts *types.Options) (*PayloadGenerator, error) {
if attackType.String() == "" {
attackType = BatteringRamAttack
}
@ -38,12 +39,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
}
}
generator := &PayloadGenerator{catalog: catalog}
generator := &PayloadGenerator{catalog: catalog, options: opts}
if err := generator.validate(payloadsFinal, templatePath); err != nil {
return nil, err
}
compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, allowLocalFileAccess)
compiled, err := generator.loadPayloads(payloadsFinal, templatePath)
if err != nil {
return nil, err
}

13
v2/pkg/protocols/common/generators/generators_test.go
View File

@ -6,13 +6,14 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
func TestBatteringRamGenerator(t *testing.T) {
usernames := []string{"admin", "password"}
catalogInstance := disk.NewCatalog("")
generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", false, catalogInstance, "")
generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance, "", getOptions(false))
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -32,7 +33,7 @@ func TestPitchforkGenerator(t *testing.T) {
passwords := []string{"password1", "password2", "password3"}
catalogInstance := disk.NewCatalog("")
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", false, catalogInstance, "")
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance, "", getOptions(false))
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -54,7 +55,7 @@ func TestClusterbombGenerator(t *testing.T) {
passwords := []string{"admin", "password", "token"}
catalogInstance := disk.NewCatalog("")
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", false, catalogInstance, "")
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance, "", getOptions(false))
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -83,3 +84,9 @@ func TestClusterbombGenerator(t *testing.T) {
}
require.Equal(t, 3, count, "could not get correct clusterbomb counts")
}
func getOptions(allowLocalFileAccess bool) *types.Options {
opts := types.DefaultOptions()
opts.AllowLocalFileAccess = allowLocalFileAccess
return opts
}

26
v2/pkg/protocols/common/generators/load.go
View File

@ -2,7 +2,7 @@ package generators
import (
"bufio"
"path/filepath"
"io"
"strings"
"github.com/pkg/errors"
@ -11,7 +11,7 @@ import (
)
// loadPayloads loads the input payloads from a map to a data map
func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, allowLocalFileAccess bool) (map[string][]string, error) {
func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath string) (map[string][]string, error) {
loadedPayloads := make(map[string][]string)
for name, payload := range payloads {
@ -22,18 +22,11 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{},
if len(elements) >= 2 {
loadedPayloads[name] = elements
} else {
if !allowLocalFileAccess {
pt = filepath.Clean(pt)
templateAbsPath, err := filepath.Abs(templatePath)
if err != nil {
return nil, errors.Wrap(err, "could not get absolute path")
}
templatePathDir := filepath.Dir(templateAbsPath)
if !(templatePathDir != "/" && strings.HasPrefix(pt, templatePathDir)) && !strings.HasPrefix(pt, templateDirectory) {
return nil, errors.New("denied payload file path specified")
}
file, err := generator.options.LoadHelperFile(pt, templatePath, generator.catalog)
if err != nil {
return nil, errors.Wrap(err, "could not load payload file")
}
payloads, err := generator.loadPayloadsFromFile(pt)
payloads, err := generator.loadPayloadsFromFile(file)
if err != nil {
return nil, errors.Wrap(err, "could not load payloads")
}
@ -47,13 +40,8 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{},
}
// loadPayloadsFromFile loads a file to a string slice
func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) {
func (generator *PayloadGenerator) loadPayloadsFromFile(file io.ReadCloser) ([]string, error) {
var lines []string
file, err := generator.catalog.OpenFile(filepath)
if err != nil {
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)

103
v2/pkg/protocols/common/generators/load_test.go
View File

@ -2,66 +2,119 @@ package generators
import (
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
osutils "github.com/projectdiscovery/utils/os"
"github.com/stretchr/testify/require"
)
func TestLoadPayloads(t *testing.T) {
tempdir, err := os.MkdirTemp("", "templates-*")
require.NoError(t, err, "could not create temp dir")
defer os.RemoveAll(tempdir)
// since we are changing value of global variable i.e templates directory
// run this test as subprocess
if os.Getenv("LOAD_PAYLOAD_NO_ACCESS") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess")
cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_NO_ACCESS=1")
err := cmd.Run()
if e, ok := err.(*exec.ExitError); ok && !e.Success() {
return
}
if err != nil {
t.Fatalf("process ran with err %v, want exit status 1", err)
}
}
templateDir := getTemplatesDir(t)
config.DefaultConfig.SetTemplatesDir(templateDir)
generator := &PayloadGenerator{catalog: disk.NewCatalog(tempdir)}
fullpath := filepath.Join(tempdir, "payloads.txt")
err = os.WriteFile(fullpath, []byte("test\nanother"), 0777)
require.NoError(t, err, "could not write payload")
generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(false)}
fullpath := filepath.Join(templateDir, "payloads.txt")
// Test sandbox
t.Run("templates-directory", func(t *testing.T) {
// testcase when loading file from template directory and template file is in root
// expected to succeed
values, err := generator.loadPayloads(map[string]interface{}{
"new": fullpath,
}, "/test", tempdir, false)
}, "/test")
require.NoError(t, err, "could not load payloads")
require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values")
})
t.Run("templates-path-relative", func(t *testing.T) {
// testcase when loading file from template directory and template file is current working directory
// expected to fail since this is LFI
_, err := generator.loadPayloads(map[string]interface{}{
"new": "../../../../../../../../../etc/passwd",
}, ".", tempdir, false)
}, ".")
require.Error(t, err, "could load payloads")
})
t.Run("template-directory", func(t *testing.T) {
// testcase when loading file from template directory and template file is inside template directory
// expected to succeed
values, err := generator.loadPayloads(map[string]interface{}{
"new": fullpath,
}, filepath.Join(tempdir, "test.yaml"), "/test", false)
}, filepath.Join(templateDir, "test.yaml"))
require.NoError(t, err, "could not load payloads")
require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values")
})
t.Run("invalid", func(t *testing.T) {
// testcase when loading file from /etc/passwd and template file is at root i.e /
// expected to fail since this is LFI
values, err := generator.loadPayloads(map[string]interface{}{
"new": "/etc/passwd",
}, "/random")
require.Error(t, err, "could load payloads")
require.Equal(t, 0, len(values), "could get values")
// testcase when loading file from template directory and template file is at root i.e /
// expected to succeed
values, err = generator.loadPayloads(map[string]interface{}{
"new": fullpath,
}, "/random")
require.NoError(t, err, "could load payloads %v", values)
require.Equal(t, 1, len(values), "could get values")
require.Equal(t, []string{"test", "another"}, values["new"], "could get values")
})
}
func TestLoadPayloadsWithAccess(t *testing.T) {
// since we are changing value of global variable i.e templates directory
// run this test as subprocess
if os.Getenv("LOAD_PAYLOAD_WITH_ACCESS") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess")
cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_WITH_ACCESS=1")
err := cmd.Run()
if e, ok := err.(*exec.ExitError); ok && !e.Success() {
return
}
if err != nil {
t.Fatalf("process ran with err %v, want exit status 1", err)
}
}
templateDir := getTemplatesDir(t)
config.DefaultConfig.SetTemplatesDir(templateDir)
generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(true)}
t.Run("no-sandbox-unix", func(t *testing.T) {
if osutils.IsWindows() {
return
}
_, err := generator.loadPayloads(map[string]interface{}{
"new": "/etc/passwd",
}, "/random", "/test", true)
}, "/random")
require.NoError(t, err, "could load payloads")
})
t.Run("invalid", func(t *testing.T) {
values, err := generator.loadPayloads(map[string]interface{}{
"new": "/etc/passwd",
}, "/random", "/test", false)
require.Error(t, err, "could load payloads")
require.Equal(t, 0, len(values), "could get values")
values, err = generator.loadPayloads(map[string]interface{}{
"new": fullpath,
}, "/random", "/test", false)
require.Error(t, err, "could load payloads")
require.Equal(t, 0, len(values), "could get values")
})
}
func getTemplatesDir(t *testing.T) string {
tempdir, err := os.MkdirTemp("", "templates-*")
require.NoError(t, err, "could not create temp dir")
fullpath := filepath.Join(tempdir, "payloads.txt")
err = os.WriteFile(fullpath, []byte("test\nanother"), 0777)
require.NoError(t, err, "could not write payload")
return tempdir
}

19
v2/pkg/protocols/common/variables/variables.go
View File

@ -66,6 +66,11 @@ func (variables *Variable) UnmarshalJSON(data []byte) error {
func (variables *Variable) Evaluate(values map[string]interface{}) map[string]interface{} {
result := make(map[string]interface{}, variables.Len())
variables.ForEach(func(key string, value interface{}) {
if sliceValue, ok := value.([]interface{}); ok {
// slices cannot be evaluated
result[key] = sliceValue
return
}
valueString := types.ToString(value)
combined := generators.MergeMaps(values, result)
if value, ok := combined[key]; ok {
@ -76,12 +81,26 @@ func (variables *Variable) Evaluate(values map[string]interface{}) map[string]in
return result
}
// GetAll returns all variables as a map
func (variables *Variable) GetAll() map[string]interface{} {
result := make(map[string]interface{}, variables.Len())
variables.ForEach(func(key string, value interface{}) {
result[key] = value
})
return result
}
// EvaluateWithInteractsh returns evaluation results of variables with interactsh
func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{}, interact *interactsh.Client) (map[string]interface{}, []string) {
result := make(map[string]interface{}, variables.Len())
var interactURLs []string
variables.ForEach(func(key string, value interface{}) {
if sliceValue, ok := value.([]interface{}); ok {
// slices cannot be evaluated
result[key] = sliceValue
return
}
valueString := types.ToString(value)
if strings.Contains(valueString, "interactsh-url") {
valueString, interactURLs = interact.Replace(valueString, interactURLs)

2
v2/pkg/protocols/dns/dns.go
View File

@ -172,7 +172,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
if len(request.Payloads) > 0 {
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType)
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}

12
v2/pkg/protocols/dns/request.go
View File

@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
// optionvars are vars passed from CLI or env variables
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
// merge with metadata (eg. from workflow context)
vars = generators.MergeMaps(vars, metadata, optionVars, request.options.TemplateCtx.GetAll())
vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll())
variablesMap := request.options.Variables.Evaluate(vars)
vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
@ -66,18 +66,18 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
break
}
value = generators.MergeMaps(vars, value)
if err := request.execute(domain, metadata, previous, value, callback); err != nil {
if err := request.execute(input, domain, metadata, previous, value, callback); err != nil {
return err
}
}
} else {
value := maps.Clone(vars)
return request.execute(domain, metadata, previous, value, callback)
return request.execute(input, domain, metadata, previous, value, callback)
}
return nil
}
func (request *Request) execute(domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error {
func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error {
if vardump.EnableVarDump {
gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(vars))
@ -151,7 +151,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna
outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData)
// expose response variables in proto_var format
// this is no-op if the template is not a multi protocol template
request.options.AddTemplateVars(request.Type(), outputEvent)
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
for k, v := range previous {
outputEvent[k] = v
}
@ -159,7 +159,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna
outputEvent[k] = v
}
// add variables from template context before matching/extraction
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
dumpResponse(event, request, request.options, response.String(), question)

18
v2/pkg/protocols/file/request.go
View File

@ -64,7 +64,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
// every new file in the compressed multi-file archive counts 1
request.options.Progress.AddToTotal(1)
archiveFileName := filepath.Join(filePath, file.Name())
event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous)
event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous)
if err != nil {
if errors.Is(err, errEmptyResult) {
// no matches but one file elaborated
@ -117,7 +117,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
_ = tmpFileOut.Sync()
// rewind the file
_, _ = tmpFileOut.Seek(0, 0)
event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous)
event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous)
if err != nil {
if errors.Is(err, errEmptyResult) {
// no matches but one file elaborated
@ -137,7 +137,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
default:
// normal file - increments the counter by 1
request.options.Progress.AddToTotal(1)
event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous)
event, fileMatches, err := request.processFile(filePath, input, previous)
if err != nil {
if errors.Is(err, errEmptyResult) {
// no matches but one file elaborated
@ -166,7 +166,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
return nil
}
func (request *Request) processFile(filePath, input string, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) {
func (request *Request) processFile(filePath string, input *contextargs.Context, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, nil, errors.Errorf("Could not open file path %s: %s\n", filePath, err)
@ -185,7 +185,7 @@ func (request *Request) processFile(filePath, input string, previousInternalEven
return request.processReader(file, filePath, input, stat.Size(), previousInternalEvent)
}
func (request *Request) processReader(reader io.Reader, filePath, input string, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) {
func (request *Request) processReader(reader io.Reader, filePath string, input *contextargs.Context, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) {
fileReader := io.LimitReader(reader, request.maxSize)
fileMatches, opResult := request.findMatchesWithReader(fileReader, input, filePath, totalBytes, previousInternalEvent)
if opResult == nil && len(fileMatches) == 0 {
@ -193,10 +193,10 @@ func (request *Request) processReader(reader io.Reader, filePath, input string,
}
// build event structure to interface with internal logic
return request.buildEvent(input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil
return request.buildEvent(input.MetaInput.Input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil
}
func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) {
func (request *Request) findMatchesWithReader(reader io.Reader, input *contextargs.Context, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) {
var bytesCount, linesCount, wordsCount int
isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse
totalBytesString := units.BytesSize(float64(totalBytes))
@ -243,12 +243,12 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath
processedBytes := units.BytesSize(float64(currentBytes))
gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytesString)
dslMap := request.responseToDSLMap(lineContent, input, filePath)
dslMap := request.responseToDSLMap(lineContent, input.MetaInput.Input, filePath)
for k, v := range previous {
dslMap[k] = v
}
// add template context variables to DSL map
dslMap = generators.MergeMaps(dslMap, request.options.TemplateCtx.GetAll())
dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll())
discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug)
newOpResult := discardEvent.OperatorsResult
if newOpResult != nil {

4
v2/pkg/protocols/headless/engine/page_actions_test.go
View File

@ -349,7 +349,7 @@ func TestActionGetResource(t *testing.T) {
<title>Nuclei Test Page</title>
</head>
<body>
<img id="test" src="https://nuclei.projectdiscovery.io/static/logo.png">
<img id="test" src="https://raw.githubusercontent.com/projectdiscovery/wallpapers/main/pd-floppy.jpg">
</body>
</html>`
@ -360,7 +360,7 @@ func TestActionGetResource(t *testing.T) {
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, len(out["src"]), 3159, "could not find resource")
require.Equal(t, len(out["src"]), 121808, "could not find resource")
})
}

5
v2/pkg/protocols/headless/headless.go
View File

@ -91,6 +91,8 @@ func (request *Request) GetID() string {
// Compile compiles the protocol request for further execution.
func (request *Request) Compile(options *protocols.ExecutorOptions) error {
request.options = options
// TODO: logic similar to network + http => probably can be refactored
// Resolve payload paths from vars if they exists
for name, payload := range options.Options.Vars.AsMap() {
@ -106,7 +108,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
if len(request.Payloads) > 0 {
var err error
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType)
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Catalog, options.Options.AttackType, request.options.Options)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}
@ -136,7 +138,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
request.CompiledOperators = compiled
}
request.options = options
if len(request.Fuzzing) > 0 {
for _, rule := range request.Fuzzing {

6
v2/pkg/protocols/headless/request.go
View File

@ -44,7 +44,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
vars := protocolutils.GenerateVariablesWithContextArgs(input, false)
payloads := generators.BuildPayloadFromOptions(request.options.Options)
// add templatecontext variables to varMap
values := generators.MergeMaps(vars, metadata, payloads, request.options.TemplateCtx.GetAll())
values := generators.MergeMaps(vars, metadata, payloads, request.options.GetTemplateCtx(input.MetaInput).GetAll())
variablesMap := request.options.Variables.Evaluate(values)
payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants)
@ -156,8 +156,8 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory())
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())
for k, v := range out {
outputEvent[k] = v
}

5
v2/pkg/protocols/http/build_request.go
View File

@ -72,7 +72,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
// 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path
// add template context values to dynamicValues (this takes care of self-contained and other types of requests)
dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll())
// Note: `iterate-all` and flow are mutually exclusive. flow uses templateCtx and iterate-all uses dynamicValues
dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
if r.request.SelfContained {
return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues)
}
@ -85,7 +86,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
}
} else {
for payloadName, payloadValue := range payloads {
payloads[payloadName] = types.ToString(payloadValue)
payloads[payloadName] = types.ToStringNSlice(payloadValue)
}
}

3
v2/pkg/protocols/http/http.go
View File

@ -190,6 +190,7 @@ type Request struct {
SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" json:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"`
// description: |
// IterateAll iterates all the values extracted from internal extractors
// Deprecated: Use flow instead . iterate-all will be removed in future releases
IterateAll bool `yaml:"iterate-all,omitempty" json:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"`
// description: |
// DigestAuthUsername specifies the username for digest authentication
@ -353,7 +354,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
if len(request.Payloads) > 0 {
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType)
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}

5
v2/pkg/protocols/http/raw/raw.go
View File

@ -102,7 +102,10 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b
if _, ok := rawrequest.Headers["Host"]; !ok {
rawrequest.Headers["Host"] = inputURL.Host
}
rawrequest.FullURL = fmt.Sprintf("%s://%s%s", inputURL.Scheme, strings.TrimSpace(inputURL.Host), rawrequest.Path)
cloned := inputURL.Clone()
cloned.Path = ""
_ = cloned.MergePath(rawrequest.Path, true)
rawrequest.FullURL = cloned.String()
}
return rawrequest, nil

4
v2/pkg/protocols/http/request.go
View File

@ -721,8 +721,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())
if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i]
}

5
v2/pkg/protocols/http/request_generator_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
func TestRequestGeneratorPaths(t *testing.T) {
@ -34,7 +35,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) {
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`},
}
catalogInstance := disk.NewCatalog("")
req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "")
req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance, "", types.DefaultOptions())
require.Nil(t, err, "could not create generator")
generator := req.newGenerator(false)
@ -58,7 +59,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) {
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
}
catalogInstance := disk.NewCatalog("")
req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "")
req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance, "", types.DefaultOptions())
require.Nil(t, err, "could not create generator")
generator := req.newGenerator(false)

6
v2/pkg/protocols/multi/doc.go
View File

@ -1,6 +0,0 @@
package multi
// multi is a wrapper protocol Request that allows multiple protocols requests to be executed
// multi protocol is just a wrapper so it should/does not include any protocol specific code

172
v2/pkg/protocols/multi/request.go
View File

@ -1,172 +0,0 @@
package multi
import (
"strconv"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
errorutil "github.com/projectdiscovery/utils/errors"
)
var _ protocols.Request = &Request{}
// refer doc.go for package description , limitations etc
// Request contains a multi protocol request
type Request struct {
// description: |
// ID is the unique id for the template.
//
// #### Good IDs
//
// A good ID uniquely identifies what the requests in the template
// are doing. Let's say you have a template that identifies a git-config
// file on the webservers, a good name would be `git-config-exposure`. Another
// example name is `azure-apps-nxdomain-takeover`.
// examples:
// - name: ID Example
// value: "\"CVE-2021-19520\""
ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"`
// description: |
// Info contains metadata information about the template.
// examples:
// - value: exampleInfoStructure
Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"`
// Queue is queue of all protocols present in the template
Queue []protocols.Request `yaml:"-" json:"-"`
// request executor options
options *protocols.ExecutorOptions `yaml:"-" json:"-"`
}
// getLastRequest returns the last request in the queue
func (r *Request) getLastRequest() protocols.Request {
if len(r.Queue) == 0 {
return nil
}
return r.Queue[len(r.Queue)-1]
}
// Requests returns the total number of requests template will send
func (r *Request) Requests() int {
var count int
for _, protocol := range r.Queue {
count += protocol.Requests()
}
return count
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(executerOptions *protocols.ExecutorOptions) error {
r.options = executerOptions
r.options.TemplateCtx = contextargs.New()
r.options.ProtocolType = types.MultiProtocol
for _, protocol := range r.Queue {
if err := protocol.Compile(r.options); err != nil {
return errorutil.NewWithErr(err).Msgf("failed to compile protocol %s", protocol.Type())
}
}
return nil
}
// GetID returns the unique template ID
func (r *Request) GetID() string {
return r.ID
}
// Match executes matcher on model and returns true or false (used for clustering if request supports clustering)
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
return protocols.MakeDefaultMatchFunc(data, matcher)
}
// Extract performs extracting operation for an extractor on model and returns true or false (used for clustering if request supports clustering)
func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
return protocols.MakeDefaultExtractFunc(data, matcher)
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var finalProtoEvent *output.InternalWrappedEvent
// callback to process results from all protocols
multiProtoCallback := func(event *output.InternalWrappedEvent) {
finalProtoEvent = event
// export dynamic values from operators (i.e internal:true)
if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 {
for k, v := range event.OperatorsResult.DynamicValues {
// TBD: iterate-all is only supported in `http` protocol
// we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context)
// currently if dynamic value array only contains one value we replace it with the value
if len(v) == 1 {
r.options.TemplateCtx.Set(k, v[0])
} else {
// Note: if extracted value contains multiple values then they can be accessed by indexing
// ex: if values are dynamic = []string{"a","b","c"} then they are available as
// dynamic = "a" , dynamic1 = "b" , dynamic2 = "c"
// we intentionally omit first index for unknown situations (where no of extracted values are not known)
for i, val := range v {
if i == 0 {
r.options.TemplateCtx.Set(k, val)
} else {
r.options.TemplateCtx.Set(k+strconv.Itoa(i), val)
}
}
}
}
}
}
// template context: contains values extracted using `internal` extractor from previous protocols
// these values are extracted from each protocol in queue and are passed to next protocol in queue
// instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows)
// this makes it possible to use multi protocol templates in workflows
// Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow)
// execute all protocols in the queue
for _, req := range r.Queue {
err := req.ExecuteWithResults(input, dynamicValues, previous, multiProtoCallback)
// if error skip execution of next protocols
if err != nil {
return err
}
}
// Review: how to handle events of multiple protocols in a single template
// currently the outer callback is only executed once (for the last protocol in queue)
// due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executer.go#L150
// this causes addition of duplicated / unncessary variables with prefix template_id_all_variables
callback(finalProtoEvent)
return nil
}
// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally
func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
if r.getLastRequest() == nil {
return nil
}
return r.getLastRequest().MakeResultEventItem(wrapped)
}
// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(r.getLastRequest(), wrapped)
}
// GetCompiledOperators returns a list of the compiled operators
func (r *Request) GetCompiledOperators() []*operators.Operators {
last := r.getLastRequest()
if last == nil {
return nil
}
return last.GetCompiledOperators()
}
// Type returns the type of the protocol request
func (r *Request) Type() types.ProtocolType {
return types.MultiProtocol
}

2
v2/pkg/protocols/network/network.go
View File

@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
if len(request.Payloads) > 0 {
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType)
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}

14
v2/pkg/protocols/network/request.go
View File

@ -56,7 +56,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
}
variables := protocolutils.GenerateVariables(address, false, nil)
// add template ctx variables to varMap
variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll())
variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll())
variablesMap := request.options.Variables.Evaluate(variables)
variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
@ -70,7 +70,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
}
visitedAddresses.Set(actualAddress, struct{}{})
if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil {
if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
outputEvent := request.responseToDSLMap("", "", "", address, "")
callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err)
@ -81,7 +81,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
}
// executeAddress executes the request for an address
func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address})
payloads := generators.BuildPayloadFromOptions(request.options.Options)
@ -114,7 +114,7 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
return nil
}
func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var (
hostname string
conn net.Conn
@ -279,10 +279,10 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
}
response := responseBuilder.String()
outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)
outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input.MetaInput.Input, actualAddress)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())
outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
if request.options.StopAtFirstMatch {
outputEvent["stop-at-first-match"] = true

4
v2/pkg/protocols/offlinehttp/request.go
View File

@ -88,8 +88,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())
outputEvent["ip"] = ""
for k, v := range previous {
outputEvent[k] = v

74
v2/pkg/protocols/protocols.go
View File

@ -1,7 +1,10 @@
package protocols
import (
"sync/atomic"
"github.com/projectdiscovery/ratelimit"
mapsutil "github.com/projectdiscovery/utils/maps"
stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/logrusorgru/aurora"
@ -86,46 +89,87 @@ type ExecutorOptions struct {
Colorizer aurora.Aurora
WorkflowLoader model.WorkflowLoader
ResumeCfg *types.ResumeCfg
// TemplateContext (contains all variables that are templatescoped i.e multi protocol)
// only used in case of multi protocol templates
TemplateCtx *contextargs.Context
// ProtocolType is the type of the template
ProtocolType templateTypes.ProtocolType
// Flow is execution flow for the template (written in javascript)
Flow string
// IsMultiProtocol is true if template has more than one protocol
IsMultiProtocol bool
// templateStore is a map which contains template context for each scan (i.e input * template-id pair)
templateCtxStore *mapsutil.SyncLockMap[string, *contextargs.Context]
}
// CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan)
func (e *ExecutorOptions) CreateTemplateCtxStore() {
e.templateCtxStore = &mapsutil.SyncLockMap[string, *contextargs.Context]{
Map: make(map[string]*contextargs.Context),
ReadOnly: atomic.Bool{},
}
}
// RemoveTemplateCtx removes template context of given scan from store
func (e *ExecutorOptions) RemoveTemplateCtx(input *contextargs.MetaInput) {
scanId := input.GetScanHash(e.TemplateID)
if e.templateCtxStore != nil {
e.templateCtxStore.Delete(scanId)
}
}
// GetTemplateCtx returns template context for given input
func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contextargs.Context {
scanId := input.GetScanHash(e.TemplateID)
templateCtx, ok := e.templateCtxStore.Get(scanId)
if !ok {
// if template context does not exist create new and add it to store and return it
templateCtx = contextargs.New()
_ = e.templateCtxStore.Set(scanId, templateCtx)
}
return templateCtx
}
// AddTemplateVars adds vars to template context with given template type as prefix
// this method is no-op if template is not multi protocol
func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) {
if e.ProtocolType != templateTypes.MultiProtocol {
// no-op if not multi protocol template
func (e *ExecutorOptions) AddTemplateVars(input *contextargs.MetaInput, reqType templateTypes.ProtocolType, reqID string, vars map[string]interface{}) {
// if we wan't to disable adding response variables and other variables to template context
// this is the statement that does it . template context is currently only enabled for
// multiprotocol and flow templates
if !e.IsMultiProtocol && e.Flow == "" {
// no-op if not multi protocol template or flow template
return
}
templateCtx := e.GetTemplateCtx(input)
for k, v := range vars {
if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") {
if templateType < templateTypes.InvalidProtocol {
k = templateType.String() + "_" + k
if reqID != "" {
k = reqID + "_" + k
} else if reqType < templateTypes.InvalidProtocol {
k = reqType.String() + "_" + k
}
e.TemplateCtx.Set(k, v)
templateCtx.Set(k, v)
}
}
}
// AddTemplateVar adds given var to template context with given template type as prefix
// this method is no-op if template is not multi protocol
func (e *ExecutorOptions) AddTemplateVar(prefix, key string, value interface{}) {
if e.ProtocolType != templateTypes.MultiProtocol {
// no-op if not multi protocol template
func (e *ExecutorOptions) AddTemplateVar(input *contextargs.MetaInput, templateType templateTypes.ProtocolType, reqID string, key string, value interface{}) {
if !e.IsMultiProtocol && e.Flow == "" {
// no-op if not multi protocol template or flow template
return
}
if prefix != "" {
key = prefix + "_" + key
templateCtx := e.GetTemplateCtx(input)
if reqID != "" {
key = reqID + "_" + key
} else if templateType < templateTypes.InvalidProtocol {
key = templateType.String() + "_" + key
}
e.TemplateCtx.Set(key, value)
templateCtx.Set(key, value)
}
// Copy returns a copy of the executeroptions structure
func (e ExecutorOptions) Copy() ExecutorOptions {
copy := e
copy.CreateTemplateCtxStore()
return copy
}

13
v2/pkg/protocols/ssl/ssl.go
View File

@ -41,6 +41,9 @@ type Request struct {
operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"`
// description: |
// Address contains address for the request
Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"`
@ -184,7 +187,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
hostnameVariables := protocolutils.GenerateDNSVariables(hostname)
// add template context variables to varMap
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll())
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(input.MetaInput).GetAll())
variablesMap := request.options.Variables.Evaluate(values)
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)
@ -219,7 +222,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}
requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err)
gologger.Verbose().Msgf("Sent SSL request to %s", hostPort)
gologger.Verbose().Msgf("[%s] Sent SSL request to %s", request.options.TemplateID, hostPort)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.MetaInput.Input)
@ -267,7 +270,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if tag == "" || f.IsZero() {
continue
}
request.options.AddTemplateVar(request.Type().String(), tag, f.Value())
request.options.AddTemplateVar(input.MetaInput, request.Type(), request.ID, tag, f.Value())
data[tag] = f.Value()
}
@ -286,12 +289,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if tag == "" || f.IsZero() {
continue
}
request.options.AddTemplateVar(request.Type().String(), tag, f.Value())
request.options.AddTemplateVar(input.MetaInput, request.Type(), request.ID, tag, f.Value())
data[tag] = f.Value()
}
// add response fields ^ to template context and merge templatectx variables to output event
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input)

17
v2/pkg/protocols/websocket/websocket.go
View File

@ -42,6 +42,8 @@ type Request struct {
operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"`
// description: |
// Address contains address for the request
Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"`
@ -106,7 +108,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
request.dialer = client
if len(request.Payloads) > 0 {
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType)
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions())
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}
@ -152,13 +154,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if !ok {
break
}
if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil {
if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil {
return err
}
}
} else {
value := make(map[string]interface{})
if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil {
if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil {
return err
}
}
@ -166,8 +168,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeRequestWithPayloads(target *contextargs.Context, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
header := http.Header{}
input := target.MetaInput.Input
parsed, err := urlutil.Parse(input)
if err != nil {
@ -176,7 +179,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
defaultVars := protocolutils.GenerateVariables(parsed, false, nil)
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
// add templatecontext variables to varMap
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll()))
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(target.MetaInput).GetAll()))
payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)
requestOptions := request.options
@ -265,8 +268,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
data["ip"] = request.dialer.GetDialedIP(hostname)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), data)
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
request.options.AddTemplateVars(target.MetaInput, request.Type(), request.ID, data)
data = generators.MergeMaps(data, request.options.GetTemplateCtx(target.MetaInput).GetAll())
for k, v := range previous {
data[k] = v

9
v2/pkg/protocols/whois/whois.go
View File

@ -34,6 +34,9 @@ type Request struct {
operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"`
// description: |
// Query contains query for the request
Query string `yaml:"query,omitempty" json:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"`
@ -91,7 +94,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
defaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
// add templatectx variables to varMap
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll()))
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(input.MetaInput).GetAll()))
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)
@ -134,8 +137,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
data["response"] = jsonDataString
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), data)
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data)
data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {

82
v2/pkg/templates/compile.go
View File

@ -3,20 +3,23 @@ package templates
import (
"fmt"
"io"
"path/filepath"
"reflect"
"strings"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/signer"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@ -59,7 +62,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo
}
defer reader.Close()
options.TemplatePath = filePath
template, err := ParseTemplateFromReader(reader, preprocessor, options)
template, err := ParseTemplateFromReader(reader, preprocessor, options.Copy())
if err != nil {
return nil, err
}
@ -124,34 +127,43 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti
var requests []protocols.Request
if len(template.MultiProtoRequest.Queue) > 0 {
template.MultiProtoRequest.ID = template.ID
template.MultiProtoRequest.Info = template.Info
requests = append(requests, &template.MultiProtoRequest)
if template.hasMultipleRequests() {
// when multiple requests are present preserve the order of requests and protocols
// which is already done during unmarshalling
requests = template.RequestsQueue
if options.Flow == "" {
options.IsMultiProtocol = true
}
} else {
switch {
case len(template.RequestsDNS) > 0:
if len(template.RequestsDNS) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
case len(template.RequestsFile) > 0:
}
if len(template.RequestsFile) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
case len(template.RequestsNetwork) > 0:
}
if len(template.RequestsNetwork) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
case len(template.RequestsHTTP) > 0:
}
if len(template.RequestsHTTP) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
case len(template.RequestsHeadless) > 0 && options.Options.Headless:
}
if len(template.RequestsHeadless) > 0 && options.Options.Headless {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
case len(template.RequestsSSL) > 0:
}
if len(template.RequestsSSL) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
case len(template.RequestsWebsocket) > 0:
}
if len(template.RequestsWebsocket) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
case len(template.RequestsWHOIS) > 0:
}
if len(template.RequestsWHOIS) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
}
if len(template.RequestsCode) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...)
}
}
if len(template.RequestsCode) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...)
}
template.Executer = executer.NewExecuter(requests, &options)
template.Executer = tmplexec.NewTemplateExecuter(requests, &options)
return nil
}
@ -196,7 +208,7 @@ mainLoop:
}
if len(operatorsList) > 0 {
options.Operators = operatorsList
template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
return nil
}
@ -237,10 +249,36 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
options.Variables = template.Variables
}
// if more than 1 request per protocol exist we add request id to protocol request
// since in template context we have proto_prefix for each protocol it is overwritten
// if request id is not present
template.validateAllRequestIDs()
// TODO: we should add a syntax check here or somehow use a javascript linter
// simplest option for now seems to compile using goja and see if it fails
if strings.TrimSpace(template.Flow) != "" {
if len(template.Flow) > 0 && filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) {
// load file respecting sandbox
file, err := options.Options.LoadHelperFile(template.Flow, options.TemplatePath, options.Catalog)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("loading flow file from %v denied", template.Flow)
}
defer file.Close()
if bin, err := io.ReadAll(file); err == nil {
template.Flow = string(bin)
} else {
return nil, errorutil.NewWithErr(err).Msgf("something went wrong failed to read file")
}
}
options.Flow = template.Flow
}
// create empty context args for template scope
options.TemplateCtx = contextargs.New()
options.CreateTemplateCtxStore()
options.ProtocolType = template.Type()
options.Constants = template.Constants
template.Options = &options
// If no requests, and it is also not a workflow, return error.
if template.Requests() == 0 {
return nil, fmt.Errorf("no requests defined for %s", template.ID)

14
v2/pkg/templates/compile_test.go
View File

@ -25,7 +25,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"github.com/projectdiscovery/ratelimit"
@ -197,16 +196,3 @@ func Test_WrongTemplate(t *testing.T) {
require.Nil(t, got, "could not parse template")
require.ErrorContains(t, err, "no requests defined ")
}
func Test_Multiprotocol(t *testing.T) {
setup()
got, err := templates.Parse("tests/multiproto.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 3, got.Requests())
require.Equal(t, types.MultiProtocol, got.Type())
got, err = templates.Parse("tests/multiproto.json", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 3, got.Requests())
require.Equal(t, types.MultiProtocol, got.Type())
}

141
v2/pkg/templates/templates.go
View File

@ -3,6 +3,7 @@ package templates
import (
"encoding/json"
"strconv"
validate "github.com/go-playground/validator/v10"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
@ -13,7 +14,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/file"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/multi"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
@ -47,6 +47,18 @@ type Template struct {
// - value: exampleInfoStructure
Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"`
// description: |
// Flow contains the execution flow for the template.
// examples:
// - flow: |
// for region in regions {
// http(0)
// }
// for vpc in vpcs {
// http(1)
// }
//
Flow string `yaml:"flow,omitempty" json:"flow,omitempty" jsonschema:"title=template execution flow in js,description=Flow contains js code which defines how the template should be executed"`
// description: |
// Requests contains the http request to make in the template.
// WARNING: 'requests' will be deprecated and will be removed in a future release. Please use 'http' instead.
// examples:
@ -134,15 +146,13 @@ type Template struct {
// Verified defines if the template signature is digitally verified
Verified bool `yaml:"-" json:"-"`
// MultiProtoRequest (Internal) contains multi protocol request if multiple protocols are used
MultiProtoRequest multi.Request `yaml:"-" json:"-"`
// RequestsQueue contains all template requests in order (both protocol & request order)
RequestsQueue []protocols.Request `yaml:"-" json:"-"`
}
// Type returns the type of the template
func (template *Template) Type() types.ProtocolType {
switch {
case len(template.MultiProtoRequest.Queue) > 0:
return types.MultiProtocol
case len(template.RequestsDNS) > 0:
return types.DNSProtocol
case len(template.RequestsFile) > 0:
@ -168,9 +178,84 @@ func (template *Template) Type() types.ProtocolType {
}
}
// validateAllRequestIDs check if that protocol already has given id if not
// then is is manually set to proto_index
func (template *Template) validateAllRequestIDs() {
// this is required in multiprotocol and flow where we save response variables
// and all other data in template context if template as two requests in a protocol
// then it is overwritten to avoid this we use proto_index as request ID
if len(template.RequestsCode) > 1 {
for i, req := range template.RequestsCode {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsDNS) > 1 {
for i, req := range template.RequestsDNS {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsFile) > 1 {
for i, req := range template.RequestsFile {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsHTTP) > 1 {
for i, req := range template.RequestsHTTP {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsHeadless) > 1 {
for i, req := range template.RequestsHeadless {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsNetwork) > 1 {
for i, req := range template.RequestsNetwork {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsSSL) > 1 {
for i, req := range template.RequestsSSL {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsWebsocket) > 1 {
for i, req := range template.RequestsWebsocket {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
if len(template.RequestsWHOIS) > 1 {
for i, req := range template.RequestsWHOIS {
if req.ID == "" {
req.ID = req.Type().String() + "_" + strconv.Itoa(i)
}
}
}
}
// MarshalYAML forces recursive struct validation during marshal operation
func (template *Template) MarshalYAML() ([]byte, error) {
out, marshalErr := yaml.Marshal(template)
// Review: we are adding requestIDs for templateContext
// if we are using this method then we might need to purge manually added IDS that start with `templatetype_`
// this is only applicable if there are more than 1 request fields in protocol
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
}
@ -205,8 +290,9 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
if err != nil {
return err
}
// check if the template contains a multi protocols
if template.isMultiProtocol() {
// check if the template contains more than 1 protocol request
// if so preserve the order of the protocols and requests
if template.hasMultipleRequests() {
var tempmap yaml.MapSlice
err = unmarshal(&tempmap)
if err != nil {
@ -221,37 +307,45 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
arr = append(arr, key)
}
// add protocols to the protocol stack (the idea is to preserve the order of the protocols)
template.addProtocolsToQueue(arr...)
template.addRequestsToQueue(arr...)
}
return nil
}
// Internal function to create a protocol stack from a template if the template is a multi protocol template
func (template *Template) addProtocolsToQueue(keys ...string) {
// addProtocolsToQueue adds protocol requests to the queue and preserves order of the protocols and requests
func (template *Template) addRequestsToQueue(keys ...string) {
for _, key := range keys {
switch key {
case types.DNSProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
case types.FileProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
case types.HTTPProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
case types.HeadlessProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
case types.NetworkProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
case types.SSLProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
case types.WebsocketProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
case types.WHOISProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
case types.CodeProtocol.String():
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsCode)...)
// for deprecated protocols
case "requests":
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
case "network":
template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
}
}
}
// isMultiProtocol checks if the template is a multi protocol template
func (template *Template) isMultiProtocol() bool {
// hasMultipleRequests checks if the template has multiple requests
// if so it preserves the order of the request during compile and execution
func (template *Template) hasMultipleRequests() bool {
counter := len(template.RequestsDNS) + len(template.RequestsFile) +
len(template.RequestsHTTP) + len(template.RequestsHeadless) +
len(template.RequestsNetwork) + len(template.RequestsSSL) +
@ -279,8 +373,9 @@ func (template *Template) UnmarshalJSON(data []byte) error {
if err != nil {
return err
}
// check if template contains multiple protocols
if template.isMultiProtocol() {
// check if the template contains more than 1 protocol request
// if so preserve the order of the protocols and requests
if template.hasMultipleRequests() {
var tempMap map[string]interface{}
err = json.Unmarshal(data, &tempMap)
if err != nil {
@ -290,7 +385,7 @@ func (template *Template) UnmarshalJSON(data []byte) error {
for k := range tempMap {
arr = append(arr, k)
}
template.addProtocolsToQueue(arr...)
template.addRequestsToQueue(arr...)
}
return nil
}

3
v2/pkg/templates/types/types.go
View File

@ -38,8 +38,6 @@ const (
WHOISProtocol
// name:code
CodeProtocol
// name: multi
MultiProtocol
limit
InvalidProtocol
)
@ -57,7 +55,6 @@ var protocolMappings = map[ProtocolType]string{
WebsocketProtocol: "websocket",
WHOISProtocol: "whois",
CodeProtocol: "code",
MultiProtocol: "multi",
}
func GetSupportedProtocolTypes() ProtocolTypes {

3
v2/pkg/testutils/testutils.go
View File

@ -16,7 +16,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -91,8 +90,8 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
Browser: nil,
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
TemplateCtx: contextargs.New(),
}
executerOpts.CreateTemplateCtxStore()
return executerOpts
}

11
v2/pkg/tmplexec/README.md Normal file
View File

@ -0,0 +1,11 @@
# tmplexec
tmplexec also known as template executer executes template it is different from `protocols` package which only contains logic within the scope of one protocol. tmplexec is resposible for executing `Template` with defined logic. with introduction of `multi protocol` and `flow` templates (deprecated package protocols/common/executer) did not seem appropriate/helpful anymore as it is outside of protocol scope and deals with execution of template which can contain 1 requests , or multiple requests of same protocol or multiple requests of different protocols. tmplexec is responsible for executing template and handling all logic related to it.
## Engine/Backends
Currently there are 3 engines for template execution
- `Generic` => executes request[s] of same/one protocol
- `MultiProtocol` => executes requests of multiple protocols with shared logic between protocol requests see [multiprotocol](multiproto/README.md)
- `Flow` => executes requests of one or multiple protocol requests as specified by template in javascript (aka flow) [flow](flow/README.md)

5
v2/pkg/tmplexec/doc.go Normal file
View File

@ -0,0 +1,5 @@
package tmplexec
// tmplexec is package that provides
// template executors it is one level higher than protocols
// and deals with execution of entire template

165
v2/pkg/tmplexec/exec.go Normal file
View File

@ -0,0 +1,165 @@
package tmplexec
import (
"errors"
"fmt"
"strings"
"sync/atomic"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/multiproto"
)
// TemplateExecutor is an executor for a template
type TemplateExecuter struct {
requests []protocols.Request
options *protocols.ExecutorOptions
engine TemplateEngine
results *atomic.Bool
}
// Both executer & Executor are correct spellings (its open to interpretation)
var _ protocols.Executer = &TemplateExecuter{}
// NewTemplateExecuter creates a new request TemplateExecuter for list of requests
func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *TemplateExecuter {
isMultiProto := false
lastProto := ""
for _, request := range requests {
if request.Type().String() != lastProto && lastProto != "" {
isMultiProto = true
break
}
lastProto = request.Type().String()
}
e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}}
if options.Flow != "" {
// we use a dummy input here because goal of flow executor at this point is to just check
// syntax and other things are correct before proceeding to actual execution
// during execution new instance of flow will be created as it is tightly coupled with lot of executor options
e.engine = flow.NewFlowExecutor(requests, contextargs.NewWithInput("dummy"), options, e.results)
} else {
// Review:
// multiproto engine is only used if there is more than one protocol in template
// else we use generic engine (should we use multiproto engine for single protocol with multiple requests as well ?)
if isMultiProto {
e.engine = multiproto.NewMultiProtocol(requests, options, e.results)
} else {
e.engine = generic.NewGenericEngine(requests, options, e.results)
}
}
return e
}
// Compile compiles the execution generators preparing any requests possible.
func (e *TemplateExecuter) Compile() error {
cliOptions := e.options.Options
for _, request := range e.requests {
if err := request.Compile(e.options); err != nil {
var dslCompilationError *dsl.CompilationError
if errors.As(err, &dslCompilationError) {
if cliOptions.Verbose {
rawErrorMessage := dslCompilationError.Error()
formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "."
gologger.Warning().Msgf(formattedErrorMessage)
gologger.Info().Msgf("The available custom DSL functions are:")
fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor))
}
}
return err
}
}
return e.engine.Compile()
}
// Requests returns the total number of requests the rule will perform
func (e *TemplateExecuter) Requests() int {
var count int
for _, request := range e.requests {
count += request.Requests()
}
return count
}
// Execute executes the protocol group and returns true or false if results were found.
func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) {
results := &atomic.Bool{}
defer func() {
// it is essential to remove template context of `Scan i.e template x input pair`
// since it is of no use after scan is completed (regardless of success or failure)
e.options.RemoveTemplateCtx(input.MetaInput)
}()
var lastMatcherEvent *output.InternalWrappedEvent
writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) {
if !results.Load() && matcherStatus {
if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil {
gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
}
results.CompareAndSwap(false, true)
}
}
cliExecutorCallback := func(event *output.InternalWrappedEvent) {
if event == nil {
// something went wrong
return
}
// If no results were found, and also interactsh is not being used
// in that case we can skip it, otherwise we've to show failure in
// case of matcher-status flag.
if !event.HasOperatorResult() && !event.UsesInteractsh {
lastMatcherEvent = event
} else {
if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) {
results.CompareAndSwap(false, true)
} else {
lastMatcherEvent = event
}
}
}
var err error
// Note: this is required for flow executor
// flow executer is tightly coupled with lot of executor options
// and map , wg and other types earlier we tried to use (compile once and run multiple times)
// but it is causing lot of panic and nil pointer dereference issues
// so in compile step earlier we compile it to validate javascript syntax and other things
// and while executing we create new instance of flow executor everytime
if e.options.Flow != "" {
flowexec := flow.NewFlowExecutor(e.requests, input, e.options, results)
if err := flowexec.Compile(); err != nil {
return false, err
}
err = flowexec.ExecuteWithResults(input, cliExecutorCallback)
} else {
err = e.engine.ExecuteWithResults(input, cliExecutorCallback)
}
if lastMatcherEvent != nil {
writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus)
}
return results.Load(), err
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
gologger.Info().Msgf("[%s] Running on %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint())
userCallback := func(event *output.InternalWrappedEvent) {
if event != nil {
callback(event)
}
}
return e.engine.ExecuteWithResults(input, userCallback)
}

319
v2/pkg/tmplexec/flow/README.md Normal file
View File

@ -0,0 +1,319 @@
# flow
flow is a new template engine/backend introduced in v3 which primarily adds 2 most awaited features
- conditional execution of requests (ex: `flow: dns() && http()`)
- request execution orchestration (iterate over a slice, request execution order, if/for statement)
both of these features are implemented using javascript (ECMAScript 5.1) with [goja](https://github.com/dop251/goja) backend.
## conditional execution
Many times when writing complex templates we might need to add some extra checks (or conditional statements) before executing certain part of request
An ideal example of this would be when bruteforcing wordpress login with default usernames and passwords. If we try to write a template for this it would be something like this
```yaml
id: wordpress-bruteforce
info:
name: WordPress Login Bruteforce
author: pdteam
severity: high
http:
- method: POST
path:
- "{{BaseURL}}/wp-login.php"
payloads:
username:
- admin
- guest
- testuser
password:
- password123
- qwertyuiop
- letmein
body: "log=§username§&pwd=§password§&wp-submit=Log+In"
attack: clusterbomb
matchers:
- type: word
words:
- "ERROR"
part: body
negative: true
```
but if we carefully re-evaluate this template, we can see that template is sending 9 requests without even checking, if the url actually exists or target site is actually a wordpress site. before v3 it was possible to do this by adding a extractor and sending additional content in say url fragment and it would fail if request was not successful and another way would be writing a workflow (2 templates and 1 workflow file i.e total 3 files for 1 template) but this is `hacky` and not a good solution.
With addition of flow in Nuclei v3 we can re-write this template to first check if target is a wordpress site, if yes then bruteforce login with default credentials and this can be achieved by simply adding one line of content i.e `flow: http("check-wp") && http("bruteforce")` and nuclei will take care of everything else.
```yaml
id: wordpress-bruteforce
info:
name: WordPress Login Bruteforce
author: pdteam
severity: high
flow: http("check-wp") && http("bruteforce")
http:
- id: check-wp
method: GET
path:
- "{{BaseURL}}/wp-login.php"
matchers:
- type: word
words:
- "WordPress"
part: body
- type: word
words:
- "wp-content"
part: body
matchers-condition: and
- id: bruteforce
method: POST
path:
- "{{BaseURL}}/wp-login.php"
payloads:
username:
- admin
- guest
- testuser
password:
- password123
- qwertyuiop
- letmein
body: "log=§username§&pwd=§password§&wp-submit=Log+In"
attack: clusterbomb
matchers:
- type: word
words:
- "ERROR"
part: body
negative: true
```
**Note:** this is just a example template with poor matchers. refer 'nuclei-templates' repo for final template
The update template now seems straight forward and easy to understand. we are first checking if target is a wordpress site and then executing bruteforce requests. This is just a simple example of conditional execution and flow accepts any Javascript (ECMAScript 5.1) expression/code so you are free to craft any conditional execution logic you want using for,if and whatnot.
## request execution orchestration
`conditional execution` is one simple use case of flow but `flow` is much more powerful than that, for example it can be used to
- iterate over a slice of values and execute requests for each value (ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml))
- extract values from one request and iterate over them and execute requests for each value ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml)
- get/set values from/to template context (global variables)
- print/log values to stdout at xyz condition or while debugging
- adding custom logic during template execution (ex: if status code is 403 => login and then re-run it)
- use any/all ECMAScript 5.1 javascript (like objects,arrays etc) and build/transform variables/input at runtime
- update variables at runtime (ex: when jwt expires update it by using refresh token and then continue execution)
- and a lot more (this is just a tip of iceberg)
simply put request execution orchestration can be understood as nuclei logic bindings for javascript (i.e two way interaction between javascript and nuclei for a specific template)
To better understand orchestration we can try to build a template for vhost enumeration using flow. which usually requires writing / using a new tool
**for basic vhost enumeration a template should**
- do a PTR lookup for given ip
- get SSL ceritificate for given ip (i.e tls-grab)
- extract subject_cn from certificate
- extract subject_alt_names(SAN) from certificate
- filter out wildcard prefix from above values
- and finally bruteforce all found vhosts
**Now if we try to implement this in template it would be**
```yaml
# send a ssl request to get certificate
ssl:
- address: "{{Host}}:{{Port}}"
# do a PTR lookup for given ip and get PTR value
dns:
- name: "{{FQDN}}"
type: PTR
matchers:
- type: word
words:
- "IN\tPTR"
extractors:
- type: regex
name: ptrValue
internal: true
group: 1
regex:
- "IN\tPTR\t(.+)"
# bruteforce all found vhosts
http:
- raw:
- |
GET / HTTP/1.1
Host: {{vhost}}
matchers:
- type: status
negative: true
status:
- 400
- 502
extractors:
- type: dsl
dsl:
- '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length' tarun@macbook:~/Codebase/nuclei/integration_tes
```
**But this template is not yet ready as it is missing core logic i.e how we use all these obtained data and do bruteforce**
and this is where flow comes into picture. flow is javascript code with two way bindings to nuclei. if we write javascript code to orchestrate vhost enumeration it is as simple as
```javascript
ssl();
dns();
for (let vhost of iterate(template["ssl_subject_cn"],template["ssl_subject_an"])) {
set("vhost", vhost);
http(); }
```
With just extra 5 lines of javascript code template can now perform vhost enumeration and this can be run on scale with all awesome features of nuclei with various supported inputs like ASN,CIDR,URL etc
In above Js code we are using some Nuclei JS bindings lets understand what they do
- `ssl()` => execute ssl request
- `dns()` => execute dns request
- `template["ssl_subject_cn"]` => get value of `ssl_subject_cn` from template context (global variables)
- `iterate()` => this is a nuclei helper function which iterates any type of value (array,map,string,number) while handling empty / nil values
- `set("vhost",vhost)` => creates new variable `vhost` in template and assigns value of `vhost` to it
- `http()` => execute http request
This template is now missing one last thing i.e
- removing wildcard prefix (*.) in subject_cn,subject_an
- trailing `.` in PTR value
and this can be done using either JS methods of using DSL helper functions as shown in below template
```yaml
id: vhost-enum-flow
info:
name: vhost enum flow
author: tarunKoyalwar
severity: info
description: |
vhost enumeration by extracting potential vhost names from ssl certificate and dns ptr records
flow: |
ssl();
dns({hide: true});
for (let vhost of iterate(template["ssl_subject_cn"],template["ssl_subject_an"])) {
vhost = vhost.replace("*.", "")
set("vhost", vhost);
http();
}
ssl:
- address: "{{Host}}:{{Port}}"
dns:
- name: "{{FQDN}}"
type: PTR
matchers:
- type: word
words:
- "IN\tPTR"
extractors:
- type: regex
name: ptrValue
internal: true
group: 1
regex:
- "IN\tPTR\t(.+)"
http:
- raw:
- |
GET / HTTP/1.1
Host: {{trim_suffix(vhost, ".")}}
matchers:
- type: status
negative: true
status:
- 400
- 502
extractors:
- type: dsl
dsl:
- '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length'
```
### Nuclei JS Bindings
This section contains a brief description of all nuclei JS bindings and their usage
**1. Protocol Execution Functions**
Any protocol that is present in a nuclei template can be called/executed in javascript in format `proto_name()` i.e `http()` , `dns()` , `ssl()` etc
If we want to execute a specific request of a protocol (ref: see [nuclei-flow-dns](testcases/nuclei-flow-dns-id.yaml)) this can be achieved by either passing
- index of that request in protocol (ex: `dns(0)`, `dns(1)` etc)
- id of that request in protocol (ex: `dns("extract-vps")`, `dns("probe-http")` etc)
For More complex use cases multiple requests of a single protocol can be executed by just specifying their index or id one after another (ex: `dns("extract-vps","1")`)
**2. Iterate Helper Function**
Iterate is a nuclei js helper function which can be used to iterate over any type of value (array,map,string,number) while handling empty / nil values.
This is addon helper function from nuclei to omit boilerplate code of checking if value is empty or not and then iterating over it
```javascript
iterate(123,{"a":1,"b":2,"c":3})
// iterate over array with custom separator
iterate([1,2,3,4,5], " ")
```
**Note:** In above example we used `iterate(template["ssl_subject_cn"],template["ssl_subject_an"])` which removed lot of boilerplate code of checking if value is empty or not and then iterating over it
**3. Set Helper Function**
When Iterating over a values/array or some other use case we might want to invoke a request with custom/given value and this can be achieved by using `set()` helper function. When invoked/called it adds given variable to template context (global variables) and that value is used during execution of request/protocol. the format of `set()` is `set("variable_name",value)` ex: `set("username","admin")` etc
```javascript
for (let vhost of myArray) {
set("vhost", vhost);
http(1)
}
```
**Note:** In above example we used `set("vhost", vhost)` which added `vhost` to template context (global variables) and then called `http(1)` which used this value in request
**4. Template Context**
when using `nuclei -jsonl` flag we get lot of data/metadata related to a vulnerability (ex: template details,extracted-values and much more) . A template context is nothing but a map/JSON containing all this data along with internal/unexported data that is only available at runtime (ex: extracted values from previous requests, variables added using `set()` etc). This template context is available in javascript as `template` variable and can be used to access any data from it. ex: `template["ssl_subject_cn"]` , `template["ssl_subject_an"]` etc
```javascript
template["ssl_subject_cn"] // returns value of ssl_subject_cn from template context which is available after executing ssl request
template["ptrValue"] // returns value of ptrValue which was extracted using regex with internal: true
```
Lot of times we don't known what all data is available in template context and this can be easily found by printing it to stdout using `log()` function
```javascript
log(template)
```
**5. Log Helper Function**
It is a nuclei js alternative to `console.log` and this pretty prints map data in readable format
**Note:** This should be used for debugging purposed only as this prints data to stdout
**6. Dedupe**
Lot of times just having arrays/slices is not enough and we might need to remove duplicate variables . for example in earlier vhost enumeration we did not remove any duplicates as there is always a chance of duplicate values in `ssl_subject_cn` and `ssl_subject_an` and this can be achieved by using `dedupe()` object. This is nuclei js helper function to abstract away boilerplate code of removing duplicates from array/slice
```javascript
let uniq = new Dedupe(); // create new dedupe object
uniq.Add(template["ptrValue"])
uniq.Add(template["ssl_subject_cn"]);
uniq.Add(template["ssl_subject_an"]);
log(uniq.Values())
```
And that's it , this automatically converts any slice/array to map and removes duplicates from it and returns a slice/array of unique values
------
> Similar to DSL helper functions . we can either use built in functions available with `Javscript (ECMAScript 5.1)` or use DSL helper functions and its upto user to decide which one to uses

67
v2/pkg/tmplexec/flow/builtin/dedupe.go Normal file
View File

@ -0,0 +1,67 @@
package builtin
import (
"crypto/md5"
"reflect"
"github.com/dop251/goja"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Dedupe is a javascript builtin for deduping values
type Dedupe struct {
m map[string]goja.Value
VM *goja.Runtime
}
// Add adds a value to the dedupe
func (d *Dedupe) Add(call goja.FunctionCall) goja.Value {
allVars := []any{}
for _, v := range call.Arguments {
if v.Export() == nil {
continue
}
if v.ExportType().Kind() == reflect.Slice {
// convert []datatype to []interface{}
// since it cannot be type asserted to []interface{} directly
rfValue := reflect.ValueOf(v.Export())
for i := 0; i < rfValue.Len(); i++ {
allVars = append(allVars, rfValue.Index(i).Interface())
}
} else {
allVars = append(allVars, v.Export())
}
}
for _, v := range allVars {
hash := hashValue(v)
if _, ok := d.m[hash]; ok {
continue
}
d.m[hash] = d.VM.ToValue(v)
}
return d.VM.ToValue(true)
}
// Values returns all values from the dedupe
func (d *Dedupe) Values(call goja.FunctionCall) goja.Value {
tmp := []goja.Value{}
for _, v := range d.m {
tmp = append(tmp, v)
}
return d.VM.ToValue(tmp)
}
// NewDedupe creates a new dedupe builtin object
func NewDedupe(vm *goja.Runtime) *Dedupe {
return &Dedupe{
m: make(map[string]goja.Value),
VM: vm,
}
}
// hashValue returns a hash of the value
func hashValue(value interface{}) string {
res := types.ToString(value)
md5sum := md5.Sum([]byte(res))
return string(md5sum[:])
}

1
v2/pkg/tmplexec/flow/doc.go Normal file
View File

@ -0,0 +1 @@
package flow

240
v2/pkg/tmplexec/flow/flow_executor.go Normal file
View File

@ -0,0 +1,240 @@
package flow
import (
"fmt"
"io"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/dop251/goja"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
mapsutil "github.com/projectdiscovery/utils/maps"
"go.uber.org/multierr"
)
var (
// ErrInvalidRequestID is a request id error
ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided")
)
// FlowExecutor is a flow executor for executing a flow
type FlowExecutor struct {
input *contextargs.Context
options *protocols.ExecutorOptions
// javascript runtime reference and compiled program
jsVM *goja.Runtime
program *goja.Program // compiled js program
// protocol requests and their callback functions
allProtocols map[string][]protocols.Request
protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js
callback func(event *output.InternalWrappedEvent) // result event callback
// logic related variables
wg sync.WaitGroup
results *atomic.Bool
allErrs mapsutil.SyncLockMap[string, error]
}
// NewFlowExecutor creates a new flow executor from a list of requests
func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor {
allprotos := make(map[string][]protocols.Request)
for _, req := range requests {
switch req.Type() {
case templateTypes.DNSProtocol:
allprotos[templateTypes.DNSProtocol.String()] = append(allprotos[templateTypes.DNSProtocol.String()], req)
case templateTypes.HTTPProtocol:
allprotos[templateTypes.HTTPProtocol.String()] = append(allprotos[templateTypes.HTTPProtocol.String()], req)
case templateTypes.NetworkProtocol:
allprotos[templateTypes.NetworkProtocol.String()] = append(allprotos[templateTypes.NetworkProtocol.String()], req)
case templateTypes.FileProtocol:
allprotos[templateTypes.FileProtocol.String()] = append(allprotos[templateTypes.FileProtocol.String()], req)
case templateTypes.HeadlessProtocol:
allprotos[templateTypes.HeadlessProtocol.String()] = append(allprotos[templateTypes.HeadlessProtocol.String()], req)
case templateTypes.SSLProtocol:
allprotos[templateTypes.SSLProtocol.String()] = append(allprotos[templateTypes.SSLProtocol.String()], req)
case templateTypes.WebsocketProtocol:
allprotos[templateTypes.WebsocketProtocol.String()] = append(allprotos[templateTypes.WebsocketProtocol.String()], req)
case templateTypes.WHOISProtocol:
allprotos[templateTypes.WHOISProtocol.String()] = append(allprotos[templateTypes.WHOISProtocol.String()], req)
case templateTypes.CodeProtocol:
allprotos[templateTypes.CodeProtocol.String()] = append(allprotos[templateTypes.CodeProtocol.String()], req)
default:
gologger.Error().Msgf("invalid request type %s", req.Type().String())
}
}
f := &FlowExecutor{
allProtocols: allprotos,
options: options,
allErrs: mapsutil.SyncLockMap[string, error]{
ReadOnly: atomic.Bool{},
Map: make(map[string]error),
},
protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{},
results: results,
jsVM: goja.New(),
input: input,
}
return f
}
// Compile compiles js program and registers all functions
func (f *FlowExecutor) Compile() error {
if f.results == nil {
f.results = new(atomic.Bool)
}
// load all variables and evaluate with existing data
variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll())
// cli options
optionVars := generators.BuildPayloadFromOptions(f.options.Options)
// constants
constants := f.options.Constants
allVars := generators.MergeMaps(variableMap, constants, optionVars)
// we support loading variables from files in variables , cli options and constants
// try to load if files exist
for k, v := range allVars {
if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) {
if value, err := f.ReadDataFromFile(str); err == nil {
allVars[k] = value
} else {
gologger.Warning().Msgf("could not load file '%s' for variable '%s': %s", str, k, err)
}
}
}
f.options.GetTemplateCtx(f.input.MetaInput).Merge(allVars) // merge all variables into template context
// ---- define callback functions/objects----
f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{}
// iterate over all protocols and generate callback functions for each protocol
for p, requests := range f.allProtocols {
// for each protocol build a requestMap with reqID and protocol request
reqMap := mapsutil.Map[string, protocols.Request]{}
counter := 0
proto := strings.ToLower(p) // donot use loop variables in callback functions directly
for index := range requests {
request := f.allProtocols[proto][index]
if request.GetID() != "" {
// if id is present use it
reqMap[request.GetID()] = request
}
// fallback to using index as id
// always allow index as id as a fallback
reqMap[strconv.Itoa(counter)] = request
counter++
}
// ---define hook that allows protocol/request execution from js-----
// --- this is the actual callback that is executed when function is invoked in js----
f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value {
opts := &ProtoOptions{
protoName: proto,
}
for _, v := range call.Arguments {
switch value := v.Export().(type) {
case map[string]interface{}:
opts.LoadOptions(value)
default:
opts.reqIDS = append(opts.reqIDS, types.ToString(value))
}
}
// parallel execution of protocols
if opts.Async {
f.wg.Add(1)
go func() {
defer f.wg.Done()
f.requestExecutor(reqMap, opts)
}()
return f.jsVM.ToValue(true)
}
return f.jsVM.ToValue(f.requestExecutor(reqMap, opts))
}
}
return f.registerBuiltInFunctions()
}
// ExecuteWithResults executes the flow and returns results
func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
defer func() {
if e := recover(); e != nil {
gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", input.MetaInput.Input, e)
panic(e)
}
}()
f.callback = callback
f.input = input
// -----Load all types of variables-----
// add all input args to template context
if f.input != nil && f.input.HasArgs() {
f.input.ForEach(func(key string, value interface{}) {
f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value)
})
}
if f.callback == nil {
return fmt.Errorf("output callback cannot be nil")
}
// pass flow and execute the js vm and handle errors
value, err := f.jsVM.RunProgram(f.program)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow)
}
f.wg.Wait()
runtimeErr := f.GetRuntimeErrors()
if runtimeErr != nil {
return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow")
}
if value.Export() != nil {
f.results.Store(value.ToBoolean())
} else {
f.results.Store(true)
}
return nil
}
// GetRuntimeErrors returns all runtime errors (i.e errors from all protocol combined)
func (f *FlowExecutor) GetRuntimeErrors() error {
errs := []error{}
for proto, err := range f.allErrs.GetAll() {
errs = append(errs, errorutil.NewWithErr(err).Msgf("failed to execute %v protocol", proto))
}
return multierr.Combine(errs...)
}
// ReadDataFromFile reads data from file respecting sandbox options
func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) {
values := []string{}
// load file respecting sandbox
reader, err := f.options.Options.LoadHelperFile(payload, f.options.TemplatePath, f.options.Catalog)
if err != nil {
return values, err
}
defer reader.Close()
bin, err := io.ReadAll(reader)
if err != nil {
return values, err
}
for _, line := range strings.Split(string(bin), "\n") {
line = strings.TrimSpace(line)
if line != "" {
values = append(values, line)
}
}
return values, nil
}
// Name returns the type of engine
func (f *FlowExecutor) Name() string {
return "flow"
}

173
v2/pkg/tmplexec/flow/flow_executor_test.go Normal file
View File

@ -0,0 +1,173 @@
package flow_test
import (
"context"
"log"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/projectdiscovery/ratelimit"
"github.com/stretchr/testify/require"
)
var executerOpts protocols.ExecutorOptions
func setup() {
options := testutils.DefaultOptions
testutils.Init(options)
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
executerOpts = protocols.ExecutorOptions{
Output: testutils.NewMockOutputWriter(),
Options: options,
Progress: progressImpl,
ProjectFile: nil,
IssuesClient: nil,
Browser: nil,
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
}
workflowLoader, err := parsers.NewLoader(&executerOpts)
if err != nil {
log.Fatalf("Could not create workflow loader: %s\n", err)
}
executerOpts.WorkflowLoader = workflowLoader
}
func TestFlowTemplateWithIndex(t *testing.T) {
// test
setup()
Template, err := templates.Parse("testcases/nuclei-flow-dns.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
input := contextargs.NewWithInput("hackerone.com")
gotresults, err := Template.Executer.Execute(input)
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}
func TestFlowTemplateWithID(t *testing.T) {
setup()
// apart from parse->compile->execution this testcase checks support for use custom id for protocol request and invocation of
// the same in js
Template, err := templates.Parse("testcases/nuclei-flow-dns-id.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
target := contextargs.NewWithInput("hackerone.com")
gotresults, err := Template.Executer.Execute(target)
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}
func TestFlowWithProtoPrefix(t *testing.T) {
// test
setup()
// apart from parse->compile->execution this testcase checks
// mix of custom protocol request id and index is supported in js
// and also validates availability of protocol response variables in template context
Template, err := templates.Parse("testcases/nuclei-flow-dns-prefix.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
input := contextargs.NewWithInput("hackerone.com")
gotresults, err := Template.Executer.Execute(input)
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}
func TestFlowWithConditionNegative(t *testing.T) {
setup()
// apart from parse->compile->execution this testcase checks
// if bitwise operator (&&) are properly executed and working
Template, err := templates.Parse("testcases/condition-flow.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
input := contextargs.NewWithInput("scanme.sh")
// expect no results and verify thant dns request is executed and http is not
gotresults, err := Template.Executer.Execute(input)
require.Nil(t, err, "could not execute template")
require.False(t, gotresults)
}
func TestFlowWithConditionPositive(t *testing.T) {
setup()
// apart from parse->compile->execution this testcase checks
// if bitwise operator (&&) are properly executed and working
Template, err := templates.Parse("testcases/condition-flow.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
input := contextargs.NewWithInput("blog.projectdiscovery.io")
// positive match . expect results also verify that both dns() and http() were executed
gotresults, err := Template.Executer.Execute(input)
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}
func TestFlowWithNoMatchers(t *testing.T) {
// when using conditional flow with no matchers at all
// we implicitly assume that request was successful and internally changed the result to true (for scope of condition only)
// testcase-1 : no matchers but contains extractor
Template, err := templates.Parse("testcases/condition-flow-extractors.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
// positive match . expect results also verify that both dns() and http() were executed
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
// testcase-2 : no matchers and no extractors
Template, err = templates.Parse("testcases/condition-flow-no-operators.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
// positive match . expect results also verify that both dns() and http() were executed
gotresults, err = Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}

217
v2/pkg/tmplexec/flow/flow_internal.go Normal file
View File

@ -0,0 +1,217 @@
package flow
import (
"reflect"
"sync/atomic"
"github.com/dop251/goja"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow/builtin"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
mapsutil "github.com/projectdiscovery/utils/maps"
)
// contains all internal/unexported methods of flow
// requestExecutor executes a protocol/request and returns true if any matcher was found
func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool {
defer func() {
// evaluate all variables after execution of each protocol
variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll())
f.options.GetTemplateCtx(f.input.MetaInput).Merge(variableMap) // merge all variables into template context
// to avoid polling update template variables everytime we execute a protocol
var m map[string]interface{} = f.options.GetTemplateCtx(f.input.MetaInput).GetAll()
_ = f.jsVM.Set("template", m)
}()
matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool
// if no id is passed execute all requests in sequence
if len(opts.reqIDS) == 0 {
// execution logic for http()/dns() etc
for index := range f.allProtocols[opts.protoName] {
req := f.allProtocols[opts.protoName][index]
err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) {
if result != nil {
f.results.CompareAndSwap(false, true)
if !opts.Hide {
f.callback(result)
}
// export dynamic values from operators (i.e internal:true)
// add add it to template context
// this is a conflicting behaviour with iterate-all
if result.HasOperatorResult() {
matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched)
if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) {
// if matcher status is false . check if template/request contains any matcher at all
// if it does then we need to set matcher status to true
matcherStatus.CompareAndSwap(false, true)
}
if len(result.OperatorsResult.DynamicValues) > 0 {
for k, v := range result.OperatorsResult.DynamicValues {
f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v)
}
}
} else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) {
// if matcher status is false . check if template/request contains any matcher at all
// if it does then we need to set matcher status to true
matcherStatus.CompareAndSwap(false, true)
}
}
})
if err != nil {
// save all errors in a map with id as key
// its less likely that there will be race condition but just in case
id := req.GetID()
if id == "" {
id, _ = reqMap.GetKeyWithValue(req)
}
err = f.allErrs.Set(opts.protoName+":"+id, err)
if err != nil {
gologger.Error().Msgf("failed to store flow runtime errors got %v", err)
}
return matcherStatus.Load()
}
}
return matcherStatus.Load()
}
// execution logic for http("0") or http("get-aws-vpcs")
for _, id := range opts.reqIDS {
req, ok := reqMap[id]
if !ok {
gologger.Error().Msgf("invalid request id '%s' provided", id)
// compile error
if err := f.allErrs.Set(opts.protoName+":"+id, ErrInvalidRequestID.Msgf(id)); err != nil {
gologger.Error().Msgf("failed to store flow runtime errors got %v", err)
}
return matcherStatus.Load()
}
err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) {
if result != nil {
f.results.CompareAndSwap(false, true)
if !opts.Hide {
f.callback(result)
}
// export dynamic values from operators (i.e internal:true)
// add add it to template context
if result.HasOperatorResult() {
matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched)
if len(result.OperatorsResult.DynamicValues) > 0 {
for k, v := range result.OperatorsResult.DynamicValues {
f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v)
}
_ = f.jsVM.Set("template", f.options.GetTemplateCtx(f.input.MetaInput).GetAll())
}
}
}
})
if err != nil {
index := id
err = f.allErrs.Set(opts.protoName+":"+index, err)
if err != nil {
gologger.Error().Msgf("failed to store flow runtime errors got %v", err)
}
}
}
return matcherStatus.Load()
}
// registerBuiltInFunctions registers all built in functions for the flow
func (f *FlowExecutor) registerBuiltInFunctions() error {
// currently we register following builtin functions
// log -> log to stdout with [JS] prefix should only be used for debugging
// set -> set a variable in template context
// proto(arg ...String) <- this is generic syntax of how a protocol/request binding looks in js
// we only register only those protocols that are available in template
// we also register a map datatype called template with all template variables
// template -> all template variables are available in js template object
if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value {
// TODO: verify string interpolation and handle multiple args
arg := call.Argument(0).Export()
switch value := arg.(type) {
case string:
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
case map[string]interface{}:
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value))
default:
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
}
return goja.Null()
}); err != nil {
return err
}
if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value {
varName := call.Argument(0).Export()
varValue := call.Argument(1).Export()
f.options.GetTemplateCtx(f.input.MetaInput).Set(types.ToString(varName), varValue)
return goja.Null()
}); err != nil {
return err
}
// iterate provides global iterator function by handling null values or strings
if err := f.jsVM.Set("iterate", func(call goja.FunctionCall) goja.Value {
allVars := []any{}
for _, v := range call.Arguments {
if v.Export() == nil {
continue
}
if v.ExportType().Kind() == reflect.Slice {
// convert []datatype to []interface{}
// since it cannot be type asserted to []interface{} directly
rfValue := reflect.ValueOf(v.Export())
for i := 0; i < rfValue.Len(); i++ {
allVars = append(allVars, rfValue.Index(i).Interface())
}
} else {
allVars = append(allVars, v.Export())
}
}
return f.jsVM.ToValue(allVars)
}); err != nil {
return err
}
// add a builtin dedupe object
if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object {
d := builtin.NewDedupe(f.jsVM)
obj := call.This
// register these methods
_ = obj.Set("Add", d.Add)
_ = obj.Set("Values", d.Values)
return nil
}); err != nil {
return err
}
var m = f.options.GetTemplateCtx(f.input.MetaInput).GetAll()
if m == nil {
m = map[string]interface{}{}
}
if err := f.jsVM.Set("template", m); err != nil {
// all template variables are available in js template object
return err
}
// register all protocols
for name, fn := range f.protoFunctions {
if err := f.jsVM.Set(name, fn); err != nil {
return err
}
}
program, err := goja.Compile("flow", f.options.Flow, false)
if err != nil {
return err
}
f.program = program
return nil
}

48
v2/pkg/tmplexec/flow/options.go Normal file
View File

@ -0,0 +1,48 @@
package flow
import (
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// ProtoOptions are options that can be passed to flow protocol callback
// ex: dns(protoOptions) <- protoOptions are optional and can be anything
type ProtoOptions struct {
Hide bool
Async bool
protoName string
reqIDS []string
}
// Examples
// dns() <- callback without any options
// dns(1) or dns(1,3) <- callback with index of protocol in template request at 1 or 1 and 3
// dns("probe-http") or dns("extract-vpc","probe-http") <- callback with id's instead of index of request in template
// dns({hide:true}) or dns({hide:true,async:true}) <- callback with protocol options
// hide - hides result/event from output & sdk
// async - executes protocols in parallel (implicit wait no need to specify wait)
// Note: all of these options are optional and can be combined together in any order
// LoadOptions loads the protocol options from a map
func (P *ProtoOptions) LoadOptions(m map[string]interface{}) {
P.Hide = GetBool(m["hide"])
P.Async = GetBool(m["async"])
}
// GetBool returns bool value from interface
func GetBool(value interface{}) bool {
if value == nil {
return false
}
switch v := value.(type) {
case bool:
return v
default:
tmpValue := types.ToString(value)
if strings.EqualFold(tmpValue, "true") {
return true
}
}
return false
}

29
v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml Normal file
View File

@ -0,0 +1,29 @@
id: ghost-blog-detection
info:
name: Ghost blog detection
author: pdteam
severity: info
flow: dns() && http()
dns:
- name: "{{FQDN}}"
type: CNAME
extractors:
- type: dsl
name: cname
internal: true
dsl:
- cname
http:
- method: GET
path:
- "{{BaseURL}}?ref={{cname}}"
matchers:
- type: word
words:
- "ghost.io"

23
v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml Normal file
View File

@ -0,0 +1,23 @@
id: ghost-blog-detection
info:
name: Ghost blog detection
author: pdteam
severity: info
flow: dns() && http()
dns:
- name: "{{FQDN}}"
type: CNAME
http:
- method: GET
path:
- "{{BaseURL}}?ref={{dns_cname}}"
matchers:
- type: word
words:
- "ghost.io"

27
v2/pkg/tmplexec/flow/testcases/condition-flow.yaml Normal file
View File

@ -0,0 +1,27 @@
id: ghost-blog-detection
info:
name: Ghost blog detection
author: pdteam
severity: info
flow: dns() && http()
dns:
- name: "{{FQDN}}"
type: CNAME
matchers:
- type: word
words:
- "ghost.io"
http:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "ghost.io"

42
v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml Normal file
View File

@ -0,0 +1,42 @@
id: nuclei-flow-dns
info:
name: Nuclei flow dns
author: pdteam
severity: info
description: Description of the Template
reference: https://example-reference-link
flow: |
dns("fetch-ns");
template["nameservers"].forEach(nameserver => {
set("nameserver",nameserver);
dns("probe-ns");
});
dns:
- id: "fetch-ns"
name: "{{FQDN}}"
type: NS
matchers:
- type: word
words:
- "IN\tNS"
extractors:
- type: regex
internal: true
name: "nameservers"
group: 1
regex:
- "IN\tNS\t(.+)"
- id: "probe-ns"
name: "{{nameserver}}"
type: A
class: inet
retries: 3
recursion: true
extractors:
- type: dsl
dsl:
- "a"

41
v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml Normal file
View File

@ -0,0 +1,41 @@
id: nuclei-flow-dns
info:
name: Nuclei flow dns
author: pdteam
severity: info
description: Description of the Template
reference: https://example-reference-link
flow: |
dns("0");
template["nameservers"].forEach(nameserver => {
set("nameserver",nameserver);
dns("probe-ns");
});
dns:
- name: "{{FQDN}}"
type: NS
matchers:
- type: word
words:
- "IN\tNS"
extractors:
- type: regex
internal: true
name: "nameservers"
group: 1
regex:
- "IN\tNS\t(.+)"
- id: "probe-ns"
name: "{{nameserver}}"
type: A
class: inet
retries: 3
recursion: true
extractors:
- type: dsl
dsl:
- "a"

40
v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml Normal file
View File

@ -0,0 +1,40 @@
id: nuclei-flow-dns
info:
name: Nuclei flow dns
author: pdteam
severity: info
description: Description of the Template
reference: https://example-reference-link
flow: |
dns("0");
template["nameservers"].forEach(nameserver => {
set("nameserver",nameserver);
dns("1");
});
dns:
- name: "{{FQDN}}"
type: NS
matchers:
- type: word
words:
- "IN\tNS"
extractors:
- type: regex
internal: true
name: "nameservers"
group: 1
regex:
- "IN\tNS\t(.+)"
- name: "{{nameserver}}"
type: A
class: inet
retries: 3
recursion: true
extractors:
- type: dsl
dsl:
- "a"

23
v2/pkg/tmplexec/flow/util.go Normal file
View File

@ -0,0 +1,23 @@
package flow
import "github.com/projectdiscovery/nuclei/v2/pkg/operators"
// Checks if template has matchers
func hasMatchers(all []*operators.Operators) bool {
for _, operator := range all {
if len(operator.Matchers) > 0 {
return true
}
}
return false
}
// hasOperators checks if template has operators (i.e matchers/extractors)
func hasOperators(all []*operators.Operators) bool {
for _, operator := range all {
if operator != nil {
return true
}
}
return false
}

94
v2/pkg/tmplexec/generic/exec.go Normal file
View File

@ -0,0 +1,94 @@
package generic
import (
"strings"
"sync/atomic"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
)
// generic engine as name suggests is a generic template
// execution engine and executes all requests one after another
// without any logic in between
type Generic struct {
requests []protocols.Request
options *protocols.ExecutorOptions
results *atomic.Bool
}
// NewGenericEngine creates a new generic engine from a list of requests
func NewGenericEngine(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *Generic {
if results == nil {
results = &atomic.Bool{}
}
return &Generic{requests: requests, options: options, results: results}
}
// Compile engine specific compilation
func (g *Generic) Compile() error {
// protocol/ request is already handled by template executer
return nil
}
// ExecuteWithResults executes the template and returns results
func (g *Generic) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
dynamicValues[key] = value
})
}
previous := make(map[string]interface{})
for _, req := range g.requests {
inputItem := input.Clone()
if g.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.MetaInput.Input = g.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
return nil
}
}
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
if event == nil {
// ideally this should never happen since protocol exits on error and callback is not called
return
}
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
for k, v := range event.InternalEvent {
builder.WriteString(ID)
builder.WriteString("_")
builder.WriteString(k)
previous[builder.String()] = v
builder.Reset()
}
}
if event.HasOperatorResult() {
g.results.CompareAndSwap(false, true)
}
// for ExecuteWithResults : this callback will execute user defined callback and some error handling
// for Execute : this callback will print the result to output
callback(event)
})
if err != nil {
if g.options.HostErrorsCache != nil {
g.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, input.MetaInput.PrettyPrint(), err)
}
// If a match was found and stop at first match is set, break out of the loop and return
if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) {
break
}
}
return nil
}
// Type returns the type of engine
func (g *Generic) Name() string {
return "generic"
}

33
v2/pkg/tmplexec/interface.go Normal file
View File

@ -0,0 +1,33 @@
package tmplexec
import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic"
"github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/multiproto"
)
var (
_ TemplateEngine = &generic.Generic{}
_ TemplateEngine = &flow.FlowExecutor{}
_ TemplateEngine = &multiproto.MultiProtocol{}
)
// TemplateEngine is a template executor with different functionality
// Ex:
// 1. generic => executes all protocol requests one after another (Done)
// 2. flow => executes protocol requests based on how they are defined in flow (Done)
// 3. multiprotocol => executes multiple protocols in parallel (Done)
type TemplateEngine interface {
// Note: below methods only need to implement extra / engine specific functionality
// basic request compilation , callbacks , cli output callback etc are handled by `TemplateExecuter` and no need to do it again
// Extra Compilation (if any)
Compile() error
// ExecuteWithResults executes the template and returns results
ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error
// Name returns name of template engine
Name() string
}

20
v2/pkg/protocols/multi/README.md → v2/pkg/tmplexec/multiproto/README.md
View File

@ -1,9 +1,8 @@
## multi protocol execution
### Implementation
when template is unmarshalled, if it uses more than one protocol, it will be converted to a multi protocol
and the order of the protocols will be preserved as they were in the template and are stored in Request.Queue
when template is compiled , we iterate over queue and compile all the requests in the queue
when template is unmarshalled, if it uses more than one protocol, then order of protocols is preserved and is same is passed to Executor
multiproto is engine/backend for TemplateExecutor which takes care of sharing logic between protocols and executing them in order
### Execution
when multi protocol template is executed , all protocol requests present in Queue are executed in order
@ -13,21 +12,14 @@ and dynamic values extracted are added to template context.
apart from extracted `internal:true` values response fields/values of protocol are added to template context at `ExecutorOptions.TemplateCtx`
which takes care of sync and other issues if any. all response fields are prefixed with template type prefix ex: `ssl_subject_dn`
### Other Methods
Such templates are usually used when a particular vulnerability requires more than one protocol to be executed
and in such cases the final result is core of the logic hence all methods such as
Ex: MakeResultEventItem, MakeResultEvent, GetCompiledOperators
are not implemented in multi protocol and just call the same method on last protocol in queue
### Adding New Protocol to multi protocol execution logic
while logic/implementation of multi protocol execution is abstracted. it requires 3 statements to be added in newly implemented protocol
to make response fields of that protocol available to global context
- Add `request.options.TemplateCtx.GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate`
- Add `request.options.GetTemplateCtx(f.input.MetaInput).GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate`
```go
// example
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll())
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(f.input.MetaInput).GetAll())
variablesMap := request.options.Variables.Evaluate(values)
```
@ -36,13 +28,13 @@ to make response fields of that protocol available to global context
outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData)
// expose response variables in proto_var format
// this is no-op if the template is not a multi protocol template
request.options.AddTemplateVars(request.Type(), outputEvent)
request.options.AddTemplateVars(request.Type(),request.ID, outputEvent)
```
- Append all available template context values to outputEvent
```go
// add variables from template context before matching/extraction
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(f.input.MetaInput).GetAll())
```
adding these 3 statements takes care of all logic related to multi protocol execution

4
v2/pkg/tmplexec/multiproto/doc.go Normal file
View File

@ -0,0 +1,4 @@
package multiproto
// multiproto is a template executer engine that executes multiple protocols
// with shared logic in between

111
v2/pkg/tmplexec/multiproto/multi.go Normal file
View File

@ -0,0 +1,111 @@
package multiproto
import (
"strconv"
"sync/atomic"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
)
// Mutliprotocol is a template executer engine that executes multiple protocols
// with logic in between
type MultiProtocol struct {
requests []protocols.Request
options *protocols.ExecutorOptions
results *atomic.Bool
readOnlyArgs map[string]interface{} // readOnlyArgs are readonly args that are available after compilation
}
// NewMultiProtocol creates a new multiprotocol template engine from a list of requests
func NewMultiProtocol(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *MultiProtocol {
if results == nil {
results = &atomic.Bool{}
}
return &MultiProtocol{requests: requests, options: options, results: results}
}
// Compile engine specific compilation
func (m *MultiProtocol) Compile() error {
// load all variables and evaluate with existing data
variableMap := m.options.Variables.GetAll()
// cli options
optionVars := generators.BuildPayloadFromOptions(m.options.Options)
// constants
constants := m.options.Constants
allVars := generators.MergeMaps(variableMap, constants, optionVars)
allVars = m.options.Variables.Evaluate(allVars)
m.readOnlyArgs = allVars
// no need to load files since they are done at template level
return nil
}
// ExecuteWithResults executes the template and returns results
func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
// put all readonly args into template context
m.options.GetTemplateCtx(input.MetaInput).Merge(m.readOnlyArgs)
var finalProtoEvent *output.InternalWrappedEvent
// callback to process results from all protocols
multiProtoCallback := func(event *output.InternalWrappedEvent) {
if event != nil {
finalProtoEvent = event
}
// export dynamic values from operators (i.e internal:true)
if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 {
for k, v := range event.OperatorsResult.DynamicValues {
// TBD: iterate-all is only supported in `http` protocol
// we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context)
// currently if dynamic value array only contains one value we replace it with the value
if len(v) == 1 {
m.options.GetTemplateCtx(input.MetaInput).Set(k, v[0])
} else {
// Note: if extracted value contains multiple values then they can be accessed by indexing
// ex: if values are dynamic = []string{"a","b","c"} then they are available as
// dynamic = "a" , dynamic1 = "b" , dynamic2 = "c"
// we intentionally omit first index for unknown situations (where no of extracted values are not known)
for i, val := range v {
if i == 0 {
m.options.GetTemplateCtx(input.MetaInput).Set(k, val)
} else {
m.options.GetTemplateCtx(input.MetaInput).Set(k+strconv.Itoa(i), val)
}
}
}
}
}
// evaluate all variables after execution of each protocol
variableMap := m.options.Variables.Evaluate(m.options.GetTemplateCtx(input.MetaInput).GetAll())
m.options.GetTemplateCtx(input.MetaInput).Merge(variableMap) // merge all variables into template context
}
// template context: contains values extracted using `internal` extractor from previous protocols
// these values are extracted from each protocol in queue and are passed to next protocol in queue
// instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows)
// this makes it possible to use multi protocol templates in workflows
// Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow)
// execute all protocols in the queue
for _, req := range m.requests {
values := m.options.GetTemplateCtx(input.MetaInput).GetAll()
err := req.ExecuteWithResults(input, output.InternalEvent(values), nil, multiProtoCallback)
// if error skip execution of next protocols
if err != nil {
return err
}
}
// Review: how to handle events of multiple protocols in a single template
// currently the outer callback is only executed once (for the last protocol in queue)
// due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executem.go#L150
// this causes addition of duplicated / unncessary variables with prefix template_id_all_variables
callback(finalProtoEvent)
return nil
}
// Name of the template engine
func (m *MultiProtocol) Name() string {
return "multiproto"
}

12
v2/pkg/protocols/multi/request_test.go → v2/pkg/tmplexec/multiproto/multi_test.go
View File

@ -1,4 +1,4 @@
package multi_test
package multiproto_test
import (
"context"
@ -47,9 +47,9 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) {
Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 2, len(Template.MultiProtoRequest.Queue))
require.Equal(t, 2, len(Template.RequestsQueue))
err = Template.MultiProtoRequest.Compile(&executerOpts)
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
@ -62,13 +62,11 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) {
Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 3, len(Template.MultiProtoRequest.Queue))
require.Equal(t, 3, len(Template.RequestsQueue))
err = Template.MultiProtoRequest.Compile(&executerOpts)
err = Template.Executer.Compile()
require.Nil(t, err, "could not compile template")
require.True(t, len(Template.MultiProtoRequest.GetCompiledOperators()) > 0, "could not compile operators")
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)

0
v2/pkg/protocols/multi/testcases/multiprotodynamic.yaml → v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml
View File

0
v2/pkg/protocols/multi/testcases/multiprotowithprefix.yaml → v2/pkg/tmplexec/multiproto/testcases/multiprotowithprefix.yaml
View File

15
v2/pkg/types/interfaces.go
View File

@ -77,6 +77,21 @@ func ToString(data interface{}) string {
}
}
// ToStringNSlice converts an interface to string in a quick way or to a slice with strings
// if the input is a slice of interfaces.
func ToStringNSlice(data interface{}) interface{} {
switch s := data.(type) {
case []interface{}:
var a []string
for _, v := range s {
a = append(a, ToString(v))
}
return a
default:
return ToString(data)
}
}
func ToHexOrString(data interface{}) string {
switch s := data.(type) {
case string:

31
v2/pkg/types/types.go
View File

@ -2,12 +2,17 @@ package types
import (
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
)
@ -464,3 +469,29 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string {
}
return optionalArguments
}
// LoadHelperFile loads a helper file needed for the template
// this respects the sandbox rules and only loads files from
// allowed directories
func (options *Options) LoadHelperFile(filePath, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
if !options.AllowLocalFileAccess {
filePath = filepath.Clean(filePath)
templateAbsPath, err := filepath.Abs(templatePath)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("could not get absolute path")
}
templateDirectory := config.DefaultConfig.TemplatesDirectory
templatePathDir := filepath.Dir(templateAbsPath)
if !(templatePathDir != "/" && strings.HasPrefix(filePath, templatePathDir)) && !strings.HasPrefix(filePath, templateDirectory) {
return nil, errorutil.New("denied payload file path specified")
}
}
if catalog != nil {
return catalog.OpenFile(filePath)
}
f, err := os.Open(filePath)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("could not open file %v", filePath)
}
return f, nil
}

4
v2/pkg/utils/insertion_ordered_map.go
View File

@ -58,7 +58,7 @@ func (insertionOrderedStringMap *InsertionOrderedStringMap) UnmarshalJSON(data [
}
// toString converts an interface to string in a quick way
func toString(data interface{}) string {
func toString(data interface{}) interface{} {
switch s := data.(type) {
case nil:
return ""
@ -92,6 +92,8 @@ func toString(data interface{}) string {
return strconv.FormatUint(uint64(s), 10)
case []byte:
return string(s)
case []interface{}:
return data
default:
return fmt.Sprintf("%v", data)
}