-ExtensionDenylist is the list of file extensions to deny during matching.
+DenyList is the list of file, directories or extensions to deny during matching.
By default, it contains some non-interesting extensions that are hardcoded
in nuclei.
@@ -3599,6 +3666,106 @@ name: prefix
+## whois.Request
+Request is a request for the WHOIS protocol
+
+Appears in:
+
+
+-
Template.whois
+
+
+
+
+
+
+
+
+
+
+Matchers contains the detection mechanism for the request to identify
+whether the request was successful by doing pattern matching
+on request/responses.
+
+Multiple matchers can be combined with `matcher-condition` flag
+which accepts either `and` or `or` as argument.
+
+
+
+
+
+
+
+
+Extractors contains the extraction mechanism for the request to identify
+and extract parts of the response.
+
+
+
+
+
+
+
+matchers-condition string
+
+
+
+
+MatchersCondition is the condition between the matchers. Default is OR.
+
+
+Valid values:
+
+
+ - and
+
+ - or
+
+
+
+
+
+
+query string
+
+
+
+
+Query contains query for the request
+
+
+
+
+
+
+
+server string
+
+
+
+
+description: |
+ Optional WHOIS server URL.
+
+ If present, specifies the WHOIS server to execute the Request on.
+ Otherwise, nil enables bootstrapping
+
+
+
+
+
+
+
+
+
## workflows.WorkflowTemplate
Appears in:
@@ -3730,3 +3897,17 @@ Subtemplates are run if the name of matcher matches.
+
+## http.SignatureTypeHolder
+SignatureTypeHolder is used to hold internal type of the signature
+
+Appears in:
+
+
+-
Template.signature
+
+
+
+
+
+
diff --git a/integration_tests/http/self-contained.yaml b/integration_tests/http/self-contained.yaml
index 4ecacbbd9..b3c9aa993 100644
--- a/integration_tests/http/self-contained.yaml
+++ b/integration_tests/http/self-contained.yaml
@@ -9,7 +9,7 @@ self-contained: true
requests:
- raw:
- |
- GET http://localhost:5431/ HTTP/1.1
+ GET http://127.0.0.1:5431/ HTTP/1.1
Host: {{Hostname}}
matchers:
diff --git a/integration_tests/network/self-contained.yaml b/integration_tests/network/self-contained.yaml
index fad3e2ac8..711af4f55 100644
--- a/integration_tests/network/self-contained.yaml
+++ b/integration_tests/network/self-contained.yaml
@@ -8,7 +8,7 @@ info:
self-contained: true
network:
- host:
- - "localhost:5431"
+ - "127.0.0.1:5431"
matchers:
- type: word
diff --git a/integration_tests/whois/basic.yaml b/integration_tests/whois/basic.yaml
new file mode 100644
index 000000000..742272a9a
--- /dev/null
+++ b/integration_tests/whois/basic.yaml
@@ -0,0 +1,14 @@
+id: basic-whois-example
+
+info:
+ name: test template for WHOIS
+ author: pdteam
+ severity: info
+
+whois:
+ - query: "{{Host}}"
+ extractors:
+ - type: kval
+ kval:
+ - "expiration date"
+ - "registrar"
\ No newline at end of file
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 156090895..f600fcc1a 100755
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -472,8 +472,8 @@
"type": "string"
},
"type": "array",
- "title": "extensions to deny match",
- "description": "List of file extensions to deny during matching"
+ "title": "denylist",
+ "description": "List of files"
},
"id": {
"type": "string",
@@ -741,6 +741,12 @@
"title": "maximum http response body size",
"description": "Maximum size of http response body to read in bytes"
},
+ "signature": {
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$ref": "#/definitions/http.SignatureTypeHolder",
+ "title": "signature is the http request signature method",
+ "description": "Signature is the HTTP Request signature Method"
+ },
"cookie-reuse": {
"type": "boolean",
"title": "optional cookie reuse enable",
@@ -790,6 +796,14 @@
"additionalProperties": false,
"type": "object"
},
+ "http.SignatureTypeHolder": {
+ "enum": [
+ "AWS"
+ ],
+ "type": "string",
+ "title": "type of the signature",
+ "description": "Type of the signature"
+ },
"network.Input": {
"properties": {
"data": {
@@ -1026,6 +1040,47 @@
"additionalProperties": false,
"type": "object"
},
+ "whois.Request": {
+ "properties": {
+ "matchers": {
+ "items": {
+ "$ref": "#/definitions/matchers.Matcher"
+ },
+ "type": "array",
+ "title": "matchers to run on response",
+ "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+ },
+ "extractors": {
+ "items": {
+ "$ref": "#/definitions/extractors.Extractor"
+ },
+ "type": "array",
+ "title": "extractors to run on response",
+ "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+ },
+ "matchers-condition": {
+ "enum": [
+ "and",
+ "or"
+ ],
+ "type": "string",
+ "title": "condition between the matchers",
+ "description": "Conditions between the matchers"
+ },
+ "query": {
+ "type": "string",
+ "title": "query for the WHOIS request",
+ "description": "Query contains query for the request"
+ },
+ "server": {
+ "type": "string",
+ "title": "server url to execute the WHOIS request on",
+ "description": "Server contains the server url to execute the WHOIS request on"
+ }
+ },
+ "additionalProperties": false,
+ "type": "object"
+ },
"templates.Template": {
"required": [
"id",
@@ -1110,6 +1165,15 @@
"title": "websocket requests to make",
"description": "Websocket requests to make for the template"
},
+ "whois": {
+ "items": {
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$ref": "#/definitions/whois.Request"
+ },
+ "type": "array",
+ "title": "whois requests to make",
+ "description": "WHOIS requests to make for the template"
+ },
"workflows": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
@@ -1128,6 +1192,11 @@
"type": "boolean",
"title": "stop at first match",
"description": "Stop at first match for the template"
+ },
+ "signature": {
+ "$ref": "#/definitions/http.SignatureTypeHolder",
+ "title": "signature is the http request signature method",
+ "description": "Signature is the HTTP Request signature Method"
}
},
"additionalProperties": false,
diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go
index c2ccaa765..3e21fe94e 100644
--- a/v2/cmd/cve-annotate/main.go
+++ b/v2/cmd/cve-annotate/main.go
@@ -71,7 +71,7 @@ func getCVEData(client *nvd.Client, filePath, data string) {
cveName := matches[0][1]
severityMatches := severityRegex.FindAllStringSubmatch(data, 1)
- if len(matches) == 0 {
+ if len(severityMatches) == 0 {
return
}
severityValue := severityMatches[0][1]
diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh
old mode 100644
new mode 100755
index d37d29061..f53978219
--- a/v2/cmd/functional-test/run.sh
+++ b/v2/cmd/functional-test/run.sh
@@ -1,16 +1,23 @@
#!/bin/bash
+# reading os type from arguments
+CURRENT_OS=$1
+
+if [ "${CURRENT_OS}" == "windows-latest" ];then
+ extension=.exe
+fi
+
echo "::group::Building functional-test binary"
-go build
+go build -o functional-test$extension
echo "::endgroup::"
echo "::group::Building Nuclei binary from current branch"
-go build -o nuclei_dev ../nuclei
+go build -o nuclei_dev$extension ../nuclei
echo "::endgroup::"
-echo "::group::Installing latest release of nuclei"
-GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+echo "::group::Building latest release of nuclei"
+go build -o nuclei$extension -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
echo "::endgroup::"
echo 'Starting Nuclei functional test'
-./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt
\ No newline at end of file
+./functional-test$extension -main ./nuclei$extension -dev ./nuclei_dev$extension -testcases testcases.txt
diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go
index be9ab4c16..4bbed10a1 100644
--- a/v2/cmd/integration-test/integration-test.go
+++ b/v2/cmd/integration-test/integration-test.go
@@ -26,6 +26,7 @@ var (
"loader": loaderTestcases,
"websocket": websocketTestCases,
"headless": headlessTestcases,
+ "whois": whoisTestCases,
}
)
diff --git a/v2/cmd/integration-test/whois.go b/v2/cmd/integration-test/whois.go
new file mode 100644
index 000000000..edb534e43
--- /dev/null
+++ b/v2/cmd/integration-test/whois.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+ "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+)
+
+var whoisTestCases = map[string]testutils.TestCase{
+ "whois/basic.yaml": &whoisBasic{},
+}
+
+type whoisBasic struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *whoisBasic) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://example.com", debug)
+ if err != nil {
+ return err
+ }
+ return expectResultsCount(results, 1)
+}
diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml
index 508446243..367b5ad28 100644
--- a/v2/cmd/nuclei/issue-tracker-config.yaml
+++ b/v2/cmd/nuclei/issue-tracker-config.yaml
@@ -1,61 +1,55 @@
-# to specify which severities should be reported
#allow-list:
-# severity: critical, high
-# to specify which severities should be excluded from reporting
+# severity: high, critical
#deny-list:
-# severity: info, low, medium
-
+# severity: low
+#
# GitHub contains configuration options for GitHub issue tracker
-#GitHub:
-# # base-url (optional) is the self-hosted GitHub application url
-# base-url: ""
+#github:
+# # base-url is the optional self-hosted GitHub application url
+# base-url: https://localhost:8443/github
# # username is the username of the GitHub user
-# username: ""
-# # owner is the owner name of the repository for issues.
-# owner: ""
-# # token is the token for GitHub account.
-# token: ""
-# # project-name is the name of the repository.
-# project-name: ""
-# # issue-label (optional) is the label of the created issue type
-# issue-label: ""
-# # severity-as-label (optional) sets the severity as the label of the created issue type
-# severity-as-label: false
-
-# GitLab contains configuration options for GitLab issue tracker
-#GitLab:
-# # base-url (optional) is the self-hosted GitLab application url
-# base-url: ""
+# username: test-username
+# # owner is the owner name of the repository for issues
+# owner: test-owner
+# # token is the token for GitHub account
+# token: test-token
+# # project-name is the name of the repository
+# project-name: test-project
+# # issue-label is the label of the created issue type
+# issue-label: bug
+#
+# GitLab contains configuration options for gitlab issue tracker
+#gitlab:
+# # base-url is the optional self-hosted GitLab application url
+# base-url: https://localhost:8443/gitlab
# # username is the username of the GitLab user
-# username: ""
-# # token is the token for GitLab account.
-# token: ""
-# # project-id is the ID of the repository.
-# project-id: ""
-# # issue-label (optional) is the label of the created issue type
-# issue-label: ""
-# # severity-as-label (optional) sets the severity as the label of the created issue type
-# severity-as-label: false
-
+# username: test-username
+# # token is the token for GitLab account
+# token: test-token
+# # project-name is the name/id of the project(repository)
+# project-name: "1234"
+# # issue-label is the label of the created issue type
+# issue-label: bug
+#
# Jira contains configuration options for Jira issue tracker
-#Jira:
-# # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#jira:
+# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
# cloud: true
-# # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created
+# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
# update-existing: false
-# # URL is the Jira application URL
-# url: ""
+# # URL is the jira application url
+# url: https://localhost/jira
# # account-id is the account-id of the Jira user or username in case of on-prem Jira
-# account-id: ""
+# account-id: test-account-id
# # email is the email of the user for Jira instance
-# email: ""
+# email: test@test.com
# # token is the token for Jira instance or password in case of on-prem Jira
-# token: ""
+# token: test-token
# # project-name is the name of the project.
-# project-name: ""
+# project-name: test-project-name
# # issue-type is the name of the created issue type
-# issue-type: ""
-
+# issue-type: bug
+#
# elasticsearch contains configuration options for elasticsearch exporter
#elasticsearch:
# # IP for elasticsearch instance
@@ -64,11 +58,11 @@
# port: 9200
# # IndexName is the name of the elasticsearch index
# index-name: nuclei
-# # SSL (optional) enables ssl for elasticsearch connection
+# # SSL enables ssl for elasticsearch connection
# ssl: false
-# # SSLVerification (optional) disables SSL verification for elasticsearch
+# # SSLVerification disables SSL verification for elasticsearch
# ssl-verification: false
# # Username for the elasticsearch instance
# username: test
# # Password is the password for elasticsearch instance
-# password: test
+# password: test
\ No newline at end of file
diff --git a/v2/go.mod b/v2/go.mod
index b42c95d45..62531b548 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -3,7 +3,6 @@ module github.com/projectdiscovery/nuclei/v2
go 1.17
require (
- github.com/Ice3man543/nvd v1.0.8
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
github.com/andygrunwald/go-jira v1.14.0
@@ -47,7 +46,6 @@ require (
github.com/shirou/gopsutil/v3 v3.21.9
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.4.1
- github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.0
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
github.com/valyala/fasttemplate v1.2.1
@@ -58,21 +56,31 @@ require (
go.uber.org/atomic v1.9.0
go.uber.org/multierr v1.7.0
go.uber.org/ratelimit v0.2.0
- golang.org/x/net v0.0.0-20211020060615-d418f374d309
+ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
golang.org/x/text v0.3.7
gopkg.in/yaml.v2 v2.4.0
moul.io/http2curl v1.0.0
)
+require github.com/aws/aws-sdk-go v1.42.3
+
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
+require (
+ github.com/Ice3man543/nvd v1.0.8
+ github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
+ github.com/stretchr/testify v1.7.0
+)
+
require (
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
github.com/Mzack9999/ldapserver v1.0.2-0.20211214172138-8f1cdd128383 // indirect
github.com/PuerkitoBio/goquery v1.6.0 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
+ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
github.com/andybalholm/cascadia v1.1.0 // indirect
github.com/antchfx/xpath v1.2.0 // indirect
@@ -103,6 +111,7 @@ require (
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
github.com/itchyny/timefmt-go v0.1.3 // indirect
github.com/jasonlvhit/gocron v0.0.1 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
@@ -110,6 +119,7 @@ require (
github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
+ github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -133,6 +143,7 @@ require (
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
+ gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
diff --git a/v2/go.sum b/v2/go.sum
index 1a6dce82d..6a9a34596 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -73,9 +73,12 @@ github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA=
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
@@ -104,6 +107,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.42.3 h1:lBKr3tQ06m1uykiychMNKLK1bRfOzaIEQpsI/S3QiNc=
+github.com/aws/aws-sdk-go v1.42.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
@@ -404,10 +409,16 @@ github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0=
github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
+github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
+github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -508,6 +519,7 @@ github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tB
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
@@ -558,6 +570,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7
github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8=
github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 h1:3Xn/CN6GVY+7mVuGgt5bfp0F9JwcWqnvwfb23Jf8Vxg=
+github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7/go.mod h1:inRbqVxN7ri77yTJY3ZtGtKegIFa3Qnarh7Xp9P7LgY=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -865,8 +879,9 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -957,8 +972,8 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1221,6 +1236,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1246,6 +1262,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go
index 54a79ac8d..9fee6c813 100644
--- a/v2/internal/runner/options.go
+++ b/v2/internal/runner/options.go
@@ -29,6 +29,10 @@ func ParseOptions(options *types.Options) {
// Show the user the banner
showBanner()
+ if !filepath.IsAbs(options.TemplatesDirectory) {
+ cwd, _ := os.Getwd()
+ options.TemplatesDirectory = filepath.Join(cwd, options.TemplatesDirectory)
+ }
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", config.Version)
os.Exit(0)
diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
index b71587267..56eb05360 100644
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@@ -73,7 +73,7 @@ func New(options *types.Options) (*Runner, error) {
options.NoUpdateTemplates = true
}
if err := runner.updateTemplates(); err != nil {
- gologger.Warning().Msgf("Could not update templates: %s\n", err)
+ gologger.Error().Msgf("Could not update templates: %s\n", err)
}
if options.Headless {
if engine.MustDisableSandbox() {
diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go
index 1cf4ccf5b..bc19933a6 100644
--- a/v2/internal/runner/update.go
+++ b/v2/internal/runner/update.go
@@ -119,7 +119,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
if err := config.WriteConfiguration(r.templatesConfig); err != nil {
return err
}
- gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
+ gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory)
return nil
}
@@ -135,13 +135,13 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
return config.WriteConfiguration(r.templatesConfig)
}
- if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil {
+ if err := r.updateTemplatesWithVersion(latestVersion, currentVersion, r, ctx); err != nil {
return err
}
return nil
}
-func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
+func (r *Runner) updateTemplatesWithVersion(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
if latestVersion.GT(currentVersion) {
gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String())
gologger.Info().Msgf("Downloading latest release...")
@@ -163,7 +163,7 @@ func updateTemplates(latestVersion semver.Version, currentVersion semver.Version
if err := config.WriteConfiguration(runner.templatesConfig); err != nil {
return err
}
- gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String())
+ gologger.Info().Msgf("Successfully updated nuclei-templates (v%s) to %s. GoodLuck!\n", latestVersion.String(), r.templatesConfig.TemplatesDirectory)
}
return nil
}
@@ -200,10 +200,6 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
return readErr
}
r.templatesConfig = configuration
-
- if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
- r.options.TemplatesDirectory = configuration.TemplatesDirectory
- }
}
return nil
}
diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go
index e8153b42e..0983aa71d 100644
--- a/v2/internal/runner/update_test.go
+++ b/v2/internal/runner/update_test.go
@@ -119,7 +119,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions")
}
-func TestCalculateTemplateAbsolutePath(t *testing.T) {
+func TestCalculateTemplateAbsolutePathPositiveScenario(t *testing.T) {
configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates")
defer os.RemoveAll(configuredTemplateDirectory)
@@ -136,24 +136,6 @@ func TestCalculateTemplateAbsolutePath(t *testing.T) {
require.False(t, skipFile)
}
})
-
- t.Run("negative scenarios", func(t *testing.T) {
- filePathsFromZip := []string{
- "./../nuclei-templates/../cve/test.yaml",
- "nuclei-templates/../cve/test.yaml",
- "nuclei-templates/cve/../test.yaml",
- "nuclei-templates/././../cve/test.yaml",
- "nuclei-templates/.././../cve/test.yaml",
- "nuclei-templates/.././../cve/../test.yaml",
- }
-
- for _, filePathFromZip := range filePathsFromZip {
- calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory)
- require.Nil(t, err)
- require.True(t, skipFile)
- require.Equal(t, "", calculatedTemplateAbsPath)
- }
- })
}
func zipFromDirectory(zipPath, directory string) error {
diff --git a/v2/internal/runner/update_unix_test.go b/v2/internal/runner/update_unix_test.go
new file mode 100644
index 000000000..c9e1a9a2d
--- /dev/null
+++ b/v2/internal/runner/update_unix_test.go
@@ -0,0 +1,34 @@
+//go:build !windows
+
+package runner
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestCalculateTemplateAbsolutePathNegativeScenario(t *testing.T) {
+ configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates")
+ defer os.RemoveAll(configuredTemplateDirectory)
+
+ t.Run("negative scenarios", func(t *testing.T) {
+ filePathsFromZip := []string{
+ "./../nuclei-templates/../cve/test.yaml",
+ "nuclei-templates/../cve/test.yaml",
+ "nuclei-templates/cve/../test.yaml",
+ "nuclei-templates/././../cve/test.yaml",
+ "nuclei-templates/.././../cve/test.yaml",
+ "nuclei-templates/.././../cve/../test.yaml",
+ }
+
+ for _, filePathFromZip := range filePathsFromZip {
+ calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory)
+ require.Nil(t, err)
+ require.True(t, skipFile)
+ require.Equal(t, "", calculatedTemplateAbsPath)
+ }
+ })
+}
diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go
index 0ac5a5ffd..7b6e6ab39 100644
--- a/v2/pkg/catalog/config/config.go
+++ b/v2/pkg/catalog/config/config.go
@@ -26,7 +26,7 @@ type Config struct {
const nucleiConfigFilename = ".templates-config.json"
// Version is the current version of nuclei
-const Version = `2.5.6-dev`
+const Version = `2.5.8-dev`
func getConfigDetails() (string, error) {
homeDir, err := os.UserHomeDir()
diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go
index 898eaca0a..052e2c856 100644
--- a/v2/pkg/operators/matchers/match.go
+++ b/v2/pkg/operators/matchers/match.go
@@ -154,7 +154,7 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
// MatchDSL matches on a generic map result
func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
- logExpressionEvaluationFailure := func (matcherName string, err error) {
+ logExpressionEvaluationFailure := func(matcherName string, err error) {
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error())
}
diff --git a/v2/pkg/projectfile/project.go b/v2/pkg/projectfile/project.go
index a64d7db1b..84e0a0cb5 100644
--- a/v2/pkg/projectfile/project.go
+++ b/v2/pkg/projectfile/project.go
@@ -1,12 +1,20 @@
package projectfile
import (
- "fmt"
"net/http"
+ "regexp"
+
+ "github.com/pkg/errors"
"github.com/projectdiscovery/hmap/store/hybrid"
)
+var (
+ ErrNotFound = errors.New("not found")
+ regexUserAgent = regexp.MustCompile(`(?mi)\r\nUser-Agent: .+\r\n`)
+ regexDefaultInteract = regexp.MustCompile(`(?mi)[a-zA-Z1-9%.]+interact.sh`)
+)
+
type Options struct {
Path string
Cleanup bool
@@ -31,15 +39,22 @@ func New(options *Options) (*ProjectFile, error) {
return &p, nil
}
+func (pf *ProjectFile) cleanupData(data []byte) []byte {
+ // ignore all user agents
+ data = regexUserAgent.ReplaceAll(data, []byte("\r\n"))
+ // ignore interact markers
+ return regexDefaultInteract.ReplaceAll(data, []byte(""))
+}
+
func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
- reqHash, err := hash(req)
+ reqHash, err := hash(pf.cleanupData(req))
if err != nil {
return nil, err
}
data, ok := pf.hm.Get(reqHash)
if !ok {
- return nil, fmt.Errorf("not found")
+ return nil, ErrNotFound
}
var httpRecord HTTPRecord
@@ -52,7 +67,7 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
}
func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error {
- reqHash, err := hash(req)
+ reqHash, err := hash(pf.cleanupData(req))
if err != nil {
return err
}
diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go
index 2aba5c324..e0a0879a8 100644
--- a/v2/pkg/protocols/common/expressions/variables.go
+++ b/v2/pkg/protocols/common/expressions/variables.go
@@ -16,49 +16,66 @@ func ContainsUnresolvedVariables(items ...string) error {
if len(matches) == 0 {
return nil
}
- errorString := &strings.Builder{}
- errorString.WriteString("unresolved variables found: ")
-
- for i, match := range matches {
+ var unresolvedVariables []string
+ for _, match := range matches {
if len(match) < 2 {
continue
}
- errorString.WriteString(match[1])
- if i != len(matches)-1 {
- errorString.WriteString(",")
- }
+ unresolvedVariables = append(unresolvedVariables, match[1])
}
- errorMessage := errorString.String()
- return errors.New(errorMessage)
+ return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ","))
}
return nil
}
+// ContainsVariablesWithNames returns an error with variable names if the passed
+// input contains unresolved {{
}} variables within the provided list
func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error {
for _, data := range items {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
- errorString := &strings.Builder{}
- errorString.WriteString("unresolved variables with values found: ")
-
- for i, match := range matches {
+ var unresolvedVariables []string
+ for _, match := range matches {
if len(match) < 2 {
continue
}
matchName := match[1]
if _, ok := names[matchName]; !ok {
- errorString.WriteString(matchName)
- if i != len(matches)-1 {
- errorString.WriteString(",")
- }
+ unresolvedVariables = append(unresolvedVariables, matchName)
}
}
- errorMessage := errorString.String()
- return errors.New(errorMessage)
+ return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
}
return nil
}
+
+// ContainsVariablesWithIgnoreList returns an error with variable names if the passed
+// input contains unresolved {{}} other than the ones listed in the ignore list
+func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...string) error {
+ var unresolvedVariables []string
+ for _, data := range items {
+ matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
+ if len(matches) == 0 {
+ return nil
+ }
+ for _, match := range matches {
+ if len(match) < 2 {
+ continue
+ }
+ matchName := match[1]
+ if _, ok := skipNames[matchName]; ok {
+ continue
+ }
+ unresolvedVariables = append(unresolvedVariables, matchName)
+ }
+ }
+
+ if len(unresolvedVariables) > 0 {
+ return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
+ }
+ return nil
+}
diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go
index 7dba43a0b..175ad0ee8 100644
--- a/v2/pkg/protocols/common/interactsh/interactsh.go
+++ b/v2/pkg/protocols/common/interactsh/interactsh.go
@@ -165,6 +165,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
+ data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false)
if !matched || result == nil {
return false // if we don't match, return
diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go
index 307c33a2f..80216dda8 100644
--- a/v2/pkg/protocols/common/protocolinit/init.go
+++ b/v2/pkg/protocols/common/protocolinit/init.go
@@ -6,6 +6,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@@ -23,6 +24,9 @@ func Init(options *types.Options) error {
if err := httpclientpool.Init(options); err != nil {
return err
}
+ if err := signerpool.Init(options); err != nil {
+ return err
+ }
return networkclientpool.Init(options)
}
diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go
index b0902e7a0..afb8b3da5 100644
--- a/v2/pkg/protocols/file/file.go
+++ b/v2/pkg/protocols/file/file.go
@@ -1,6 +1,7 @@
package file
import (
+ "path/filepath"
"strings"
"github.com/pkg/errors"
@@ -19,13 +20,13 @@ type Request struct {
// - value: '[]string{".txt", ".go", ".json"}'
Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
// description: |
- // ExtensionDenylist is the list of file extensions to deny during matching.
+ // DenyList is the list of file, directories or extensions to deny during matching.
//
// By default, it contains some non-interesting extensions that are hardcoded
// in nuclei.
// examples:
// - value: '[]string{".avi", ".mov", ".mp3"}'
- ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"`
+ DenyList []string `yaml:"denylist,omitempty" jsonschema:"title=denylist, directories and extentions to deny match,description=List of files, directories and extensions to deny during matching"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"`
@@ -41,9 +42,9 @@ type Request struct {
CompiledOperators *operators.Operators `yaml:"-"`
// cache any variables that may be needed for operation.
- options *protocols.ExecuterOptions
- extensions map[string]struct{}
- extensionDenylist map[string]struct{}
+ options *protocols.ExecuterOptions
+ extensions map[string]struct{}
+ denyList map[string]struct{}
// description: |
// NoRecursive specifies whether to not do recursive checks if folders are provided.
@@ -89,7 +90,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
request.options = options
request.extensions = make(map[string]struct{})
- request.extensionDenylist = make(map[string]struct{})
+ request.denyList = make(map[string]struct{})
for _, extension := range request.Extensions {
if extension == "all" {
@@ -101,17 +102,17 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
request.extensions[extension] = struct{}{}
}
}
- for _, extension := range defaultDenylist {
- if !strings.HasPrefix(extension, ".") {
- extension = "." + extension
+ // process default denylist (extensions)
+ for _, excludeItem := range defaultDenylist {
+ if !strings.HasPrefix(excludeItem, ".") {
+ excludeItem = "." + excludeItem
}
- request.extensionDenylist[extension] = struct{}{}
+ request.denyList[excludeItem] = struct{}{}
}
- for _, extension := range request.ExtensionDenylist {
- if !strings.HasPrefix(extension, ".") {
- extension = "." + extension
- }
- request.extensionDenylist[extension] = struct{}{}
+ for _, excludeItem := range request.DenyList {
+ request.denyList[excludeItem] = struct{}{}
+ // also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d)
+ request.denyList[filepath.Clean(excludeItem)] = struct{}{}
}
return nil
}
diff --git a/v2/pkg/protocols/file/file_test.go b/v2/pkg/protocols/file/file_test.go
index d568f9810..c32b7a313 100644
--- a/v2/pkg/protocols/file/file_test.go
+++ b/v2/pkg/protocols/file/file_test.go
@@ -16,11 +16,11 @@ func TestFileCompile(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"all", ".lock"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"all", ".lock"},
+ DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@@ -29,7 +29,7 @@ func TestFileCompile(t *testing.T) {
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
- require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist")
+ require.Contains(t, request.denyList, ".go", "could not get .go in denylist")
require.NotContains(t, request.extensions, ".go", "could get .go in allowlist")
require.True(t, request.allExtensions, "could not get correct allExtensions")
}
diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go
index f4deaa100..63b1597c8 100644
--- a/v2/pkg/protocols/file/find.go
+++ b/v2/pkg/protocols/file/find.go
@@ -7,7 +7,7 @@ import (
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
-
+ "github.com/projectdiscovery/folderutil"
"github.com/projectdiscovery/gologger"
)
@@ -51,7 +51,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string
return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
for _, match := range matches {
- if !request.validatePath(match) {
+ if !request.validatePath(absPath, match) {
continue
}
if _, ok := processed[match]; !ok {
@@ -73,7 +73,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
return false, nil
}
if _, ok := processed[absPath]; !ok {
- if !request.validatePath(absPath) {
+ if !request.validatePath(absPath, absPath) {
return false, nil
}
processed[absPath] = struct{}{}
@@ -93,7 +93,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
if d.IsDir() {
return nil
}
- if !request.validatePath(path) {
+ if !request.validatePath(absPath, path) {
return nil
}
if _, ok := processed[path]; !ok {
@@ -107,7 +107,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
}
// validatePath validates a file path for blacklist and whitelist options
-func (request *Request) validatePath(item string) bool {
+func (request *Request) validatePath(absPath, item string) bool {
extension := filepath.Ext(item)
if len(request.extensions) > 0 {
@@ -117,9 +117,81 @@ func (request *Request) validatePath(item string) bool {
return false
}
}
- if _, ok := request.extensionDenylist[extension]; ok {
- gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)
+ if matchingRule, ok := request.isInDenyList(absPath, item); ok {
+ gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule)
return false
}
+
return true
}
+
+func (request *Request) isInDenyList(absPath, item string) (string, bool) {
+ extension := filepath.Ext(item)
+ // check for possible deny rules
+ // - extension is in deny list
+ if _, ok := request.denyList[extension]; ok {
+ return extension, true
+ }
+
+ // - full path is in deny list
+ if _, ok := request.denyList[item]; ok {
+ return item, true
+ }
+
+ // file is in a forbidden subdirectory
+ filename := filepath.Base(item)
+ fullPathWithoutFilename := strings.TrimSuffix(item, filename)
+ relativePathWithFilename := strings.TrimPrefix(item, absPath)
+ relativePath := strings.TrimSuffix(relativePathWithFilename, filename)
+
+ // - filename is in deny list
+ if _, ok := request.denyList[filename]; ok {
+ return filename, true
+ }
+
+ // - relative path is in deny list
+ if _, ok := request.denyList[relativePath]; ok {
+ return relativePath, true
+ }
+
+ // relative path + filename are in the forbidden list
+ if _, ok := request.denyList[relativePathWithFilename]; ok {
+ return relativePathWithFilename, true
+ }
+
+ // root path + relative path are in the forbidden list
+ if _, ok := request.denyList[fullPathWithoutFilename]; ok {
+ return fullPathWithoutFilename, true
+ }
+
+ // check any progressive combined part of the relative and absolute path with filename for matches within rules prefixes
+ if pathTreeItem, ok := request.isAnyChunkInDenyList(relativePath, false); ok {
+ return pathTreeItem, true
+ }
+ if pathTreeItem, ok := request.isAnyChunkInDenyList(item, true); ok {
+ return pathTreeItem, true
+ }
+
+ return "", false
+}
+
+func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) {
+ var paths []string
+
+ if splitWithUtils {
+ pathInfo, _ := folderutil.NewPathInfo(path)
+ paths, _ = pathInfo.Paths()
+ } else {
+ pathTree := strings.Split(path, string(os.PathSeparator))
+ for i := range pathTree {
+ paths = append(paths, filepath.Join(pathTree[:i]...))
+ }
+ }
+ for _, pathTreeItem := range paths {
+ if _, ok := request.denyList[pathTreeItem]; ok {
+ return pathTreeItem, true
+ }
+ }
+
+ return "", false
+}
diff --git a/v2/pkg/protocols/file/find_test.go b/v2/pkg/protocols/file/find_test.go
index 58eb128b6..9ca543adc 100644
--- a/v2/pkg/protocols/file/find_test.go
+++ b/v2/pkg/protocols/file/find_test.go
@@ -19,11 +19,11 @@ func TestFindInputPaths(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"all", ".lock"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"all", ".lock"},
+ DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go
index 37cc62d52..cfafe5b50 100644
--- a/v2/pkg/protocols/file/operators_test.go
+++ b/v2/pkg/protocols/file/operators_test.go
@@ -20,11 +20,11 @@ func TestResponseToDSLMap(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"*", ".lock"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"*", ".lock"},
+ DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@@ -45,11 +45,11 @@ func TestFileOperatorMatch(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"*", ".lock"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"*", ".lock"},
+ DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@@ -133,11 +133,11 @@ func TestFileOperatorExtract(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"*", ".lock"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"*", ".lock"},
+ DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@@ -240,11 +240,11 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"*", ".lock"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"*", ".lock"},
+ DenyList: []string{".go"},
Operators: operators.Operators{
MatchersCondition: matcherCondition,
Matchers: matchers,
diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go
index 076c3b884..2f720e59a 100644
--- a/v2/pkg/protocols/file/request_test.go
+++ b/v2/pkg/protocols/file/request_test.go
@@ -23,11 +23,11 @@ func TestFileExecuteWithResults(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
- ID: templateID,
- MaxSize: 1024,
- NoRecursive: false,
- Extensions: []string{"all"},
- ExtensionDenylist: []string{".go"},
+ ID: templateID,
+ MaxSize: 1024,
+ NoRecursive: false,
+ Extensions: []string{"all"},
+ DenyList: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go
index 59fa9e703..70e212c70 100644
--- a/v2/pkg/protocols/http/build_request.go
+++ b/v2/pkg/protocols/http/build_request.go
@@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
@@ -121,13 +122,32 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam
if len(parts) < 3 {
return nil, fmt.Errorf("malformed request supplied")
}
+
+ payloads := generators.BuildPayloadFromOptions(r.request.options.Options)
+ // in case cases (eg requests signing, some variables uses default values if missing)
+ if defaultList := GetVariablesDefault(r.request.Signature.Value); defaultList != nil {
+ payloads = generators.MergeMaps(defaultList, payloads)
+ }
+ parts[1] = replacer.Replace(parts[1], payloads)
+
+ // the url might contain placeholders with ignore list
+ if ignoreList := GetVariablesNamesSkipList(r.request.Signature.Value); ignoreList != nil {
+ if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil {
+ return nil, err
+ }
+ } else { // the url might contain placeholders
+ if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+ return nil, err
+ }
+ }
+
parsed, err := url.Parse(parts[1])
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
}
values := generators.MergeMaps(
generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
- generators.BuildPayloadFromOptions(r.request.options.Options),
+ payloads,
)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go
index 72b7d1589..7ac9b36fc 100644
--- a/v2/pkg/protocols/http/http.go
+++ b/v2/pkg/protocols/http/http.go
@@ -128,6 +128,12 @@ type Request struct {
// SelfContained specifies if the request is self-contained.
SelfContained bool `yaml:"-" json:"-"`
+ // description: |
+ // Signature is the request signature method
+ // values:
+ // - "AWS"
+ Signature SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
+
// description: |
// CookieReuse is an optional setting that enables cookie reuse for
// all requests defined in raw section.
diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go
index 60db9b387..0dadc66f3 100644
--- a/v2/pkg/protocols/http/raw/raw.go
+++ b/v2/pkg/protocols/http/raw/raw.go
@@ -8,7 +8,7 @@ import (
"io"
"io/ioutil"
"net/url"
- "path/filepath"
+ "path"
"strings"
"github.com/projectdiscovery/rawhttp/client"
@@ -30,7 +30,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
rawRequest := &Request{
Headers: make(map[string]string),
}
-
+
parsedURL, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
@@ -143,7 +143,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
}
func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) []byte {
- fixedPath := filepath.Join(baseURL.Path, requestPath)
+ fixedPath := path.Join(baseURL.Path, requestPath)
fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1)
return fixed
}
diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go
index 8b8cfb593..37987adf5 100644
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@@ -29,7 +29,10 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+ "github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/stringsutil"
)
@@ -336,24 +339,27 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
err error
)
+ // Dump request for variables checks
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
if !generatedRequest.original.Race {
var dumpError error
+ // TODO: dump is currently not working with post-processors - somehow it alters the signature
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
if dumpError != nil {
return dumpError
}
dumpedRequestString := string(dumpedRequest)
- // Check if are there any unresolved variables. If yes, skip unless overridden by user.
- if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
- gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
- return errStopExecution
- }
-
- if request.options.Options.Debug || request.options.Options.DebugRequests {
- gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
- gologger.Print().Msgf("%s", dumpedRequestString)
+ if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
+ if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
+ gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
+ return errStopExecution
+ }
+ } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
+ if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
+ gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
+ return errStopExecution
+ }
}
}
var formedURL string
@@ -391,9 +397,27 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
}
}
if resp == nil {
+ if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
+ return errSignature
+ }
resp, err = request.httpClient.Do(generatedRequest.request)
}
}
+
+ // Dump the requests containing all headers
+ if !generatedRequest.original.Race {
+ var dumpError error
+ dumpedRequest, dumpError = dump(generatedRequest, reqURL)
+ if dumpError != nil {
+ return dumpError
+ }
+ dumpedRequestString := string(dumpedRequest)
+ if request.options.Options.Debug || request.options.Options.DebugRequests {
+ gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
+ gologger.Print().Msgf("%s", dumpedRequestString)
+ }
+ }
+
if err != nil {
// rawhttp doesn't support draining response bodies.
if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil {
@@ -533,6 +557,40 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
return nil
}
+// handleSignature of the http request
+func (request *Request) handleSignature(generatedRequest *generatedRequest) error {
+ switch request.Signature.Value {
+ case AWSSignature:
+ var awsSigner signer.Signer
+ payloads := request.options.Options.Vars.AsMap()
+ awsAccessKeyId := types.ToString(payloads["aws-id"])
+ awsSecretAccessKey := types.ToString(payloads["aws-secret"])
+ awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey}
+ service := types.ToString(payloads["service"])
+ region := types.ToString(payloads["region"])
+ // if region is empty use default value
+ if region == "" {
+ region = types.ToString(signer.AwsDefaultVars["region"])
+ }
+ awsSignatureArguments := signer.AwsSignatureArguments{
+ Service: types.ToString(service),
+ Region: types.ToString(region),
+ Time: time.Now(),
+ }
+
+ awsSigner, err := signerpool.Get(request.options.Options, &signerpool.Configuration{SignerArgs: awsSignerArgs})
+ if err != nil {
+ return err
+ }
+ err = awsSigner.SignHTTP(generatedRequest.request.Request, awsSignatureArguments)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
// setCustomHeaders sets the custom headers for generated request
func (request *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range request.customHeaders {
diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go
new file mode 100644
index 000000000..290c628e2
--- /dev/null
+++ b/v2/pkg/protocols/http/signature.go
@@ -0,0 +1,107 @@
+package http
+
+import (
+ "encoding/json"
+
+ "github.com/alecthomas/jsonschema"
+ "github.com/pkg/errors"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
+)
+
+// SignatureType is the type of signature
+type SignatureType int
+
+// Supported values for the SignatureType
+const (
+ AWSSignature SignatureType = iota + 1
+ signatureLimit
+)
+
+// signatureTypeMappings is a table for conversion of signature type from string.
+var signatureTypeMappings = map[SignatureType]string{
+ AWSSignature: "AWS",
+}
+
+func GetSupportedSignaturesTypes() []SignatureType {
+ var result []SignatureType
+ for index := SignatureType(1); index < signatureLimit; index++ {
+ result = append(result, index)
+ }
+ return result
+}
+
+func toSignatureType(valueToMap string) (SignatureType, error) {
+ normalizedValue := normalizeValue(valueToMap)
+ for key, currentValue := range signatureTypeMappings {
+ if normalizedValue == currentValue {
+ return key, nil
+ }
+ }
+ return -1, errors.New("invalid signature type: " + valueToMap)
+}
+
+func (t SignatureType) String() string {
+ return signatureTypeMappings[t]
+}
+
+// SignatureTypeHolder is used to hold internal type of the signature
+type SignatureTypeHolder struct {
+ Value SignatureType
+}
+
+func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Type {
+ gotType := &jsonschema.Type{
+ Type: "string",
+ Title: "type of the signature",
+ Description: "Type of the signature",
+ }
+ for _, types := range GetSupportedSignaturesTypes() {
+ gotType.Enum = append(gotType.Enum, types.String())
+ }
+ return gotType
+}
+
+func (holder *SignatureTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
+ var marshalledTypes string
+ if err := unmarshal(&marshalledTypes); err != nil {
+ return err
+ }
+
+ computedType, err := toSignatureType(marshalledTypes)
+ if err != nil {
+ return err
+ }
+
+ holder.Value = computedType
+ return nil
+}
+
+func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) {
+ return json.Marshal(holder.Value.String())
+}
+
+func (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) {
+ return holder.Value.String(), nil
+}
+
+var ErrNoIgnoreList = errors.New("uknown signature types")
+
+// GetVariablesNamesSkipList depending on the signature type
+func GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {
+ switch signature {
+ case AWSSignature:
+ return signer.AwsSkipList
+ default:
+ return nil
+ }
+}
+
+// GetVariablesNamesSkipList depending on the signature type
+func GetVariablesDefault(signature SignatureType) map[string]interface{} {
+ switch signature {
+ case AWSSignature:
+ return signer.AwsDefaultVars
+ default:
+ return nil
+ }
+}
diff --git a/v2/pkg/protocols/http/signer/aws.go b/v2/pkg/protocols/http/signer/aws.go
new file mode 100644
index 000000000..3b6488712
--- /dev/null
+++ b/v2/pkg/protocols/http/signer/aws.go
@@ -0,0 +1,140 @@
+package signer
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io/ioutil"
+ "net/http"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+type AwsSigner struct {
+ creds *credentials.Credentials
+ signer *v4.Signer
+}
+
+type AwsSignerArgs struct {
+ AwsId string
+ AwsSecretToken string
+}
+
+func (awsSignerArgs AwsSignerArgs) Validate() error {
+ if awsSignerArgs.AwsId == "" {
+ return errors.New("empty id")
+ }
+ if awsSignerArgs.AwsSecretToken == "" {
+ return errors.New("empty token")
+ }
+
+ return nil
+}
+
+type AwsSignatureArguments struct {
+ Service string
+ Region string
+ Time time.Time
+}
+
+func (awsSignatureArguments AwsSignatureArguments) Validate() error {
+ if awsSignatureArguments.Region == "" {
+ return errors.New("empty region")
+ }
+ if awsSignatureArguments.Service == "" {
+ return errors.New("empty service")
+ }
+
+ return nil
+}
+
+func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) {
+ if err := args.Validate(); err != nil {
+ return nil, err
+ }
+ creds := credentials.NewStaticCredentials(args.AwsId, args.AwsSecretToken, "")
+ if creds == nil {
+ return nil, errors.New("couldn't create the credentials structure")
+ }
+ signer := v4.NewSigner(creds)
+ return &AwsSigner{creds: creds, signer: signer}, nil
+}
+
+func NewAwsSignerFromEnv() (*AwsSigner, error) {
+ creds := credentials.NewEnvCredentials()
+ if creds == nil {
+ return nil, errors.New("couldn't create the credentials structure")
+ }
+ signer := v4.NewSigner(creds)
+ return &AwsSigner{creds: creds, signer: signer}, nil
+}
+
+func NewAwsSignerFromFile() (*AwsSigner, error) {
+ creds := credentials.NewSharedCredentials("", "")
+ if creds == nil {
+ return nil, errors.New("couldn't create the credentials structure")
+ }
+ signer := v4.NewSigner(creds)
+ return &AwsSigner{creds: creds, signer: signer}, nil
+}
+
+func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) error {
+ signatureArgs, err := awsSigner.checkSignatureArgs(args)
+ if err != nil {
+ return err
+ }
+
+ awsSigner.prepareRequest(request)
+ var body *bytes.Reader
+ if request.Body != nil {
+ bodyBytes, err := ioutil.ReadAll(request.Body)
+ if err != nil {
+ return err
+ }
+ request.Body.Close()
+ body = bytes.NewReader(bodyBytes)
+ }
+ if _, err := awsSigner.signer.Sign(request, body, signatureArgs.Service, signatureArgs.Region, signatureArgs.Time); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error) {
+ signatureArgs, err := awsSigner.checkSignatureArgs(args)
+ if err != nil {
+ return nil, err
+ }
+
+ reqClone := request.Clone(context.Background())
+ awsSigner.prepareRequest(reqClone)
+ err = awsSigner.SignHTTP(reqClone, signatureArgs)
+ if err != nil {
+ return nil, err
+ }
+ headers := make(map[string]string)
+ headers["X-Amz-Date"] = reqClone.Header.Get("X-Amz-Date")
+ headers["Authorization"] = reqClone.Header.Get("Authorization")
+ return headers, nil
+}
+
+func (awsSigner *AwsSigner) checkSignatureArgs(args interface{}) (AwsSignatureArguments, error) {
+ if signatureArgs, ok := args.(AwsSignatureArguments); ok {
+ return signatureArgs, signatureArgs.Validate()
+ }
+ return AwsSignatureArguments{}, errors.New("wrong signature type")
+}
+
+func (awsSigner *AwsSigner) prepareRequest(request *http.Request) {
+ request.Header.Del("Host")
+}
+
+var AwsSkipList = map[string]interface{}{
+ "region": struct{}{},
+}
+
+var AwsDefaultVars = map[string]interface{}{
+ "region": "us-east-2",
+}
diff --git a/v2/pkg/protocols/http/signer/signer.go b/v2/pkg/protocols/http/signer/signer.go
new file mode 100644
index 000000000..132eddd11
--- /dev/null
+++ b/v2/pkg/protocols/http/signer/signer.go
@@ -0,0 +1,40 @@
+package signer
+
+import (
+ "errors"
+ "net/http"
+)
+
+type Signer interface {
+ SignHTTP(request *http.Request, args interface{}) error
+ CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error)
+}
+
+type SignerArgs interface {
+ Validate() error
+}
+
+type SignatureArguments interface {
+ Validate() error
+}
+
+func NewSigner(args SignerArgs) (signer Signer, err error) {
+ switch signerArgs := args.(type) {
+ case AwsSignerArgs:
+ awsSigner, err := NewAwsSigner(signerArgs)
+ if err != nil {
+ // $HOME/.aws/credentials
+ awsSigner, err = NewAwsSignerFromFile()
+ if err != nil {
+ // env variables
+ awsSigner, err = NewAwsSignerFromEnv()
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+ return awsSigner, err
+ default:
+ return nil, errors.New("unknown signature arguments type")
+ }
+}
diff --git a/v2/pkg/protocols/http/signerpool/signerpool.go b/v2/pkg/protocols/http/signerpool/signerpool.go
new file mode 100644
index 000000000..94fd9a7ca
--- /dev/null
+++ b/v2/pkg/protocols/http/signerpool/signerpool.go
@@ -0,0 +1,57 @@
+package signerpool
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
+
+ "github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var (
+ poolMutex *sync.RWMutex
+ clientPool map[string]signer.Signer
+)
+
+// Init initializes the clientpool implementation
+func Init(options *types.Options) error {
+ poolMutex = &sync.RWMutex{}
+ clientPool = make(map[string]signer.Signer)
+ return nil
+}
+
+// Configuration contains the custom configuration options for a client
+type Configuration struct {
+ SignerArgs signer.SignerArgs
+}
+
+// Hash returns the hash of the configuration to allow client pooling
+func (c *Configuration) Hash() string {
+ builder := &strings.Builder{}
+ builder.WriteString(fmt.Sprintf("%v", c.SignerArgs))
+ hash := builder.String()
+ return hash
+}
+
+// Get creates or gets a client for the protocol based on custom configuration
+func Get(options *types.Options, configuration *Configuration) (signer.Signer, error) {
+ hash := configuration.Hash()
+ poolMutex.RLock()
+ if client, ok := clientPool[hash]; ok {
+ poolMutex.RUnlock()
+ return client, nil
+ }
+ poolMutex.RUnlock()
+
+ client, err := signer.NewSigner(configuration.SignerArgs)
+ if err != nil {
+ return nil, err
+ }
+
+ poolMutex.Lock()
+ clientPool[hash] = client
+ poolMutex.Unlock()
+ return client, nil
+}
diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go
new file mode 100644
index 000000000..5f46dc433
--- /dev/null
+++ b/v2/pkg/protocols/whois/whois.go
@@ -0,0 +1,193 @@
+package whois
+
+import (
+ "net/url"
+ "strings"
+ "time"
+
+ jsoniter "github.com/json-iterator/go"
+ "github.com/openrdap/rdap"
+ "github.com/pkg/errors"
+
+ "github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/nuclei/v2/pkg/operators"
+ "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+ "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+ "github.com/projectdiscovery/nuclei/v2/pkg/output"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+ templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+ "github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// Request is a request for the WHOIS protocol
+type Request struct {
+ // Operators for the current request go here.
+ operators.Operators `yaml:",inline,omitempty"`
+ CompiledOperators *operators.Operators `yaml:"-"`
+
+ // description: |
+ // Query contains query for the request
+ Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"`
+
+ // description: |
+ // Optional WHOIS server URL.
+ //
+ // If present, specifies the WHOIS server to execute the Request on.
+ // Otherwise, nil enables bootstrapping
+ Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"`
+ // cache any variables that may be needed for operation.
+ client *rdap.Client
+ options *protocols.ExecuterOptions
+ parsedServerURL *url.URL
+}
+
+// Compile compiles the request generators preparing any requests possible.
+func (request *Request) Compile(options *protocols.ExecuterOptions) error {
+ var err error
+ if request.Server != "" {
+ request.parsedServerURL, err = url.Parse(request.Server)
+ if err != nil {
+ return errors.Wrap(err, "failed to parse server URL")
+ }
+ }
+
+ request.options = options
+ request.client = &rdap.Client{}
+
+ if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
+ compiled := &request.Operators
+ if err := compiled.Compile(); err != nil {
+ return errors.Wrap(err, "could not compile operators")
+ }
+ request.CompiledOperators = compiled
+ }
+ return nil
+}
+
+// Requests returns the total number of requests the rule will perform
+func (request *Request) Requests() int {
+ return 1
+}
+
+// GetID returns the ID for the request if any.
+func (request *Request) GetID() string {
+ return ""
+}
+
+// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
+func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+ // generate variables
+ variables := generateVariables(input)
+ // and replace placeholders
+ query := replacer.Replace(request.Query, variables)
+ // build an rdap request
+ rdapReq := rdap.NewAutoRequest(query)
+ rdapReq.Server = request.parsedServerURL
+ res, err := request.client.Do(rdapReq)
+ if err != nil {
+ return errors.Wrap(err, "could not make whois request")
+ }
+ gologger.Verbose().Msgf("Sent WHOIS request to %s", query)
+ if request.options.Options.Debug || request.options.Options.DebugRequests {
+ gologger.Debug().Msgf("[%s] Dumped WHOIS request for %s", request.options.TemplateID, query)
+ }
+
+ data := make(map[string]interface{})
+ var response interface{}
+ switch rdapReq.Type {
+ case rdap.DomainRequest:
+ // convert the rdap response to a whois style response (for domain request type only)
+ whoisResp := res.ToWhoisStyleResponse()
+ for k, v := range whoisResp.Data {
+ data[strings.ToLower(k)] = strings.Join(v, ",")
+ }
+ response = whoisResp
+ default:
+ response = res.Object
+ }
+ jsonData, _ := jsoniter.Marshal(response)
+ jsonDataString := string(jsonData)
+
+ data["type"] = request.Type().String()
+ data["host"] = query
+ data["response"] = jsonDataString
+
+ event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
+ if request.options.Options.Debug || request.options.Options.DebugResponse {
+ gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query)
+ gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false))
+ }
+
+ callback(event)
+ return nil
+}
+
+// Match performs matching operation for a matcher on model and returns:
+// true and a list of matched snippets if the matcher type is supports it
+// otherwise false and an empty string slice
+func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
+ return protocols.MakeDefaultMatchFunc(data, matcher)
+}
+
+// Extract performs extracting operation for an extractor on model and returns true or false.
+func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
+ return protocols.MakeDefaultExtractFunc(data, matcher)
+}
+
+// MakeResultEvent creates a result event from internal wrapped event
+func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
+ return protocols.MakeDefaultResultEvent(request, wrapped)
+}
+
+// GetCompiledOperators returns a list of the compiled operators
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+ return []*operators.Operators{request.CompiledOperators}
+}
+
+func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
+ data := &output.ResultEvent{
+ TemplateID: types.ToString(request.options.TemplateID),
+ TemplatePath: types.ToString(request.options.TemplatePath),
+ Info: request.options.TemplateInfo,
+ Type: types.ToString(wrapped.InternalEvent["type"]),
+ Host: types.ToString(wrapped.InternalEvent["host"]),
+ Metadata: wrapped.OperatorsResult.PayloadValues,
+ ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
+ Timestamp: time.Now(),
+ MatcherStatus: true,
+ Request: types.ToString(wrapped.InternalEvent["request"]),
+ Response: types.ToString(wrapped.InternalEvent["response"]),
+ }
+ return data
+}
+
+// Type returns the type of the protocol request
+func (request *Request) Type() templateTypes.ProtocolType {
+ return templateTypes.WHOISProtocol
+}
+
+// generateVariables will create default variables after parsing a url
+func generateVariables(input string) map[string]interface{} {
+ var domain string
+
+ parsed, err := url.Parse(input)
+ if err != nil {
+ return map[string]interface{}{"Input": input}
+ }
+ domain = parsed.Host
+ if domain == "" {
+ domain = input
+ }
+ if strings.Contains(domain, ":") {
+ domain = strings.Split(domain, ":")[0]
+ }
+
+ return map[string]interface{}{
+ "Input": input,
+ "Hostname": parsed.Host,
+ "Host": domain,
+ }
+}
diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go
index 079c96f4a..6ed64dc6d 100644
--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@@ -108,6 +108,11 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
// parseSelfContainedRequests parses the self contained template requests.
func (template *Template) parseSelfContainedRequests() {
+ if template.Signature.Value.String() != "" {
+ for _, request := range template.RequestsHTTP {
+ request.Signature = template.Signature
+ }
+ }
if !template.SelfContained {
return
}
@@ -128,7 +133,8 @@ func (template *Template) Requests() int {
len(template.RequestsHeadless) +
len(template.Workflows) +
len(template.RequestsSSL) +
- len(template.RequestsWebsocket)
+ len(template.RequestsWebsocket) +
+ len(template.RequestsWHOIS)
}
// compileProtocolRequests compiles all the protocol requests for the template
@@ -166,6 +172,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
case len(template.RequestsWebsocket) > 0:
requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket)
+
+ case len(template.RequestsWHOIS) > 0:
+ requests = template.convertRequestToProtocolsRequest(template.RequestsWHOIS)
}
template.Executer = executer.NewExecuter(requests, &options)
return nil
diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go
index 0eb400afe..ce3bdcafc 100644
--- a/v2/pkg/templates/templates.go
+++ b/v2/pkg/templates/templates.go
@@ -2,6 +2,9 @@
package templates
import (
+ "encoding/json"
+
+ validate "github.com/go-playground/validator/v10"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
@@ -11,8 +14,11 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
+ "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+ "go.uber.org/multierr"
+ "gopkg.in/yaml.v2"
)
// Template is a YAML input file which defines all the requests and
@@ -66,6 +72,9 @@ type Template struct {
// Websocket contains the Websocket request to make in the template.
RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"`
+ // description: |
+ // WHOIS contains the WHOIS request to make in the template.
+ RequestsWHOIS []*whois.Request `yaml:"whois,omitempty" json:"whois,omitempty" jsonschema:"title=whois requests to make,description=WHOIS requests to make for the template"`
// description: |
// Workflows is a yaml based workflow declaration code.
workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"`
@@ -78,6 +87,12 @@ type Template struct {
// Stop execution once first match is found
StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"`
+ // description: |
+ // Signature is the request signature method
+ // values:
+ // - "AWS"
+ Signature http.SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
+
// TotalRequests is the total number of requests for the template.
TotalRequests int `yaml:"-" json:"-"`
// Executer is the actual template executor for running template requests
@@ -96,6 +111,7 @@ var TemplateProtocols = []string{
"workflow",
"ssl",
"websocket",
+ "whois",
}
// Type returns the type of the template
@@ -117,7 +133,47 @@ func (template *Template) Type() types.ProtocolType {
return types.SSLProtocol
case len(template.RequestsWebsocket) > 0:
return types.WebsocketProtocol
+ case len(template.RequestsWHOIS) > 0:
+ return types.WHOISProtocol
default:
return types.InvalidProtocol
}
}
+
+// MarshalYAML forces recursive struct validation during marshal operation
+func (template *Template) MarshalYAML() ([]byte, error) {
+ out, marshalErr := yaml.Marshal(template)
+ errValidate := validate.New().Struct(template)
+ return out, multierr.Append(marshalErr, errValidate)
+}
+
+// MarshalYAML forces recursive struct validation after unmarshal operation
+func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error {
+ type Alias Template
+ alias := &Alias{}
+ err := unmarshal(alias)
+ if err != nil {
+ return err
+ }
+ *template = Template(*alias)
+ return validate.New().Struct(template)
+}
+
+// MarshalJSON forces recursive struct validation during marshal operation
+func (template *Template) MarshalJSON() ([]byte, error) {
+ out, marshalErr := json.Marshal(template)
+ errValidate := validate.New().Struct(template)
+ return out, multierr.Append(marshalErr, errValidate)
+}
+
+// UnmarshalJSON forces recursive struct validation after unmarshal operation
+func (template *Template) UnmarshalJSON(data []byte) error {
+ type Alias Template
+ alias := &Alias{}
+ err := json.Unmarshal(data, alias)
+ if err != nil {
+ return err
+ }
+ *template = Template(*alias)
+ return validate.New().Struct(template)
+}
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go
index e1b739e9f..4abe1612d 100644
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@@ -21,6 +21,7 @@ var (
ExtractorTypeHolderDoc encoder.Doc
GENERATORSAttackTypeHolderDoc encoder.Doc
HTTPMethodTypeHolderDoc encoder.Doc
+ SignatureTypeHolderDoc encoder.Doc
DNSRequestDoc encoder.Doc
DNSRequestTypeHolderDoc encoder.Doc
FILERequestDoc encoder.Doc
@@ -33,15 +34,17 @@ var (
SSLRequestDoc encoder.Doc
WEBSOCKETRequestDoc encoder.Doc
WEBSOCKETInputDoc encoder.Doc
+ WHOISRequestDoc encoder.Doc
WORKFLOWSWorkflowTemplateDoc encoder.Doc
WORKFLOWSMatcherDoc encoder.Doc
+ HTTPSignatureTypeHolderDoc encoder.Doc
)
func init() {
TemplateDoc.Type = "Template"
TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and"
TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template."
- TemplateDoc.Fields = make([]encoder.Doc, 12)
+ TemplateDoc.Fields = make([]encoder.Doc, 14)
TemplateDoc.Fields[0].Name = "id"
TemplateDoc.Fields[0].Type = "string"
TemplateDoc.Fields[0].Note = ""
@@ -99,21 +102,34 @@ func init() {
TemplateDoc.Fields[8].Note = ""
TemplateDoc.Fields[8].Description = "Websocket contains the Websocket request to make in the template."
TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Websocket contains the Websocket request to make in the template."
- TemplateDoc.Fields[9].Name = "workflows"
- TemplateDoc.Fields[9].Type = "[]workflows.WorkflowTemplate"
+ TemplateDoc.Fields[9].Name = "whois"
+ TemplateDoc.Fields[9].Type = "[]whois.Request"
TemplateDoc.Fields[9].Note = ""
- TemplateDoc.Fields[9].Description = "Workflows is a list of workflows to execute for a template."
- TemplateDoc.Fields[9].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
- TemplateDoc.Fields[10].Name = "self-contained"
- TemplateDoc.Fields[10].Type = "bool"
+ TemplateDoc.Fields[9].Description = "WHOIS contains the WHOIS request to make in the template."
+ TemplateDoc.Fields[9].Comments[encoder.LineComment] = "WHOIS contains the WHOIS request to make in the template."
+ TemplateDoc.Fields[10].Name = "workflows"
+ TemplateDoc.Fields[10].Type = "[]workflows.WorkflowTemplate"
TemplateDoc.Fields[10].Note = ""
- TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained"
- TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
- TemplateDoc.Fields[11].Name = "stop-at-first-match"
+ TemplateDoc.Fields[10].Description = "Workflows is a list of workflows to execute for a template."
+ TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
+ TemplateDoc.Fields[11].Name = "self-contained"
TemplateDoc.Fields[11].Type = "bool"
TemplateDoc.Fields[11].Note = ""
- TemplateDoc.Fields[11].Description = "Stop execution once first match is found"
- TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Stop execution once first match is found"
+ TemplateDoc.Fields[11].Description = "Self Contained marks Requests for the template as self-contained"
+ TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
+ TemplateDoc.Fields[12].Name = "stop-at-first-match"
+ TemplateDoc.Fields[12].Type = "bool"
+ TemplateDoc.Fields[12].Note = ""
+ TemplateDoc.Fields[12].Description = "Stop execution once first match is found"
+ TemplateDoc.Fields[12].Comments[encoder.LineComment] = "Stop execution once first match is found"
+ TemplateDoc.Fields[13].Name = "signature"
+ TemplateDoc.Fields[13].Type = "http.SignatureTypeHolder"
+ TemplateDoc.Fields[13].Note = ""
+ TemplateDoc.Fields[13].Description = "Signature is the request signature method"
+ TemplateDoc.Fields[13].Comments[encoder.LineComment] = "Signature is the request signature method"
+ TemplateDoc.Fields[13].Values = []string{
+ "AWS",
+ }
MODELInfoDoc.Type = "model.Info"
MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template"
@@ -372,7 +388,7 @@ func init() {
Value: "HTTP response headers in name:value format",
},
}
- HTTPRequestDoc.Fields = make([]encoder.Doc, 27)
+ HTTPRequestDoc.Fields = make([]encoder.Doc, 28)
HTTPRequestDoc.Fields[0].Name = "matchers"
HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher"
HTTPRequestDoc.Fields[0].Note = ""
@@ -492,51 +508,59 @@ func init() {
HTTPRequestDoc.Fields[17].Comments[encoder.LineComment] = "MaxSize is the maximum size of http response body to read in bytes."
HTTPRequestDoc.Fields[17].AddExample("Read max 2048 bytes of the response", 2048)
- HTTPRequestDoc.Fields[18].Name = "cookie-reuse"
- HTTPRequestDoc.Fields[18].Type = "bool"
+ HTTPRequestDoc.Fields[18].Name = "signature"
+ HTTPRequestDoc.Fields[18].Type = "SignatureTypeHolder"
HTTPRequestDoc.Fields[18].Note = ""
- HTTPRequestDoc.Fields[18].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
- HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
- HTTPRequestDoc.Fields[19].Name = "redirects"
+ HTTPRequestDoc.Fields[18].Description = "Signature is the request signature method"
+ HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "Signature is the request signature method"
+ HTTPRequestDoc.Fields[18].Values = []string{
+ "AWS",
+ }
+ HTTPRequestDoc.Fields[19].Name = "cookie-reuse"
HTTPRequestDoc.Fields[19].Type = "bool"
HTTPRequestDoc.Fields[19].Note = ""
- HTTPRequestDoc.Fields[19].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
- HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
- HTTPRequestDoc.Fields[20].Name = "pipeline"
+ HTTPRequestDoc.Fields[19].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
+ HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
+ HTTPRequestDoc.Fields[20].Name = "redirects"
HTTPRequestDoc.Fields[20].Type = "bool"
HTTPRequestDoc.Fields[20].Note = ""
- HTTPRequestDoc.Fields[20].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
- HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
- HTTPRequestDoc.Fields[21].Name = "unsafe"
+ HTTPRequestDoc.Fields[20].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
+ HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
+ HTTPRequestDoc.Fields[21].Name = "pipeline"
HTTPRequestDoc.Fields[21].Type = "bool"
HTTPRequestDoc.Fields[21].Note = ""
- HTTPRequestDoc.Fields[21].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
- HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
- HTTPRequestDoc.Fields[22].Name = "race"
+ HTTPRequestDoc.Fields[21].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
+ HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
+ HTTPRequestDoc.Fields[22].Name = "unsafe"
HTTPRequestDoc.Fields[22].Type = "bool"
HTTPRequestDoc.Fields[22].Note = ""
- HTTPRequestDoc.Fields[22].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
- HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
- HTTPRequestDoc.Fields[23].Name = "req-condition"
+ HTTPRequestDoc.Fields[22].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
+ HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
+ HTTPRequestDoc.Fields[23].Name = "race"
HTTPRequestDoc.Fields[23].Type = "bool"
HTTPRequestDoc.Fields[23].Note = ""
- HTTPRequestDoc.Fields[23].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
- HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
- HTTPRequestDoc.Fields[24].Name = "stop-at-first-match"
+ HTTPRequestDoc.Fields[23].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
+ HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
+ HTTPRequestDoc.Fields[24].Name = "req-condition"
HTTPRequestDoc.Fields[24].Type = "bool"
HTTPRequestDoc.Fields[24].Note = ""
- HTTPRequestDoc.Fields[24].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
- HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
- HTTPRequestDoc.Fields[25].Name = "skip-variables-check"
+ HTTPRequestDoc.Fields[24].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
+ HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
+ HTTPRequestDoc.Fields[25].Name = "stop-at-first-match"
HTTPRequestDoc.Fields[25].Type = "bool"
HTTPRequestDoc.Fields[25].Note = ""
- HTTPRequestDoc.Fields[25].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
- HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
- HTTPRequestDoc.Fields[26].Name = "iterate-all"
+ HTTPRequestDoc.Fields[25].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ HTTPRequestDoc.Fields[26].Name = "skip-variables-check"
HTTPRequestDoc.Fields[26].Type = "bool"
HTTPRequestDoc.Fields[26].Note = ""
- HTTPRequestDoc.Fields[26].Description = "IterateAll iterates all the values extracted from internal extractors"
- HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
+ HTTPRequestDoc.Fields[26].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
+ HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
+ HTTPRequestDoc.Fields[27].Name = "iterate-all"
+ HTTPRequestDoc.Fields[27].Type = "bool"
+ HTTPRequestDoc.Fields[27].Note = ""
+ HTTPRequestDoc.Fields[27].Description = "IterateAll iterates all the values extracted from internal extractors"
+ HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
MATCHERSMatcherDoc.Type = "matchers.Matcher"
MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol."
@@ -570,6 +594,10 @@ func init() {
TypeName: "websocket.Request",
FieldName: "matchers",
},
+ {
+ TypeName: "whois.Request",
+ FieldName: "matchers",
+ },
}
MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13)
MATCHERSMatcherDoc.Fields[0].Name = "type"
@@ -731,6 +759,10 @@ func init() {
TypeName: "websocket.Request",
FieldName: "extractors",
},
+ {
+ TypeName: "whois.Request",
+ FieldName: "extractors",
+ },
}
EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11)
EXTRACTORSExtractorDoc.Fields[0].Name = "name"
@@ -892,6 +924,17 @@ func init() {
"PURGE",
}
+ SignatureTypeHolderDoc.Type = "SignatureTypeHolder"
+ SignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature"
+ SignatureTypeHolderDoc.Description = "SignatureTypeHolder is used to hold internal type of the signature"
+ SignatureTypeHolderDoc.AppearsIn = []encoder.Appearance{
+ {
+ TypeName: "http.Request",
+ FieldName: "signature",
+ },
+ }
+ SignatureTypeHolderDoc.Fields = make([]encoder.Doc, 0)
+
DNSRequestDoc.Type = "dns.Request"
DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template"
DNSRequestDoc.Description = "Request contains a DNS protocol request to be made from a template"
@@ -1139,8 +1182,8 @@ func init() {
FILERequestDoc.Fields[4].Name = "denylist"
FILERequestDoc.Fields[4].Type = "[]string"
FILERequestDoc.Fields[4].Note = ""
- FILERequestDoc.Fields[4].Description = "ExtensionDenylist is the list of file extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei."
- FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "ExtensionDenylist is the list of file extensions to deny during matching."
+ FILERequestDoc.Fields[4].Description = "DenyList is the list of file, directories or extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei."
+ FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "DenyList is the list of file, directories or extensions to deny during matching."
FILERequestDoc.Fields[4].AddExample("", []string{".avi", ".mov", ".mp3"})
FILERequestDoc.Fields[5].Name = "id"
@@ -1645,6 +1688,46 @@ func init() {
WEBSOCKETInputDoc.Fields[1].AddExample("", "prefix")
+ WHOISRequestDoc.Type = "whois.Request"
+ WHOISRequestDoc.Comments[encoder.LineComment] = " Request is a request for the WHOIS protocol"
+ WHOISRequestDoc.Description = "Request is a request for the WHOIS protocol"
+ WHOISRequestDoc.AppearsIn = []encoder.Appearance{
+ {
+ TypeName: "Template",
+ FieldName: "whois",
+ },
+ }
+ WHOISRequestDoc.Fields = make([]encoder.Doc, 5)
+ WHOISRequestDoc.Fields[0].Name = "matchers"
+ WHOISRequestDoc.Fields[0].Type = "[]matchers.Matcher"
+ WHOISRequestDoc.Fields[0].Note = ""
+ WHOISRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+ WHOISRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+ WHOISRequestDoc.Fields[1].Name = "extractors"
+ WHOISRequestDoc.Fields[1].Type = "[]extractors.Extractor"
+ WHOISRequestDoc.Fields[1].Note = ""
+ WHOISRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+ WHOISRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+ WHOISRequestDoc.Fields[2].Name = "matchers-condition"
+ WHOISRequestDoc.Fields[2].Type = "string"
+ WHOISRequestDoc.Fields[2].Note = ""
+ WHOISRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+ WHOISRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+ WHOISRequestDoc.Fields[2].Values = []string{
+ "and",
+ "or",
+ }
+ WHOISRequestDoc.Fields[3].Name = "query"
+ WHOISRequestDoc.Fields[3].Type = "string"
+ WHOISRequestDoc.Fields[3].Note = ""
+ WHOISRequestDoc.Fields[3].Description = "Query contains query for the request"
+ WHOISRequestDoc.Fields[3].Comments[encoder.LineComment] = "Query contains query for the request"
+ WHOISRequestDoc.Fields[4].Name = "server"
+ WHOISRequestDoc.Fields[4].Type = "string"
+ WHOISRequestDoc.Fields[4].Note = ""
+ WHOISRequestDoc.Fields[4].Description = "description: |\n Optional WHOIS server URL.\n\n If present, specifies the WHOIS server to execute the Request on.\n Otherwise, nil enables bootstrapping"
+ WHOISRequestDoc.Fields[4].Comments[encoder.LineComment] = " description: |"
+
WORKFLOWSWorkflowTemplateDoc.Type = "workflows.WorkflowTemplate"
WORKFLOWSWorkflowTemplateDoc.Comments[encoder.LineComment] = ""
WORKFLOWSWorkflowTemplateDoc.Description = ""
@@ -1708,6 +1791,17 @@ func init() {
WORKFLOWSMatcherDoc.Fields[1].Note = ""
WORKFLOWSMatcherDoc.Fields[1].Description = "Subtemplates are run if the name of matcher matches."
WORKFLOWSMatcherDoc.Fields[1].Comments[encoder.LineComment] = "Subtemplates are run if the name of matcher matches."
+
+ HTTPSignatureTypeHolderDoc.Type = "http.SignatureTypeHolder"
+ HTTPSignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature"
+ HTTPSignatureTypeHolderDoc.Description = "SignatureTypeHolder is used to hold internal type of the signature"
+ HTTPSignatureTypeHolderDoc.AppearsIn = []encoder.Appearance{
+ {
+ TypeName: "Template",
+ FieldName: "signature",
+ },
+ }
+ HTTPSignatureTypeHolderDoc.Fields = make([]encoder.Doc, 0)
}
// GetTemplateDoc returns documentation for the file templates_doc.go.
@@ -1728,6 +1822,7 @@ func GetTemplateDoc() *encoder.FileDoc {
&ExtractorTypeHolderDoc,
&GENERATORSAttackTypeHolderDoc,
&HTTPMethodTypeHolderDoc,
+ &SignatureTypeHolderDoc,
&DNSRequestDoc,
&DNSRequestTypeHolderDoc,
&FILERequestDoc,
@@ -1740,8 +1835,10 @@ func GetTemplateDoc() *encoder.FileDoc {
&SSLRequestDoc,
&WEBSOCKETRequestDoc,
&WEBSOCKETInputDoc,
+ &WHOISRequestDoc,
&WORKFLOWSWorkflowTemplateDoc,
&WORKFLOWSMatcherDoc,
+ &HTTPSignatureTypeHolderDoc,
},
}
}
diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go
index be6b72453..cc0c8b93d 100644
--- a/v2/pkg/templates/types/types.go
+++ b/v2/pkg/templates/types/types.go
@@ -34,6 +34,8 @@ const (
SSLProtocol
// name:websocket
WebsocketProtocol
+ // name:whois
+ WHOISProtocol
limit
InvalidProtocol
)
@@ -49,6 +51,7 @@ var protocolMappings = map[ProtocolType]string{
WorkflowProtocol: "workflow",
SSLProtocol: "ssl",
WebsocketProtocol: "websocket",
+ WHOISProtocol: "whois",
}
func GetSupportedProtocolTypes() ProtocolTypes {