diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..d849b1032 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,15 @@ +blank_issues_enabled: false + +contact_links: + + - name: Ask an question / advise on using nuclei + url: https://github.com/projectdiscovery/nuclei/discussions/categories/q-a + about: Ask a question or request support for using nuclei + + - name: Share idea / feature to discuss for nuclei + url: https://github.com/projectdiscovery/nuclei/discussions/categories/ideas + about: Share idea / feature to discuss for nuclei + + - name: Connect with PD Team (Discord) + url: https://discord.gg/projectdiscovery + about: Connect with PD Team for direct communication \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 5a71fc093..8b41d50c2 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,14 +1,21 @@ --- name: Feature request -about: Suggest an idea for this project -title: "[feature]" -labels: '' +about: Request feature to implement in this project +title: "" +labels: 'Type: Enhancement' assignees: '' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +### Please describe your feature request: + + +### Describe the use case of this feature: + diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md index 00bb7867e..aeec33c17 100644 --- a/.github/ISSUE_TEMPLATE/issue-report.md +++ b/.github/ISSUE_TEMPLATE/issue-report.md @@ -1,18 +1,36 @@ --- name: Issue report -about: Create a report to help us improve -title: "[issue]" -labels: '' -assignees: '' +about: Create a report to help us to improve the project +labels: 'Type: Bug' --- -**Describe the bug** -A clear and concise description of what the bug is. + -**Nuclei version** -Please share the version of the nuclei you are running with `nuclei -version` + + +### Nuclei version: + + + + +### Current Behavior: + + +### Expected Behavior: + + +### Steps To Reproduce: + -**Screenshot of the error or bug** -please add the screenshot showing bug or issue you are facing. +### Anything else: + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..807314510 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Proposed changes + + + + +## Checklist + + + +- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/nuclei/tree/dev) branch +- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added necessary documentation (if appropriate) \ No newline at end of file diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index c0748e870..3a9983470 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -1,30 +1,26 @@ name: 🎉 Release Binary on: create: - tags: - - v* workflow_dispatch: jobs: release: runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + steps: - - - name: "Check out code" - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v2 + + - uses: actions/setup-go@v2 with: go-version: 1.17 - - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 + + - uses: goreleaser/goreleaser-action@v2 with: args: "release --rm-dist" version: latest - workdir: v2/ \ No newline at end of file + workdir: v2/ + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/README.md b/README.md index b68d2e700..bfa68972e 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp # Install Nuclei +Nuclei requires **go1.17** to install successfully. Run the following command to install the latest version - + ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest ``` @@ -88,30 +90,29 @@ TARGET: -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) TEMPLATES: - -tl list all available templates -t, -templates string[] template or template directory paths to include in the scan - -w, -workflows string[] list of workflows to run - -nt, -new-templates run newly added templates only + -nt, -new-templates run only new templates added in latest nuclei-templates release + -w, -workflows string[] workflow or workflow directory paths to include in the scan -validate validate the passed templates to nuclei + -tl list all available templates FILTERING: - -tags string[] execute a subset of templates that contain the provided tags - -include-tags string[] tags from the default deny list that permit executing more intrusive templates - -etags, -exclude-tags string[] exclude templates with the provided tags - -include-templates string[] templates to be executed even if they are excluded either by default or configuration - -exclude-templates, -exclude string[] template or template directory paths to exclude - -severity, -impact value[] Templates to run based on severity. Possible values: info, low, medium, high, critical - -author string[] execute templates that are (co-)created by the specified authors + -tags string[] execute a subset of templates that contain the provided tags + -etags, -exclude-tags string[] exclude templates with the provided tags + -itags, -include-tags string[] tags from the default deny list that permit executing more intrusive templates + -et, -exclude-templates string[] template or template directory paths to exclude + -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration + -s, -severity value[] Templates to run based on severity. Possible values - info,low,medium,high,critical + -es, -exclude-severity value[] Templates to exclude based on severity. Possible values - info,low,medium,high,critical + -a, -author string[] execute templates that are (co-)created by the specified authors OUTPUT: -o, -output string output file to write found issues/vulnerabilities -silent display findings only - -v, -verbose show verbose output - -vv display extra verbose information -nc, -no-color disable output content coloring (ANSI escape codes) -json write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) - -nm, -no-meta don't display match metadata in CLI output + -nm, -no-meta don't display match metadata -nts, -no-timestamp don't display timestamp metadata in CLI output -rdb, -report-db string local nuclei reporting database (always use this to persist report data) -me, -markdown-export string directory to export results in markdown format @@ -123,37 +124,39 @@ CONFIGURATIONS: -H, -header string[] custom headers in header:value format -V, -var value custom vars in var=value format -r, -resolvers string file containing resolver list for nuclei - -system-resolvers use system DNS resolving as error fallback + -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars enable environment variables support + -ev, -env-vars enable environment variables to be used in template INTERACTSH: - -no-interactsh disable interactsh server for OOB testing - -interactsh-url string interactsh server url for self-hosted instance (default "https://interactsh.com") - -interactsh-token string authentication token for self-hosted interactsh server - -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) - -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) - -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) - -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interactsh.com") + -itoken, -interactsh-token string authentication token for self-hosted interactsh server + -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) + -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) + -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) + -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) -rlm, -rate-limit-minute int maximum number of requests to send per minute -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) - -c, -concurrency int maximum number of templates to be executed in parallel (default 10) + -c, -concurrency int maximum number of templates to be executed in parallel (default 25) OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 5) -retries int number of times to retry a failed request (default 1) - -max-host-error int max errors for a host before skipping from scan (default 30) + -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path (default "$TMPDIR/") + -project-path string set a specific project path -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + -stream Stream mode - start elaborating without sorting the input HEADLESS: - -headless enable templates that require headless browser support - -page-timeout int seconds to wait for each page in headless mode (default 20) - -show-browser show the browser on the screen when running templates with headless mode + -headless enable templates that require headless browser support + -page-timeout int seconds to wait for each page in headless mode (default 20) + -sb, -show-browser show the browser on the screen when running templates with headless mode + -sc, -system-chrome Use local installed chrome browser instead of nuclei installed DEBUG: -debug show all requests and responses @@ -161,22 +164,24 @@ DEBUG: -debug-resp show all received responses -proxy, -proxy-url string URL of the HTTP proxy server -proxy-socks-url string URL of the SOCKS proxy server - -trace-log string file to write sent requests trace log + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version + -v, -verbose show verbose output + -vv display extra verbose information -tv, -templates-version shows the version of the installed nuclei-templates UPDATE: - -update update nuclei to the latest released version - -ut, -update-templates update the community templates to latest released version - -nut, -no-update-templates do not check for nuclei-templates updates - -ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates") + -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-directory string overwrite the default directory to install nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan - -stats-json write statistics data to an output file in JSONL(ines) format + -sj, -stats-json write statistics data to an output file in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) - -metrics expose nuclei metrics on a port - -metrics-port int port to expose nuclei metrics on (default 9092) + -m, -metrics expose nuclei metrics on a port + -mp, -metrics-port int port to expose nuclei metrics on (default 9092) ``` ### Running Nuclei diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 691c25af7..a3fffe351 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -230,6 +230,19 @@ Workflows is a list of workflows to execute for a template.
+
+ +self-contained bool + +
+
+ +Self Contained marks Requests for the template as self-contained + +
+ +
+ @@ -823,14 +836,14 @@ in a combined manner allowing multirequest based matchers. Attack is the type of payload combinations to perform. -Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. Valid values: - - sniper + - batteringram - pitchfork @@ -2312,14 +2325,14 @@ host: Attack is the type of payload combinations to perform. -Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. Valid values: - - sniper + - batteringram - pitchfork @@ -2379,6 +2392,31 @@ read-size: 2048 ``` + + +
+ +
+ +read-all bool + +
+
+ +ReadAll determines if the data stream should be read till the end regardless of the size + +Default value for read-all is false. + + + +Examples: + + +```yaml +read-all: false +``` + +
diff --git a/integration_tests/http/interactsh.yaml b/integration_tests/http/interactsh.yaml new file mode 100644 index 000000000..28d9c5606 --- /dev/null +++ b/integration_tests/http/interactsh.yaml @@ -0,0 +1,19 @@ +id: interactsh-integration-test + +info: + name: Interactsh Integration Test + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + url: 'http://{{interactsh-url}}' + + matchers: + - type: word + part: interactsh_protocol # Confirms the HTTP Interaction + words: + - "http" \ No newline at end of file diff --git a/integration_tests/http/self-contained.yaml b/integration_tests/http/self-contained.yaml new file mode 100644 index 000000000..4ecacbbd9 --- /dev/null +++ b/integration_tests/http/self-contained.yaml @@ -0,0 +1,18 @@ +id: example-self-contained-input + +info: + name: example-self-contained + author: pd-team + severity: info + +self-contained: true +requests: + - raw: + - | + GET http://localhost:5431/ HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: word + words: + - This is self-contained response \ No newline at end of file diff --git a/integration_tests/network/self-contained.yaml b/integration_tests/network/self-contained.yaml new file mode 100644 index 000000000..fad3e2ac8 --- /dev/null +++ b/integration_tests/network/self-contained.yaml @@ -0,0 +1,16 @@ +id: example-self-contained-input + +info: + name: example-self-contained + author: pd-team + severity: info + +self-contained: true +network: + - host: + - "localhost:5431" + + matchers: + - type: word + words: + - "Authentication successful" \ No newline at end of file diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 1578630eb..1db207d77 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -606,7 +606,7 @@ }, "attack": { "enum": [ - "sniper", + "batteringram", "pitchfork", "clusterbomb" ], @@ -777,7 +777,7 @@ }, "attack": { "enum": [ - "sniper", + "batteringram", "pitchfork", "clusterbomb" ], @@ -809,6 +809,11 @@ "title": "size of network response to read", "description": "Size of response to read at the end. Default is 1024 bytes" }, + "read-all": { + "type": "boolean", + "title": "read all response stream", + "description": "Read all response stream till the server stops sending" + }, "matchers": { "items": { "$ref": "#/definitions/matchers.Matcher" @@ -845,6 +850,7 @@ ], "properties": { "id": { + "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$", "type": "string", "title": "id of the template", "description": "The Unique ID for the template", @@ -911,6 +917,11 @@ "type": "array", "title": "list of workflows to execute", "description": "List of workflows to execute for template" + }, + "self-contained": { + "type": "boolean", + "title": "mark requests as self-contained", + "description": "Mark Requests for the template as self-contained" } }, "additionalProperties": false, diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 030b25443..8f6c635c4 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -7,7 +7,7 @@ echo 'Building Nuclei binary from current branch' go build -o nuclei_dev ../nuclei echo 'Installing latest release of nuclei' -GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo 'Starting Nuclei functional test' ./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 28b52d7c1..c8bc6acc1 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -31,6 +31,34 @@ var httpTestcases = map[string]testutils.TestCase{ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/interactsh.yaml": &httpInteractshRequest{}, + "http/self-contained.yaml": &httpRequestSelContained{}, +} + +type httpInteractshRequest struct{} + +// Executes executes a test case and returns an error if occurred +func (h *httpInteractshRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + value := r.Header.Get("url") + if value != "" { + if resp, _ := http.DefaultClient.Get(value); resp != nil { + resp.Body.Close() + } + } + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil } type httpGetHeaders struct{} @@ -493,3 +521,35 @@ func (h *httpRequestCondition) Execute(filePath string) error { } return nil } + +type httpRequestSelContained struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpRequestSelContained) Execute(filePath string) error { + router := httprouter.New() + var routerErr error + + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("This is self-contained response")) + }) + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), + Handler: router, + } + go func() { + _ = server.ListenAndServe() + }() + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 8c738537d..e170b07b9 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -7,11 +7,14 @@ import ( ) var networkTestcases = map[string]testutils.TestCase{ - "network/basic.yaml": &networkBasic{}, - "network/hex.yaml": &networkBasic{}, - "network/multi-step.yaml": &networkMultiStep{}, + "network/basic.yaml": &networkBasic{}, + "network/hex.yaml": &networkBasic{}, + "network/multi-step.yaml": &networkMultiStep{}, + "network/self-contained.yaml": &networkRequestSelContained{}, } +const defaultStaticPort = 5431 + type networkBasic struct{} // Execute executes a test case and returns an error if occurred @@ -94,3 +97,28 @@ func (h *networkMultiStep) Execute(filePath string) error { } return nil } + +type networkRequestSelContained struct{} + +// Execute executes a test case and returns an error if occurred +func (h *networkRequestSelContained) Execute(filePath string) error { + var routerErr error + + ts := testutils.NewTCPServer(func(conn net.Conn) { + defer conn.Close() + + _, _ = conn.Write([]byte("Authentication successful")) + }, defaultStaticPort) + defer ts.Close() + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index fd3cab274..5364db5f5 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -7,7 +7,7 @@ # github contains configuration options for github issue tracker #github: -# # base-url is the optional self-hosted github application url +# # base-url (optional) is the self-hosted github application url # base-url: "" # # username is the username of the github user # username: "" @@ -17,12 +17,14 @@ # token: "" # # project-name is the name of the repository. # project-name: "" -# # issue-label is the label of the created issue type +# # issue-label (optional) is the label of the created issue type # issue-label: "" +# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# severity-as-label: false # gitlab contains configuration options for gitlab issue tracker #gitlab: -# # base-url is the optional self-hosted gitlab application url +# # base-url (optional) is the self-hosted gitlab application url # base-url: "" # # username is the username of the gitlab user # username: "" @@ -30,14 +32,16 @@ # token: "" # # project-id is the ID of the repository. # project-id: "" -# # issue-label is the label of the created issue type +# # issue-label (optional) is the label of the created issue type # issue-label: "" +# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# severity-as-label: false # jira contains configuration options for jira issue tracker #jira: -# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +# # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true -# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created +# # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created # update-existing: false # # URL is the jira application url # url: "" @@ -60,11 +64,11 @@ # port: 9200 # # IndexName is the name of the elasticsearch index # index-name: nuclei -# # SSL enables ssl for elasticsearch connection +# # SSL (optional) enables ssl for elasticsearch connection # ssl: false -# # SSLVerification disables SSL verification for elasticsearch +# # SSLVerification (optional) disables SSL verification for elasticsearch # ssl-verification: false # # Username for the elasticsearch instance # username: test # # Password is the password for elasticsearch instance -# password: test \ No newline at end of file +# password: test diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 81e701fc6..87f329379 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -53,45 +53,35 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "templates", "Templates", - flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), - flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), - flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), + flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), + flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), - - flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), + flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), ) createGroup(flagSet, "filters", "Filtering", flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), - flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list + flagSet.NormalizedStringSliceVarP(&options.IncludeTags, "include-tags", "itags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), - - flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"), - flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"), - - flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.VarP(&options.ExcludeSeverities, "exclude-impact", "exclude-severity", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"), + flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), + flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"), - flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), - flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), - flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), - flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), - flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), - flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) @@ -99,50 +89,46 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "configs", "Configurations", flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"), flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking - flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"), - flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"), - flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"), - flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), + flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), - flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), + flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"), ) createGroup(flagSet, "interactsh", "interactsh", - flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"), - flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interactsh.com", "interactsh server url for self-hosted instance"), - flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"), + flagSet.StringVarP(&options.InteractshURL, "interactsh-server", "iserver", "https://interactsh.com", "interactsh server url for self-hosted instance"), + flagSet.StringVarP(&options.InteractshToken, "interactsh-token", "itoken", "", "authentication token for self-hosted interactsh server"), flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), - flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"), + flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), ) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), - flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"), - + flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), - flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"), + flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input"), ) createGroup(flagSet, "headless", "Headless", flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), - flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"), - flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "Use local installed chrome browser instead of nuclei installed"), + flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), + flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "Use local installed chrome browser instead of nuclei installed"), ) createGroup(flagSet, "debug", "Debug", @@ -154,27 +140,26 @@ on extensive configurability, massive extensibility and ease of use.`) TODO should auto-set the HTTP_PROXY variable for the process? */ flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), - - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), - + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), + flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), + flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), ) createGroup(flagSet, "update", "Update", - flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"), - flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"), - flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"), - flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"), + flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei engine to the latest released version"), + flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"), + flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default directory to install nuclei-templates"), + flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"), ) createGroup(flagSet, "stats", "Statistics", flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), - flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"), + flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "write statistics data to an output file in JSONL(ines) format"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), - - flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"), - flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), + flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"), + flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"), ) _ = flagSet.Parse() diff --git a/v2/go.mod b/v2/go.mod index 471e12c5c..e5e74498f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,6 +31,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 github.com/projectdiscovery/gologger v1.1.4 @@ -40,7 +41,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 + github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 @@ -64,6 +65,7 @@ require ( golang.org/x/text v0.3.7 google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 + moul.io/http2curl v1.0.0 ) require ( @@ -73,6 +75,9 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -86,6 +91,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect @@ -95,6 +101,7 @@ require ( github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect + github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -104,6 +111,7 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect github.com/trivago/tgo v1.0.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index e85a7fce4..b95089c7e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -96,16 +96,23 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA= +github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -318,8 +325,11 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -393,6 +403,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= @@ -468,6 +479,8 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= +github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -562,7 +575,10 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= +github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -607,8 +623,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -647,6 +664,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -662,8 +681,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -1199,6 +1220,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index b75f09c33..56bf4ea4b 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -20,6 +20,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") - gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") + gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 745f15d02..59791fe63 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -22,11 +22,6 @@ func ParseOptions(options *types.Options) { // Check if stdin pipe was given options.Stdin = hasStdin() - // if VerboseVerbose is set, it implicitly enables the Verbose option as well - if options.VerboseVerbose { - options.Verbose = true - } - // Read the inputs and configure the logging configureOutput(options) @@ -127,7 +122,7 @@ func isValidURL(urlString string) bool { // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output - if options.Verbose || options.VerboseVerbose { + if options.Verbose { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.Debug { diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index f24055ccf..6b14b5f91 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -7,11 +7,20 @@ import ( "go.uber.org/atomic" ) +// processSelfContainedTemplates execute a self-contained template. +func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool { + match, err := template.Executer.Execute("") + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } + return match +} + // processTemplateWithList execute a template against the list of user provided targets func (r *Runner) processTemplateWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hostMap.Scan(func(k, _ []byte) error { + processItem := func(k, _ []byte) error { URL := string(k) // Skip if the host has had errors @@ -29,7 +38,13 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { results.CAS(false, match) }(URL) return nil - }) + } + if r.options.Stream { + _ = r.hostMapStream.Scan(processItem) + } else { + r.hostMap.Scan(processItem) + } + wg.Wait() return results.Load() } @@ -39,7 +54,7 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hostMap.Scan(func(k, _ []byte) error { + processItem := func(k, _ []byte) error { URL := string(k) // Skip if the host has had errors @@ -53,7 +68,14 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { results.CAS(false, match) }(URL) return nil - }) + } + + if r.options.Stream { + _ = r.hostMapStream.Scan(processItem) + } else { + r.hostMap.Scan(processItem) + } + wg.Wait() return results.Load() } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 06abc70ae..9eeb3f543 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,8 @@ import ( "go.uber.org/ratelimit" "gopkg.in/yaml.v2" + "github.com/projectdiscovery/filekv" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" @@ -45,6 +47,7 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { hostMap *hybrid.HybridMap + hostMapStream *filekv.FileDB output output.Writer interactsh *interactsh.Client inputCount int64 @@ -119,6 +122,20 @@ func New(options *types.Options) (*Runner, error) { } runner.hostMap = hm + if options.Stream { + fkvOptions := filekv.DefaultOptions + if tmpFileName, err := fileutil.GetTempFileName(); err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } else { + fkvOptions.Path = tmpFileName + } + fkv, err := filekv.Open(fkvOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary unsorted input file") + } + runner.hostMapStream = fkv + } + runner.inputCount = 0 dupeCount := 0 @@ -138,6 +155,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + _ = runner.hostMapStream.Set([]byte(url), nil) + } } } @@ -158,6 +178,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + _ = runner.hostMapStream.Set([]byte(url), nil) + } } } @@ -180,6 +203,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + _ = runner.hostMapStream.Set([]byte(url), nil) + } } input.Close() } @@ -290,6 +316,9 @@ func (r *Runner) Close() { if r.projectFile != nil { r.projectFile.Close() } + if r.options.Stream { + r.hostMapStream.Close() + } protocolinit.Close() } @@ -511,7 +540,9 @@ func (r *Runner) RunEnumeration() error { go func(template *templates.Template) { defer wgtemplates.Done() - if len(template.Workflows) > 0 { + if template.SelfContained { + results.CAS(false, r.processSelfContainedTemplates(template)) + } else if len(template.Workflows) > 0 { results.CAS(false, r.processWorkflowWithList(template)) } else { results.CAS(false, r.processTemplateWithList(template)) diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 4435b957f..c1cdfcc89 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -92,10 +92,14 @@ type TCPServer struct { } // NewTCPServer creates a new TCP server from a handler -func NewTCPServer(handler func(conn net.Conn)) *TCPServer { +func NewTCPServer(handler func(conn net.Conn), port ...int) *TCPServer { server := &TCPServer{} - l, err := net.Listen("tcp", "127.0.0.1:0") + var gotPort int + if len(port) > 0 { + gotPort = port[0] + } + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", gotPort)) if err != nil { panic(err) } diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 41ce53613..91b153cf2 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.3-dev` +const Version = `2.5.4-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 056f4ace7..787cd8ff6 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -224,6 +224,15 @@ var functions = map[string]govaluate.ExpressionFunction{ } return rand.Intn(max-min) + min, nil }, + "unixtime": func(args ...interface{}) (interface{}, error) { + seconds := 0 + if len(args) >= 1 { + seconds = int(args[0].(float64)) + } + now := time.Now() + offset := now.Add(time.Duration(seconds) * time.Second) + return offset.Unix(), nil + }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { seconds := args[0].(float64) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 8f42646cf..b90be29c3 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -59,15 +59,15 @@ type InternalWrappedEvent struct { // ResultEvent is a wrapped result event for a single nuclei output. type ResultEvent struct { // TemplateID is the ID of the template for the result. - TemplateID string `json:"templateID"` + TemplateID string `json:"template-id"` // TemplatePath is the path of template TemplatePath string `json:"-"` // Info contains information block of the template for the result. Info model.Info `json:"info,inline"` // MatcherName is the name of the matcher matched if any. - MatcherName string `json:"matcher_name,omitempty"` + MatcherName string `json:"matcher-name,omitempty"` // ExtractorName is the name of the extractor matched if any. - ExtractorName string `json:"extractor_name,omitempty"` + ExtractorName string `json:"extractor-name,omitempty"` // Type is the type of the result event. Type string `json:"type"` // Host is the host input on which match was found. @@ -75,9 +75,9 @@ type ResultEvent struct { // Path is the path input on which match was found. Path string `json:"path,omitempty"` // Matched contains the matched input in its transformed form. - Matched string `json:"matched,omitempty"` + Matched string `json:"matched-at,omitempty"` // ExtractedResults contains the extraction result from the inputs. - ExtractedResults []string `json:"extracted_results,omitempty"` + ExtractedResults []string `json:"extracted-results,omitempty"` // Request is the optional, dumped request for the match. Request string `json:"request,omitempty"` // Response is the optional, dumped response for the match. @@ -90,7 +90,9 @@ type ResultEvent struct { Timestamp time.Time `json:"timestamp"` // Interaction is the full details of interactsh interaction. Interaction *server.Interaction `json:"interaction,omitempty"` - + // CURLCommand is an optional curl command to reproduce the request + // Only applicable if the report is for HTTP. + CURLCommand string `json:"curl-command,omitempty"` FileToIndexPosition map[string]int `json:"-"` } diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index b65cb79e2..7cd0c7b99 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "gopkg.in/yaml.v2" @@ -17,7 +18,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) -const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" +const ( + mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" + invalidFieldFormatTemplate = "invalid field format for '%s' (allowed format is %s)" +) // LoadTemplate returns true if the template is valid and matches the filtering criteria. func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) { @@ -30,12 +34,12 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, nil } - templateInfo := template.Info - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { + stats.Increment(SyntaxErrorStats) return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -45,10 +49,8 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, templateParseError } - templateInfo := template.Info - if len(template.Workflows) > 0 { - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { return false, validationError } return true, nil @@ -71,18 +73,29 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode return match, err } -func validateMandatoryInfoFields(info *model.Info) error { - if info == nil { - return fmt.Errorf(mandatoryFieldMissingTemplate, "info") - } +func validateTemplateFields(template *templates.Template) error { + info := template.Info + + var errors []string if utils.IsBlank(info.Name) { - return fmt.Errorf(mandatoryFieldMissingTemplate, "name") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name")) } if info.Authors.IsEmpty() { - return fmt.Errorf(mandatoryFieldMissingTemplate, "author") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author")) } + + if template.ID == "" { + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id")) + } else if !templateIDRegexp.MatchString(template.ID) { + errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())) + } + + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, ", ")) + } + return nil } @@ -90,6 +103,7 @@ var ( parsedTemplatesCache *cache.Templates ShouldValidate bool fieldErrorRegexp = regexp.MustCompile(`not found in`) + templateIDRegexp = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`) ) const ( diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go new file mode 100644 index 000000000..ef74a317e --- /dev/null +++ b/v2/pkg/parsers/parser_test.go @@ -0,0 +1,110 @@ +package parsers + +import ( + "errors" + "fmt" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/stretchr/testify/require" +) + +func TestLoadTemplate(t *testing.T) { + origTemplatesCache := parsedTemplatesCache + defer func() { parsedTemplatesCache = origTemplatesCache }() + + tt := []struct { + name string + template *templates.Template + templateErr error + + expectedErr error + }{ + { + name: "valid", + template: &templates.Template{ + ID: "CVE-2021-27330", + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + }, + { + name: "emptyTemplate", + template: &templates.Template{}, + expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"), + }, + { + name: "emptyNameWithInvalidID", + template: &templates.Template{ + ID: "invalid id", + Info: model.Info{ + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(tc.name, tagFilter, nil) + if tc.expectedErr == nil { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, tc.expectedErr, err) + require.False(t, success) + } + }) + } + + t.Run("invalidTemplateID", func(t *testing.T) { + tt := []struct { + id string + success bool + }{ + {id: "A-B-C", success: true}, + {id: "A-B-C-1", success: true}, + {id: "CVE_2021_27330", success: true}, + {id: "ABC DEF", success: false}, + {id: "_-__AAA_", success: false}, + {id: " CVE-2021-27330", success: false}, + {id: "CVE-2021-27330 ", success: false}, + {id: "CVE-2021-27330-", success: false}, + {id: "-CVE-2021-27330-", success: false}, + {id: "CVE-2021--27330", success: false}, + {id: "CVE-2021+27330", success: false}, + } + for i, tc := range tt { + name := fmt.Sprintf("regexp%d", i) + t.Run(name, func(t *testing.T) { + template := &templates.Template{ + ID: tc.id, + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + } + parsedTemplatesCache.Store(name, template, nil) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(name, tagFilter, nil) + if tc.success { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err) + require.False(t, success) + } + }) + } + }) +} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 07c72d200..a63fbcb2b 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -14,7 +14,7 @@ type Generator struct { type Type int const ( - // Sniper replaces one iteration of the payload with a value. + // Batteringram replaces same payload into all of the defined payload positions at once. BatteringRam Type = iota + 1 // PitchFork replaces variables with positional value from multiple wordlists PitchFork @@ -43,10 +43,10 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string) generator.Type = payloadType generator.payloads = compiled - // Validate the sniper/batteringram payload set + // Validate the batteringram payload set if payloadType == BatteringRam { if len(payloads) != 1 { - return nil, errors.New("sniper/batteringram must have single payload set") + return nil, errors.New("batteringram must have single payload set") } } return generator, nil diff --git a/v2/pkg/protocols/common/generators/options.go b/v2/pkg/protocols/common/generators/options.go new file mode 100644 index 000000000..868ccef7c --- /dev/null +++ b/v2/pkg/protocols/common/generators/options.go @@ -0,0 +1,20 @@ +package generators + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// BuildPayloadFromOptions returns a map with the payloads provided via CLI +func BuildPayloadFromOptions(options *types.Options) map[string]interface{} { + m := make(map[string]interface{}) + // merge with vars + if !options.Vars.IsEmpty() { + m = MergeMaps(m, options.Vars.AsMap()) + } + + // merge with env vars + if options.EnvironmentVariables { + m = MergeMaps(EnvVars(), m) + } + return m +} diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index c9ec6e0ce..4c53ebee2 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -1,12 +1,18 @@ package engine import ( + "context" "crypto/tls" + "fmt" + "net" "net/http" + "net/http/cookiejar" + "net/url" "time" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "golang.org/x/net/proxy" ) // newhttpClient creates a new http client for headless communication with a timeout @@ -22,5 +28,40 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } else if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } + + jar, _ := cookiejar.New(nil) + + httpclient := &http.Client{ + Transport: transport, + Timeout: time.Duration(options.Timeout*3) * time.Second, + Jar: jar, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // the browser should follow redirects not us + return http.ErrUseLastResponse + }, + } + + return httpclient } diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 0ae5f0ac7..afd59fd29 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -8,7 +8,7 @@ import ( "github.com/go-rod/rod/lib/proto" ) -// Page is a single page in an isolated browser instanace +// Page is a single page in an isolated browser instance type Page struct { page *rod.Page rules []requestRule diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index d5cb56350..902b978ac 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -2,9 +2,11 @@ package engine import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" "net/url" + "os" "strings" "testing" "time" @@ -16,18 +18,7 @@ import ( ) func TestActionNavigate(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -35,270 +26,429 @@ func TestActionNavigate(t *testing.T) {

Nuclei Test

- `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + }) } func TestActionScript(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") + timeout := 2 * time.Second t.Run("run-and-results", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") + }) }) t.Run("hook", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") + }) }) } func TestActionClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) } func TestActionRightClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) } func TestActionTextInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - val := el.MustAttribute("event") - require.Equal(t, "input-change", *val, "could not get input change") - require.Equal(t, "test", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + val := el.MustAttribute("event") + require.Equal(t, "input-change", *val, "could not get input change") + require.Equal(t, "test", el.MustText(), "could not get input change value") + }) } func TestActionHeadersChange(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - fmt.Fprintln(w, `found`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") -} + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } -func TestActionWaitVisible(t *testing.T) { - t.Run("wait for an element being visible", func(t *testing.T) { - testWaitVisible(t, 2*time.Second, func(page *Page, err error) { - require.Nil(t, err, "could not run page actions") - - page.Page().MustElement("button").MustVisible() - page.Close() - }) - }) - - t.Run("timeout because of element not visible", func(t *testing.T) { - testWaitVisible(t, time.Second/2, func(page *Page, err error) { - require.Error(t, err) - require.Contains(t, err.Error(), "Element did not appear in the given amount of time") - }) + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") }) } -func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page, err error)) { +func TestActionScreenshot(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page() + require.FileExists(t, "test.png", el, "could not get screenshot file") + _ = os.Remove("test.png") + }) +} + +func TestActionTimeInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") + }) +} + +func TestActionSelectInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) +} + +func TestActionFilesInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") + }) +} + +func TestActionWaitLoad(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("button") + style, attributeErr := el.Attribute("style") + require.Nil(t, attributeErr) + require.Equal(t, "color: red;", *style, "could not get color") + }) +} + +func TestActionGetResource(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, len(out["src"]), 3159, "could not find resource") + }) +} + +func TestActionExtract(t *testing.T) { + response := ` + + + Nuclei Test Page + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Wait for me!", out["extract"], "could not extract text") + }) +} + +func TestActionSetMethod(t *testing.T) { + response := ` + + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") + }) +} + +func TestActionAddHeader(t *testing.T) { + actions := []*Action{ + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) +} + +func TestActionDeleteHeader(t *testing.T) { + actions := []*Action{ + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { + _, _ = fmt.Fprintln(w, `header deleted`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") + }) +} + +func TestActionSetBody(t *testing.T) { + actions := []*Action{ + {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + _, _ = fmt.Fprintln(w, string(body)) + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) +} + +func TestActionKeyboard(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "click", Data: map[string]string{"selector": "input"}}, + {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("input") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) +} + +func TestActionSleep(t *testing.T) { response := ` @@ -310,6 +460,59 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page ` + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") + }) +} + +func TestActionWaitVisible(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + + t.Run("wait for an element being visible", func(t *testing.T) { + testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + + page.Page().MustElement("button").MustVisible() + }) + }) + + t.Run("timeout because of element not visible", func(t *testing.T) { + testHeadlessSimpleResponse(t, response, actions, time.Second/2, func(page *Page, err error, out map[string]string) { + require.Error(t, err) + require.Contains(t, err.Error(), "Element did not appear in the given amount of time") + }) + }) +} + +func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) { + t.Helper() + testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, response) + }, assert) +} + +func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { + t.Helper() _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) @@ -318,19 +521,17 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, response) - })) + ts := httptest.NewServer(http.HandlerFunc(handler)) defer ts.Close() parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse URL") + extractedData, page, err := instance.Run(parsed, actions, timeout) + assert(page, err, extractedData) - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + if page != nil { + page.Close() } - _, page, err := instance.Run(parsed, actions, timeout) - assert(page, err) } diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index a254902eb..a802a64a9 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -8,6 +8,9 @@ import ( // routingRuleHandler handles proxy rule for actions related to request/response modification func (p *Page) routingRuleHandler(ctx *rod.Hijack) { + // usually browsers don't use chunked transfer encoding so we set the content-length nevertheless + ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) + for _, rule := range p.rules { if rule.Part != "request" { continue diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ee3e42ca5..ea83aea72 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -1,6 +1,7 @@ package http import ( + "bufio" "context" "fmt" "io" @@ -20,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" ) @@ -38,9 +40,22 @@ type generatedRequest struct { dynamicValues map[string]interface{} } +func (g *generatedRequest) URL() string { + if g.request != nil { + return g.request.URL.String() + } + if g.rawRequest != nil { + return g.rawRequest.FullURL + } + return "" +} + // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { + if r.request.SelfContained { + return r.makeSelfContainedRequest(dynamicValues, interactURL) + } // We get the next payload for the request. data, payloads, ok := r.nextValue() if !ok { @@ -48,6 +63,14 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() + if interactURL != "" { + data = r.options.Interactsh.ReplaceMarkers(data, interactURL) + + for payloadName, payloadValue := range payloads { + payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL) + } + } + parsed, err := url.Parse(baseURL) if err != nil { return nil, err @@ -55,22 +78,22 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa data, parsed = baseURLWithTemplatePrefs(data, parsed) - trailingSlash := false isRawRequest := len(r.request.Raw) > 0 + + // If the request is not a raw request, and the URL input path is suffixed with + // a trailing slash, and our Input URL is also suffixed with a trailing slash, + // mark trailingSlash bool as true which will be later used during variable generation + // to generate correct path removed slash which would otherwise generate // invalid sequence. + // TODO: Figure out a cleaner way to do this sanitization. + trailingSlash := false if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { trailingSlash = true } - values := generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)) - // merge with vars - if !r.options.Options.Vars.IsEmpty() { - values = generators.MergeMaps(values, r.options.Options.Vars.AsMap()) - } - - // merge with env vars - if r.options.Options.EnvironmentVariables { - values = generators.MergeMaps(generators.EnvVars(), values) - } + values := generators.MergeMaps( + generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)), + generators.BuildPayloadFromOptions(r.request.options.Options), + ) // If data contains \n it's a raw request, process it like raw. Else // continue with the template based request flow. @@ -80,6 +103,48 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) } +func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { + // We get the next payload for the request. + data, payloads, ok := r.nextValue() + if !ok { + return nil, io.EOF + } + ctx := context.Background() + + isRawRequest := r.request.isRaw() + + // If the request is a raw request, get the URL from the request + // header and use it to make the request. + if isRawRequest { + // Get the hostname from the URL section to build the request. + reader := bufio.NewReader(strings.NewReader(data)) + s, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("could not read request: %s", err) + } + + parts := strings.Split(s, " ") + if len(parts) < 3 { + return nil, fmt.Errorf("malformed request supplied") + } + parsed, err := url.Parse(parts[1]) + if err != nil { + return nil, fmt.Errorf("could not parse request URL: %s", err) + } + values := generators.MergeMaps( + generators.MergeMaps(dynamicValues, generateVariables(parsed, false)), + generators.BuildPayloadFromOptions(r.request.options.Options), + ) + + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + } + values := generators.MergeMaps( + dynamicValues, + generators.BuildPayloadFromOptions(r.request.options.Options), + ) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) +} + // Total returns the total number of requests for the generator func (r *requestGenerator) Total() int { if r.payloadIterator != nil { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 67dc173b3..304f1a497 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -46,13 +46,13 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: - // - "sniper" + // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. // values: @@ -137,6 +137,10 @@ type Request struct { rawhttpClient *rawhttp.Client dynamicValues map[string]interface{} + // description: | + // SelfContained specifies if the request is self contained. + SelfContained bool `yaml:"-" json:"-"` + // description: | // CookieReuse is an optional setting that enables cookie reuse for // all requests defined in raw section. @@ -180,6 +184,10 @@ func (request *Request) GetID() string { return request.ID } +func (request *Request) isRaw() bool { + return len(request.Raw) > 0 +} + // Compile compiles the protocol request for further execution. func (request *Request) Compile(options *protocols.ExecuterOptions) error { connectionConfiguration := &httpclientpool.Configuration{ diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index c91ee0ec1..9ae939cf6 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -147,6 +147,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), + CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), } return data } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 582cd1351..2b3d44af5 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -90,8 +90,10 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { return nil, fmt.Errorf("could not parse request URL: %s", parseErr) } - rawRequest.Path = parts[1] - rawRequest.Headers["Host"] = parsed.Host + rawRequest.Path = parsed.Path + if _, ok := rawRequest.Headers["Host"]; !ok { + rawRequest.Headers["Host"] = parsed.Host + } } else if len(parts) > 1 { rawRequest.Path = parts[1] } @@ -104,7 +106,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") { parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/") } - rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + if parsedURL.Path != rawRequest.Path { + rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + } if strings.HasSuffix(rawRequest.Path, "//") { rawRequest.Path = strings.TrimSuffix(rawRequest.Path, "/") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 979e0eae2..3451e5f3d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" + "moul.io/http2curl" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -100,6 +101,9 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err @@ -160,6 +164,9 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err @@ -215,6 +222,9 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err @@ -373,6 +383,16 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate resp.Body.Close() }() + var curlCommand string + if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil { + bodyBytes, _ := generatedRequest.request.BodyBytes() + resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + command, _ := http2curl.GetCurlCommand(resp.Request) + if err == nil && command != nil { + curlCommand = command.String() + } + } + gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) @@ -429,7 +449,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data) // Decode gbk response content-types - if contentType := strings.ToLower(resp.Header.Get("Content-Type")); contentType != "" && (strings.Contains(contentType, "gbk") || strings.Contains(contentType, "gb2312")) { + // gb18030 supersedes gb2312 + if isContentTypeGbk(resp.Header.Get("Content-Type")) { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") @@ -438,6 +459,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if err != nil { return errors.Wrap(err, "could not gbk decode") } + + // the uncompressed body needs to be decoded to standard utf8 + data, err = decodegbk(data) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } } // if nuclei-project is enabled store the response if not previously done @@ -460,6 +487,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } + outputEvent["curl-command"] = curlCommand outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) for k, v := range previousEvent { diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index bb12b5a66..89c35a233 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) @@ -135,3 +136,9 @@ func decodegbk(s []byte) ([]byte, error) { } return d, nil } + +// isContentTypeGbk checks if the content-type header is gbk +func isContentTypeGbk(contentType string) bool { + contentType = strings.ToLower(contentType) + return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") +} diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index e54a66849..12094ae6e 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -35,13 +35,13 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: - // - "sniper" + // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // @@ -60,6 +60,17 @@ type Request struct { // examples: // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | + // ReadAll determines if the data stream should be read till the end regardless of the size + // + // Default value for read-all is false. + // examples: + // - value: false + ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` + + // description: | + // SelfContained specifies if the request is self contained. + SelfContained bool `yaml:"-" json:"-"` // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ba996da8c..66da31e9b 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -6,6 +6,7 @@ import ( "io" "net" "net/url" + "os" "strings" "time" @@ -26,7 +27,14 @@ var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - address, err := getAddress(input) + var address string + var err error + + if request.SelfContained { + address = "" + } else { + address, err = getAddress(input) + } if err != nil { request.options.Output.Request(request.options.TemplateID, input, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) @@ -41,6 +49,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } actualAddress = net.JoinHostPort(actualAddress, kv.port) } + if input != "" { + input = actualAddress + } if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) @@ -59,6 +70,8 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho return err } + payloads := generators.BuildPayloadFromOptions(request.options.Options) + if request.generator != nil { iterator := request.generator.NewIterator() @@ -67,12 +80,13 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho if !ok { break } + value = generators.MergeMaps(value, payloads) if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { - value := make(map[string]interface{}) + value := generators.MergeMaps(map[string]interface{}{}, payloads) if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } @@ -86,6 +100,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input conn net.Conn err error ) + request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { @@ -186,13 +201,48 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input if request.ReadSize != 0 { bufferSize = request.ReadSize } - final := make([]byte, bufferSize) - n, err := conn.Read(final) - if err != nil && err != io.EOF { - request.options.Output.Request(request.options.TemplateID, address, "network", err) - return errors.Wrap(err, "could not read from server") + + var ( + final []byte + n int + ) + + if request.ReadAll { + readInterval := time.NewTimer(time.Second * 1) + // stop the timer and drain the channel + closeTimer := func(t *time.Timer) { + if !t.Stop() { + <-t.C + } + } + read_socket: + for { + select { + case <-readInterval.C: + closeTimer(readInterval) + break read_socket + default: + buf := make([]byte, bufferSize) + nBuf, err := conn.Read(buf) + if err != nil && !os.IsTimeout(err) { + request.options.Output.Request(request.options.TemplateID, address, "network", err) + closeTimer(readInterval) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(buf[:nBuf]) + final = append(final, buf...) + n += nBuf + } + } + } else { + final = make([]byte, bufferSize) + n, err = conn.Read(final) + if err != nil && err != io.EOF { + request.options.Output.Request(request.options.TemplateID, address, "network", err) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(final[:n]) } - responseBuilder.Write(final[:n]) response := responseBuilder.String() outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 59d23470d..b5141037b 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -98,6 +98,13 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw for k, v := range extra { data[k] = v } + for _, cookie := range resp.Cookies() { + data[strings.ToLower(cookie.Name)] = cookie.Value + } + for k, v := range resp.Header { + k = strings.ToLower(strings.TrimSpace(k)) + data[k] = strings.Join(v, " ") + } data["path"] = host data["matched"] = matched @@ -106,13 +113,6 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode data["body"] = body - for _, cookie := range resp.Cookies() { - data[strings.ToLower(cookie.Name)] = cookie.Value - } - for k, v := range resp.Header { - k = strings.ToLower(strings.TrimSpace(k)) - data[k] = strings.Join(v, " ") - } data["all_headers"] = headers data["duration"] = duration.Seconds() data["template-id"] = request.options.TemplateID diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index cf625eb66..784e587d0 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "time" "encoding/base64" @@ -22,9 +23,9 @@ type Options struct { IP string `yaml:"ip"` // Port is the port of elasticsearch instance Port int `yaml:"port"` - // SSL enables ssl for elasticsearch connection + // SSL (optional) enables ssl for elasticsearch connection SSL bool `yaml:"ssl"` - // SSLVerification disables SSL verification for elasticsearch + // SSLVerification (optional) disables SSL verification for elasticsearch SSLVerification bool `yaml:"ssl-verification"` // Username for the elasticsearch instance Username string `yaml:"username"` @@ -49,6 +50,10 @@ type Exporter struct { // New creates and returns a new exporter for elasticsearch func New(option *Options) (*Exporter, error) { var ei *Exporter + err := validateOptions(option) + if err != nil { + return nil, err + } client := &http.Client{ Timeout: 5 * time.Second, @@ -81,6 +86,31 @@ func New(option *Options) (*Exporter, error) { return ei, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.IP == "" { + errs = append(errs, "IP") + } + if options.Port == 0 { + errs = append(errs, "Port") + } + if options.Username == "" { + errs = append(errs, "Username") + } + if options.Password == "" { + errs = append(errs, "Password") + } + if options.IndexName == "" { + errs = append(errs, "IndexName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // Export exports a passed result event to elasticsearch func (i *Exporter) Export(event *output.ResultEvent) error { // creating a request @@ -105,9 +135,9 @@ func (i *Exporter) Export(event *output.ResultEvent) error { res, err := i.elasticsearch.Do(req) if err != nil { - return err + return err } - + b, err = ioutil.ReadAll(res.Body) if err != nil { return errors.New(err.Error() + "error thrown by elasticsearch " + string(b)) diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index e1a0f8995..4f9d3e526 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -131,6 +131,13 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the } } } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n**CURL Command**\n```\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n```") + } builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 5074ac9c7..8d6cdb864 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "strings" "golang.org/x/oauth2" @@ -22,22 +23,29 @@ type Integration struct { // Options contains the configuration options for github issue tracker client type Options struct { - // BaseURL is the optional self-hosted github application url + // BaseURL (optional) is the self-hosted github application url BaseURL string `yaml:"base-url"` // Username is the username of the github user Username string `yaml:"username"` - // Owner is the owner name of the repository for issues. + // Owner (manadatory) is the owner name of the repository for issues. Owner string `yaml:"owner"` // Token is the token for github account. Token string `yaml:"token"` // ProjectName is the name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel is the label of the created issue type + // IssueLabel (optional) is the label of the created issue type IssueLabel string `yaml:"issue-label"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, err + } ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: options.Token}, @@ -50,21 +58,53 @@ func New(options *Options) (*Integration, error) { if err != nil { return nil, errors.Wrap(err, "could not parse custom baseurl") } + if !strings.HasSuffix(parsed.Path, "/") { + parsed.Path += "/" + } client.BaseURL = parsed } return &Integration{client: client, options: options}, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.Username == "" { + errs = append(errs, "Username") + } + if options.Owner == "" { + errs = append(errs, "Owner") + } + if options.Token == "" { + errs = append(errs, "Token") + } + if options.ProjectName == "" { + errs = append(errs, "ProjectName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueLabel; label != "" { + labels = append(labels, label) + } req := &github.IssueRequest{ Title: &summary, Body: &description, - Labels: &[]string{i.options.IssueLabel, severityLabel}, + Labels: &labels, Assignees: &[]string{i.options.Username}, } _, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index c02479662..35922970b 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -2,6 +2,9 @@ package gitlab import ( "fmt" + "strings" + + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" @@ -17,7 +20,7 @@ type Integration struct { // Options contains the configuration options for gitlab issue tracker client type Options struct { - // BaseURL is the optional self-hosted gitlab application url + // BaseURL (optional) is the self-hosted gitlab application url BaseURL string `yaml:"base-url"` // Username is the username of the gitlab user Username string `yaml:"username"` @@ -27,10 +30,17 @@ type Options struct { ProjectName string `yaml:"project-name"` // IssueLabel is the label of the created issue type IssueLabel string `yaml:"issue-label"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, err + } gitlabOpts := []gitlab.ClientOptionFunc{} if options.BaseURL != "" { gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) @@ -46,16 +56,42 @@ func New(options *Options) (*Integration, error) { return &Integration{client: git, userID: user.ID, options: options}, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.Username == "" { + errs = append(errs, "Username") + } + if options.Token == "" { + errs = append(errs, "Token") + } + if options.ProjectName == "" { + errs = append(errs, "ProjectName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueLabel; label != "" { + labels = append(labels, label) + } _, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{ Title: &summary, Description: &description, - Labels: gitlab.Labels{i.options.IssueLabel, severityLabel}, + Labels: labels, AssigneeIDs: []int{i.userID}, }) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 337ba3f5d..3237e929b 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -2,6 +2,7 @@ package jira import ( "bytes" + "errors" "fmt" "io/ioutil" "strings" @@ -22,9 +23,9 @@ type Integration struct { // Options contains the configuration options for jira client type Options struct { - // Cloud value is set to true when Jira cloud is used + // Cloud value (optional) is set to true when Jira cloud is used Cloud bool `yaml:"cloud"` - // UpdateExisting value if true, the existing opened issue is updated + // UpdateExisting value (optional) if true, the existing opened issue is updated UpdateExisting bool `yaml:"update-existing"` // URL is the URL of the jira server URL string `yaml:"url"` @@ -36,12 +37,19 @@ type Options struct { Token string `yaml:"token"` // ProjectName is the name of the project. ProjectName string `yaml:"project-name"` - // IssueType is the name of the created issue type + // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, err + } username := options.Email if !options.Cloud { username = options.AccountID @@ -57,10 +65,42 @@ func New(options *Options) (*Integration, error) { return &Integration{jira: jiraClient, options: options}, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.URL == "" { + errs = append(errs, "URL") + } + if options.AccountID == "" { + errs = append(errs, "AccountID") + } + if options.Email == "" { + errs = append(errs, "Email") + } + if options.Token == "" { + errs = append(errs, "Token") + } + if options.ProjectName == "" { + errs = append(errs, "ProjectName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // CreateNewIssue creates a new issue in the tracker func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) - severityLabel := fmt.Sprintf("Severity:%s", event.Info.SeverityHolder.Severity.String()) + labels := []string{} + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueType; label != "" { + labels = append(labels, label) + } fields := &jira.IssueFields{ Assignee: &jira.User{AccountID: i.options.AccountID}, @@ -69,7 +109,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, - Labels: []string{severityLabel}, + Labels: labels, } // On-prem version of Jira server does not use AccountID if !i.options.Cloud { @@ -244,6 +284,13 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th } } } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n*CURL Command*\n{code}\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n{code}") + } builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei v%s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() return data diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index d7f545bc3..bf4e79b18 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -144,6 +144,21 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } template.Path = filePath + template.parseSelfContainedRequests() + parsedTemplatesCache.Store(filePath, template, err) return template, nil } + +// parseSelfContainedRequests parses the self contained template requests. +func (t *Template) parseSelfContainedRequests() { + if !t.SelfContained { + return + } + for _, request := range t.RequestsHTTP { + request.SelfContained = true + } + for _, request := range t.RequestsNetwork { + request.SelfContained = true + } +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491ac..61cb49ff1 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` // description: | // Info contains metadata information about the template. // examples: @@ -62,6 +62,10 @@ type Template struct { workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"` CompiledWorkflow *workflows.Workflow `yaml:"-" json:"-" jsonschema:"-"` + // description: | + // Self Contained marks Requests for the template as self-contained + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` // Executer is the actual template executor for running template requests diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 30ac25af7..d6042dfbd 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -31,7 +31,7 @@ func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 8) + TemplateDoc.Fields = make([]encoder.Doc, 9) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -84,6 +84,11 @@ func init() { TemplateDoc.Fields[7].Note = "" TemplateDoc.Fields[7].Description = "Workflows is a list of workflows to execute for a template." TemplateDoc.Fields[7].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[8].Name = "self-contained" + TemplateDoc.Fields[8].Type = "bool" + TemplateDoc.Fields[8].Note = "" + TemplateDoc.Fields[8].Description = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -317,10 +322,10 @@ func init() { HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "string" HTTPRequestDoc.Fields[7].Note = "" - HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." HTTPRequestDoc.Fields[7].Values = []string{ - "sniper", + "batteringram", "pitchfork", "clusterbomb", } @@ -838,7 +843,7 @@ func init() { FieldName: "network", }, } - NETWORKRequestDoc.Fields = make([]encoder.Doc, 9) + NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" NETWORKRequestDoc.Fields[0].Note = "" @@ -854,10 +859,10 @@ func init() { NETWORKRequestDoc.Fields[2].Name = "attack" NETWORKRequestDoc.Fields[2].Type = "string" NETWORKRequestDoc.Fields[2].Note = "" - NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." NETWORKRequestDoc.Fields[2].Values = []string{ - "sniper", + "batteringram", "pitchfork", "clusterbomb", } @@ -878,22 +883,29 @@ func init() { NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end" NETWORKRequestDoc.Fields[5].AddExample("", 2048) - NETWORKRequestDoc.Fields[6].Name = "matchers" - NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher" + NETWORKRequestDoc.Fields[6].Name = "read-all" + NETWORKRequestDoc.Fields[6].Type = "bool" NETWORKRequestDoc.Fields[6].Note = "" - NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." - NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" - NETWORKRequestDoc.Fields[7].Name = "extractors" - NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor" + NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false." + NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size" + + NETWORKRequestDoc.Fields[6].AddExample("", false) + NETWORKRequestDoc.Fields[7].Name = "matchers" + NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher" NETWORKRequestDoc.Fields[7].Note = "" - NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." - NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" - NETWORKRequestDoc.Fields[8].Name = "matchers-condition" - NETWORKRequestDoc.Fields[8].Type = "string" + NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + NETWORKRequestDoc.Fields[8].Name = "extractors" + NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor" NETWORKRequestDoc.Fields[8].Note = "" - NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Values = []string{ + NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + NETWORKRequestDoc.Fields[9].Name = "matchers-condition" + NETWORKRequestDoc.Fields[9].Type = "string" + NETWORKRequestDoc.Fields[9].Note = "" + NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Values = []string{ "and", "or", } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index b5762a861..4a593f169 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -150,6 +150,8 @@ type Options struct { Stdin bool // StopAtFirstMatch stops processing template at first full match (this may break chained requests) StopAtFirstMatch bool + // Stream the input without sorting + Stream bool // NoMeta disables display of metadata for the matches NoMeta bool // NoTimestamp disables display of timestamp for the matcher