diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fddde924e..dbe5721d0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,18 +6,6 @@ version: 2 updates: - # Maintain dependencies for GitHub Actions - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - target-branch: "dev" - commit-message: - prefix: "chore" - include: "scope" - labels: - - "Type: Maintenance" - # Maintain dependencies for go modules - package-ecosystem: "gomod" directory: "v2/" @@ -29,15 +17,29 @@ updates: include: "scope" labels: - "Type: Maintenance" + allow: + - dependency-name: "github.com/projectdiscovery/*" - # Maintain dependencies for docker - - package-ecosystem: "docker" - directory: "/" - schedule: - interval: "weekly" - target-branch: "dev" - commit-message: - prefix: "chore" - include: "scope" - labels: - - "Type: Maintenance" +# # Maintain dependencies for GitHub Actions +# - package-ecosystem: "github-actions" +# directory: "/" +# schedule: +# interval: "weekly" +# target-branch: "dev" +# commit-message: +# prefix: "chore" +# include: "scope" +# labels: +# - "Type: Maintenance" +# +# # Maintain dependencies for docker +# - package-ecosystem: "docker" +# directory: "/" +# schedule: +# interval: "weekly" +# target-branch: "dev" +# commit-message: +# prefix: "chore" +# include: "scope" +# labels: +# - "Type: Maintenance" diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..01b4c9d33 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,17 @@ +changelog: + exclude: + authors: + - dependabot + categories: + - title: 🎉 Features + labels: + - "Type: Enhancement" + - title: 🐞 Bugs + labels: + - "Type: Bug" + - title: 🔨 Maintenance + labels: + - "Type: Maintenance" + - title: Other Changes + labels: + - "*" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 599940d7a..8173b810d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -25,7 +25,7 @@ jobs: - name: Check out code uses: actions/checkout@v3 - - name: Go Mod hygine + - name: Go Mod hygiene run: | go clean -modcache go mod tidy diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml new file mode 100644 index 000000000..84b26e1fe --- /dev/null +++ b/.github/workflows/dep-auto-merge.yml @@ -0,0 +1,26 @@ +name: 🤖 dep auto merge + +on: + pull_request: + branches: + - dev + workflow_dispatch: + +permissions: + pull-requests: write + issues: write + repository-projects: write + +jobs: + automerge: + runs-on: ubuntu-latest + if: github.actor == 'dependabot[bot]' + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.DEPENDABOT_PAT }} + + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + github-token: ${{ secrets.DEPENDABOT_PAT }} + target: all \ No newline at end of file diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 428add3e4..da0887e79 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -14,7 +14,7 @@ jobs: - name: Git Checkout uses: actions/checkout@v3 - - name: Get Github tag + - name: Get GitHub tag id: meta run: | curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..ad05027e8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,30 @@ +# Contributing to ProjectDiscovery/Nuclei + +We appreciate your interest in contributing to the projectdiscovery/nuclei! This document provides some basic guidelines for contributors. + +## Getting Started + +- Always base your work from the `dev` branch, which is the development branch with the latest code. +- Before creating a Pull Request (PR), make sure there is a corresponding issue for your contribution. If there isn't one already, please create one. +- Include the problem description in the issue. + +## Pull Requests + +When creating a PR, please follow these guidelines: + +- Link your PR to the corresponding issue. +- Provide context in the PR description to help reviewers understand the changes. The more information you provide, the faster the review process will be. +- Include an example of running the tool with the changed code, if applicable. Provide 'before' and 'after' examples if possible. +- Include steps for functional testing or replication. +- If you're adding a new feature, make sure to include unit tests. + +## Code Style + +Please adhere to the existing coding style for consistency. + +## Questions + +If you have any questions or need further guidance, please feel free to ask in the issue or PR, or [reach out to the maintainers](https://discord.gg/projectdiscovery). + +Thank you for your contribution! + diff --git a/DEBUG.md b/DEBUG.md new file mode 100644 index 000000000..2fbf6b56d --- /dev/null +++ b/DEBUG.md @@ -0,0 +1,44 @@ +## Debugging Nuclei + +While Adding new features or fixing bugs or writing new templates to properly understand the behavior of that component, it is essential to understand what debugging options are available in nuclei. This guide lists all the debugging options available in nuclei. + +### Template related debugging + +- `-debug` flag + +When this flag is provided, nuclei will print all requests that are being sent by nuclei to the target as well as the response received from the target. + +- `-debug-req` flag + +When this flag is provided, nuclei will print all requests that are being sent by nuclei to the target. + +- `-debug-resp` flag + +When this flag is provided, nuclei will print all responses that are being received by nuclei from the target. + +- `-ldf` flag + +When this flag is provided, nuclei will print the list of all helper functions available in this release of nuclei and exit. + +- `-svd` flag + +When this flag is provided, nuclei will print all `variables` pre and post execution of a request for a template. This is useful to understand what variables are available for a template and what values they have. + +- `-elog = errors.txt` flag + +When this flag is provided, nuclei will log all errors to the file specified. This is helpful when running large scans. + + + +### Environment Variable Switches + +Nuclei was built with some environment variables in mind to help with debugging. These environment variables can be set to enable debugging of a particular component/functionality for nuclei. + +| Environment Variable | Description | +| -------------------------------- | -------------------------------------------------------- | +| `DEBUG=true` | Enables Printing Stack Traces for all errors | +| `SHOW_DSL_ERRORS=true` | Enables Printing DSL Errors (that are hidden by default) | +| `HIDE_TEMPLATE_SIG_WARNING=true` | Hides Template Signature Verification Warnings | + + + diff --git a/DESIGN.md b/DESIGN.md index 969dc84ed..b02067f24 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -145,7 +145,7 @@ type Exporter interface { } ``` -Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `Gitlab` and `Jira`. +Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `GitLab` and `Jira`. Each exporter and trackers implement their own configuration in YAML format and are very modular in nature, so adding new ones is easy. @@ -484,7 +484,7 @@ $ go tool pprof -http=:8081 mem.pprof - [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter - [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/es) - Elasticsearch Result Exporter - [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results -- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter +- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - GitLab Issue Tracker Exporter - [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter - [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - GitHub Issue Tracker Exporter - [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions diff --git a/Dockerfile b/Dockerfile index d75c1c17c..23a40a61e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.20.5-alpine AS build-env +FROM golang:1.20.6-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app diff --git a/README.md b/README.md index d2a53ca88..06845dcdc 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,6 @@ Nuclei is used to send requests across targets based on a template, leading to z We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 300** security researchers and engineers. - - ## How it works @@ -51,6 +49,11 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp +| :exclamation: **Disclaimer** | +|---------------------------------| +| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. | +| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. | + # Install Nuclei Nuclei requires **go1.20** to install successfully. Run the following command to install the latest version - @@ -136,7 +139,7 @@ FILTERING: -tags string[] templates to run based on tags (comma-separated, file) -etags, -exclude-tags string[] templates to exclude based on tags (comma-separated, file) -itags, -include-tags string[] tags to be executed even if they are excluded either by default or configuration - -id, -template-id string[] templates to run based on template ids (comma-separated, file) + -id, -template-id string[] templates to run based on template ids (comma-separated, file, allow-wildcard) -eid, -exclude-id string[] templates to exclude based on template ids (comma-separated, file) -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration -et, -exclude-templates string[] template or template directory to exclude (comma-separated, file) @@ -154,7 +157,8 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSONL output (for findings only) + -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true) + -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) -nm, -no-meta disable printing result metadata in cli output -ts, -timestamp enables printing timestamp in cli output -rdb, -report-db string nuclei reporting database (always use this to persist report data) @@ -165,35 +169,36 @@ OUTPUT: -jle, -jsonl-export string file to export results in JSONL(ine) format CONFIGURATIONS: - -config string path to the nuclei configuration file - -fr, -follow-redirects enable following redirects for http templates - -fhr, -follow-host-redirects follow redirects on the same host - -mr, -max-redirects int max number of redirects to follow for http templates (default 10) - -dr, -disable-redirects disable redirects for http templates - -rc, -report-config string nuclei reporting module configuration file - -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) - -V, -var value custom vars in key=value format - -r, -resolvers string file containing resolver list for nuclei - -sr, -system-resolvers use system DNS resolving as error fallback - -dc, -disable-clustering disable clustering of requests - -passive enable passive HTTP response processing mode - -fh2, -force-http2 force http2 connection on requests - -ev, -env-vars enable environment variables to be used in template - -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts - -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts - -sml, -show-match-line show match lines for file templates, works with extractors only - -ztls use ztls library with autofallback to standard one for tls13 - -sni string tls sni hostname to use (default: input domain name) - -sandbox sandbox nuclei for safe templates execution - -i, -interface string network interface to use for network scan - -at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb) - -sip, -source-ip string source ip address to use for network scan - -config-directory string override the default config path ($home/.config) - -rsr, -response-size-read int max response size to read in bytes (default 10485760) - -rss, -response-size-save int max response size to read in bytes (default 1048576) - -reset reset removes all nuclei configuration and data files (including nuclei-templates) - -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization + -config string path to the nuclei configuration file + -fr, -follow-redirects enable following redirects for http templates + -fhr, -follow-host-redirects follow redirects on the same host + -mr, -max-redirects int max number of redirects to follow for http templates (default 10) + -dr, -disable-redirects disable redirects for http templates + -rc, -report-config string nuclei reporting module configuration file + -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) + -V, -var value custom vars in key=value format + -r, -resolvers string file containing resolver list for nuclei + -sr, -system-resolvers use system DNS resolving as error fallback + -dc, -disable-clustering disable clustering of requests + -passive enable passive HTTP response processing mode + -fh2, -force-http2 force http2 connection on requests + -ev, -env-vars enable environment variables to be used in template + -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -sml, -show-match-line show match lines for file templates, works with extractors only + -ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default + -sni string tls sni hostname to use (default: input domain name) + -lfa, -allow-local-file-access allows file (payload) access anywhere on the system + -lna, -restrict-local-network-access blocks connections to the local / private network + -i, -interface string network interface to use for network scan + -at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb) + -sip, -source-ip string source ip address to use for network scan + -config-directory string override the default config path ($home/.config) + -rsr, -response-size-read int max response size to read in bytes (default 10485760) + -rss, -response-size-save int max response size to read in bytes (default 1048576) + -reset reset removes all nuclei configuration and data files (including nuclei-templates) + -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) @@ -211,10 +216,10 @@ FUZZING: UNCOVER: -uc, -uncover enable uncover engine -uq, -uncover-query string[] uncover search query - -ue, -uncover-engine string[] uncover search engine (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan) + -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan) -uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port") -ul, -uncover-limit int uncover results to return (default 100) - -ucd, -uncover-delay int delay between uncover query requests in seconds (0 to disable) (default 1) + -ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60) RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) @@ -241,11 +246,12 @@ OPTIMIZATIONS: -no-stdin disable stdin processing HEADLESS: - -headless enable templates that require headless browser support (root user on Linux will disable sandbox) - -page-timeout int seconds to wait for each page in headless mode (default 20) - -sb, -show-browser show the browser on the screen when running templates with headless mode - -sc, -system-chrome use local installed Chrome browser instead of nuclei installed - -lha, -list-headless-action list available headless actions + -headless enable templates that require headless browser support (root user on Linux will disable sandbox) + -page-timeout int seconds to wait for each page in headless mode (default 20) + -sb, -show-browser show the browser on the screen when running templates with headless mode + -ho, -headless-options string[] start headless chrome with additional options + -sc, -system-chrome use local installed Chrome browser instead of nuclei installed + -lha, -list-headless-action list available headless actions DEBUG: -debug show all requests and responses @@ -278,29 +284,6 @@ STATISTICS: -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) -m, -metrics expose nuclei metrics on a port -mp, -metrics-port int port to expose nuclei metrics on (default 9092) - -CLOUD: - -cloud run scan on nuclei cloud - -ads, -add-datasource string add specified data source (s3,github) - -atr, -add-target string add target(s) to cloud - -atm, -add-template string add template(s) to cloud - -lsn, -list-scan list previous cloud scans - -lso, -list-output string list scan output by scan id - -ltr, -list-target list cloud target by id - -ltm, -list-template list cloud template by id - -lds, -list-datasource list cloud datasource by id - -lrs, -list-reportsource list reporting sources - -dsn, -delete-scan string delete cloud scan by id - -dtr, -delete-target string delete target(s) from cloud - -dtm, -delete-template string delete template(s) from cloud - -dds, -delete-datasource string delete specified data source - -drs, -disable-reportsource string disable specified reporting source - -ers, -enable-reportsource string enable specified reporting source - -gtr, -get-target string get target content by id - -gtm, -get-template string get template content by id - -nos, -no-store disable scan/output storage on cloud - -no-tables do not display pretty-printed tables - -limit int limit the number of output to display (default 100) ``` ### Running Nuclei diff --git a/README_CN.md b/README_CN.md index f2fd13997..c8c8ffa4b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -134,6 +134,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板 -nc, -no-color 禁用输出内容着色（ANSI转义码） -j, -jsonl 输出为jsonL（ines） -irr, -include-rr 在JSONL中输出对应的请求和相应（仅结果） + -or, -omit-raw -nm, -no-meta 不显示匹配的元数据 -nts, -no-timestamp 不在输出中显示时间戳 -rdb, -report-db string 本地的Nuclei结果数据库（始终使用该数据库保存结果） diff --git a/README_ID.md b/README_ID.md index ac24b0d77..04fcd3e8c 100644 --- a/README_ID.md +++ b/README_ID.md @@ -90,13 +90,15 @@ Nuclei is a fast, template based vulnerability scanner focusing on extensive configurability, massive extensibility and ease of use. Usage: - nuclei [flags] + ./nuclei [flags] Flags: TARGET: - -u, -target string[] target URLs/hosts to scan - -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) - -resume string Resume scan using resume.cfg (clustering will be disabled) + -u, -target string[] target URLs/hosts to scan + -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) + -resume string resume scan using resume.cfg (clustering will be disabled) + -sa, -scan-all-ips scan all the IP's associated with dns record + -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TEMPLATES: -nt, -new-templates run only new templates added in latest nuclei-templates release @@ -107,7 +109,8 @@ TEMPLATES: -w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file) -wu, -workflow-url string[] list of workflow urls to run (comma-separated, file) -validate validate the passed templates to nuclei - -nss, -no-strict-syntax Disable strict syntax check on templates + -nss, -no-strict-syntax disable strict syntax check on templates + -td, -template-display displays the templates content -tl list all available templates FILTERING: @@ -122,8 +125,8 @@ FILTERING: -em, -exclude-matchers string[] template matchers to exclude in result -s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical, unknown -es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown - -pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois - -ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois + -pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois + -ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois -tc, -template-condition string[] templates to run based on expression condition OUTPUT: @@ -133,40 +136,48 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSONL output (for findings only) + -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true) + -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) -nm, -no-meta disable printing result metadata in cli output - -nts, -no-timestamp disable printing timestamp in cli output + -ts, -timestamp enables printing timestamp in cli output -rdb, -report-db string nuclei reporting database (always use this to persist report data) -ms, -matcher-status display match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format - -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans - -jle, -jsonl-export string file to export results in JSONL(ine) format as a list of line-delimited JSON objects + -je, -json-export string file to export results in JSON format + -jle, -jsonl-export string file to export results in JSONL(ine) format CONFIGURATIONS: - -config string path to the nuclei configuration file - -fr, -follow-redirects enable following redirects for http templates - -fhr, -follow-host-redirects follow redirects on the same host - -mr, -max-redirects int max number of redirects to follow for http templates (default 10) - -dr, -disable-redirects disable redirects for http templates - -rc, -report-config string nuclei reporting module configuration file - -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) - -V, -var value custom vars in key=value format - -r, -resolvers string file containing resolver list for nuclei - -sr, -system-resolvers use system DNS resolving as error fallback - -passive enable passive HTTP response processing mode - -ev, -env-vars enable environment variables to be used in template - -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts - -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts - -sml, -show-match-line show match lines for file templates, works with extractors only - -ztls use ztls library with autofallback to standard one for tls13 - -sni string tls sni hostname to use (default: input domain name) - -i, -interface string network interface to use for network scan - -sip, -source-ip string source ip address to use for network scan - -config-directory string Override the default config path ($home/.config) - -rsr, -response-size-read int max response size to read in bytes (default 10485760) - -rss, -response-size-save int max response size to save in bytes (default 10485760) + -config string path to the nuclei configuration file + -fr, -follow-redirects enable following redirects for http templates + -fhr, -follow-host-redirects follow redirects on the same host + -mr, -max-redirects int max number of redirects to follow for http templates (default 10) + -dr, -disable-redirects disable redirects for http templates + -rc, -report-config string nuclei reporting module configuration file + -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) + -V, -var value custom vars in key=value format + -r, -resolvers string file containing resolver list for nuclei + -sr, -system-resolvers use system DNS resolving as error fallback + -dc, -disable-clustering disable clustering of requests + -passive enable passive HTTP response processing mode + -fh2, -force-http2 force http2 connection on requests + -ev, -env-vars enable environment variables to be used in template + -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -sml, -show-match-line show match lines for file templates, works with extractors only + -ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default + -sni string tls sni hostname to use (default: input domain name) + -lfa, -allow-local-file-access allows file (payload) access anywhere on the system + -lna, -restrict-local-network-access blocks connections to the local / private network + -i, -interface string network interface to use for network scan + -at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb) + -sip, -source-ip string source ip address to use for network scan + -config-directory string override the default config path ($home/.config) + -rsr, -response-size-read int max response size to read in bytes (default 10485760) + -rss, -response-size-save int max response size to read in bytes (default 1048576) + -reset reset removes all nuclei configuration and data files (including nuclei-templates) + -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) @@ -177,6 +188,18 @@ INTERACTSH: -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates +FUZZING: + -ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix) + -fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single) + +UNCOVER: + -uc, -uncover enable uncover engine + -uq, -uncover-query string[] uncover search query + -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan) + -uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port") + -ul, -uncover-limit int uncover results to return (default 100) + -ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60) + RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) -rlm, -rate-limit-minute int maximum number of requests to send per minute @@ -188,22 +211,24 @@ RATE-LIMIT: OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 10) -retries int number of times to retry a failed request (default 1) - -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443 + -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443) -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) -te, -track-error string[] adds given error to max-host-error watchlist (standard, file) -nmhe, -no-mhe disable skipping host from scan based on errors -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path - -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + -project-path string set a specific project path (default "/tmp") + -spm, -stop-at-first-match stop processing HTTP requests after the first match (may break template/workflow logic) -stream stream mode - start elaborating without sorting the input + -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout duration timeout on input read (default 3m0s) - -no-stdin Disable Stdin processing + -nh, -no-httpx disable httpx probing for non-url input + -no-stdin disable stdin processing HEADLESS: - -headless enable templates that require headless browser support (root user on linux will disable sandbox) + -headless enable templates that require headless browser support (root user on Linux will disable sandbox) -page-timeout int seconds to wait for each page in headless mode (default 20) -sb, -show-browser show the browser on the screen when running templates with headless mode - -sc, -system-chrome Use local installed chrome browser instead of nuclei installed + -sc, -system-chrome use local installed Chrome browser instead of nuclei installed -lha, -list-headless-action list available headless actions DEBUG: @@ -220,22 +245,46 @@ DEBUG: -v, -verbose show verbose output -profile-mem string optional nuclei memory profile dump file -vv display templates loaded for scan + -svd, -show-var-dump show variables dump for debugging -ep, -enable-pprof enable pprof debugging server -tv, -templates-version shows the version of the installed nuclei-templates -hc, -health-check run diagnostic check up UPDATE: - -update update nuclei engine to the latest released version - -ut, -update-templates update nuclei-templates to latest released version - -ud, -update-directory string overwrite the default directory to install nuclei-templates - -duc, -disable-update-check disable automatic nuclei/templates update check + -up, -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-template-dir string custom directory to install / update nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan - -sj, -stats-json dispaly statistics in JSONL(ines) format + -sj, -stats-json display statistics in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) -m, -metrics expose nuclei metrics on a port -mp, -metrics-port int port to expose nuclei metrics on (default 9092) + +CLOUD: + -cloud run scan on nuclei cloud + -ads, -add-datasource string add specified data source (s3,github) + -atr, -add-target string add target(s) to cloud + -atm, -add-template string add template(s) to cloud + -lsn, -list-scan list previous cloud scans + -lso, -list-output string list scan output by scan id + -ltr, -list-target list cloud target by id + -ltm, -list-template list cloud template by id + -lds, -list-datasource list cloud datasource by id + -lrs, -list-reportsource list reporting sources + -dsn, -delete-scan string delete cloud scan by id + -dtr, -delete-target string delete target(s) from cloud + -dtm, -delete-template string delete template(s) from cloud + -dds, -delete-datasource string delete specified data source + -drs, -disable-reportsource string disable specified reporting source + -ers, -enable-reportsource string enable specified reporting source + -gtr, -get-target string get target content by id + -gtm, -get-template string get template content by id + -nos, -no-store disable scan/output storage on cloud + -no-tables do not display pretty-printed tables + -limit int limit the number of output to display (default 100) ``` ### Menjalankan Nuclei diff --git a/README_KR.md b/README_KR.md index 8ab6cfd75..9608083d6 100644 --- a/README_KR.md +++ b/README_KR.md @@ -130,6 +130,7 @@ OUTPUT: -nc, -no-color 출력 내용 색상 비활성화 (ANSI escape codes) -j, -jsonl JSONL(ines) 형식으로 출력 -irr, -include-rr JSONL 출력에 요청/응답 쌍 포함(결과만) + -or, -omit-raw -nm, -no-meta cli 출력에서 결과 메타데이터 출력 비활성화 -nts, -no-timestamp cli 출력에서 결과 타임스탬프 출력 비활성화 -rdb, -report-db string nuclei 보고 데이터베이스(보고서 데이터를 유지하려면 항상 이것을 사용) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index df18968e1..0f43e0c99 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -358,7 +358,7 @@ Variables contains any variables for the current request.
-Constants contains any scalar costant for the current template +Constants contains any scalar constant for the current template
@@ -825,6 +825,29 @@ epss-score: "0.42509" ``` + + +
+ +
+ +epss-percentile float64 + +
+
+ +EPSS Percentile for the template. + + + +Examples: + + +```yaml +epss-percentile: "0.42509" +``` + +
diff --git a/integration_tests/debug.sh b/integration_tests/debug.sh index c3a968a82..57bfd0382 100755 --- a/integration_tests/debug.sh +++ b/integration_tests/debug.sh @@ -8,7 +8,7 @@ if [ $1 = "-h" ]; then printf " \$ ./debug.sh http self\n\n" printf "3. To run all integration tests of 'x' protocol that contains 'y' in template name and pass extra args to nuclei:\n" printf " \$ ./debug.sh http self -svd -debug-req\n\n" - printf "nuclei binary is created everytime script is run but integration-test binary is not" + printf "nuclei binary is created every time script is run but integration-test binary is not" exit 0 fi diff --git a/integration_tests/dsl/hide-version-warning.yaml b/integration_tests/dsl/hide-version-warning.yaml new file mode 100644 index 000000000..32df2954f --- /dev/null +++ b/integration_tests/dsl/hide-version-warning.yaml @@ -0,0 +1,18 @@ +id: basic-example + +info: + name: Test HTTP Template + author: pdteam + severity: info + reference: | + test case for default behaviour of version warning (dsl parsing error) + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: dsl + dsl: + - compare_versions("GG", '< 4.8.5') \ No newline at end of file diff --git a/integration_tests/dsl/show-version-warning.yaml b/integration_tests/dsl/show-version-warning.yaml new file mode 100644 index 000000000..b0d5aa8b7 --- /dev/null +++ b/integration_tests/dsl/show-version-warning.yaml @@ -0,0 +1,18 @@ +id: basic-example + +info: + name: Test HTTP Template + author: pdteam + severity: info + reference: | + test case where version warning is shown when env `SHOW_DSL_ERRORS=true` is set + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: dsl + dsl: + - compare_versions("GG", '< 4.8.5') \ No newline at end of file diff --git a/integration_tests/generic/auth/certificate/assets/client.crt b/integration_tests/generic/auth/certificate/assets/client.crt index 4bbdee24b..8e92ac8ce 100644 --- a/integration_tests/generic/auth/certificate/assets/client.crt +++ b/integration_tests/generic/auth/certificate/assets/client.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDETCCAfkCFFSLOinkkPWOfwuzMHF0B9EZIIghMA0GCSqGSIb3DQEBCwUAMEUx +MIIDEzCCAfsCFBDZsFEIb3QwKLzXLoqR/oaDwakYMA0GCSqGSIb3DQEBCwUAMEUx CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MzA2WhcNMjMwNzIxMDA0 -MzA2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAp187MX/sPGlTl8WklyTSh8+kYURy2zWmp2FglMvo058NanCD -rjYJvS2FNM2ZwRZH52qBr6rpqIsa8QDFjnlFJK5y4FNr104ZjDr1YXDfX5ftQeeX -wCavFRDdH/s43kGyNjNeSH78EhqmC3z7LxCJT5k8rXRAyC02uBRqQTrRHfNuVvGI -JZ59jxESzRJt89kWuYymkAj16LHffMvcq6HLwr/KG8IyrJJRj5KcDzYorFonTPe9 -rwBlAbU3LpQ4ZlEP1mQA2PdCg3t85pz3n+57Iw839bWrwbjfijXHY3yjbqfEAaqa -md08nByTg8TZlt9UzjWgj0K1DXDMDDkJKqi39QIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQAH0MhznadrMFuY2ZYA69FbsvOygMctv8qZW1HrHS0X13IXeW+8uxfb5+gk -yKFgXNMFueyd5PoN9vyC1t9AOBPnI56gaYm/MbDtwEqGo8C+9fjJasY23J90p54t -G6vxcXwo33HVpWBeRBkVF/SePeCn+MKk0jd/JgJS0T0s1Ih6wkn84/83hDk4M2M2 -/yhc3wuNYdf/WB8QAfJAc2YpIfkMOoxGPTsxvREiZrPUyGiWa507hrHcQU0GV8qC -KcnS7UUCT9TtJvQIKHwW68XjBudWpaILBj1TS8hOGseOJydJqbk8wyMTE6fgc1Ss -KfTrfa0HOHIkAU/TfE22Zqfw4z70 +cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwNzI4MTAwODIyWhgPMzAwMzA5Mjkx +MDA4MjJaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCp8/P9JAyE90ZrE1LZcJ/B24f79aazY8S/eeRRZsTvUP73 +NrOznv1zhvJ9TKHUNcOouZ/NPQanNOiqkoigQwP7L2FA2bPOPAPIWBPWGdjSkeyZ +8MYbA7Or+16k2ZYvKsCarG/PgGeL0UFLe6INvZRMnk1s+iF0upcHv5BhjIfBwzh4 +o2pLY1d9bbnEsuSNagOzIkQS3mI22d1YbJKxXP0m+tBk1gTqhUhwEAXNaIBCRscs +xyv9pW7ZSjPabf/L0Md2yMcVs0+oK6rkQbAWrTTjN1lJ603BHh+keIDMwQnbMB0U +AStJdyQpwa7hZ+5767+GxR7n85Twe1rSexmTl9/fAgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAIOQE2DWqwse0srtG+7IS0EO3iP27lRKxd387wY1xq00o3depKReVpYm +R8sZM1meumniH1QKoVFJpBHYoPzQMi8vMmI9AV3KWNFcCyf+jwc69Qab2erDNVsw +5mCCGXkrzLbCzmbPFZoyvMmBlsQSmOjwyGGIeXwfqKv/TPwOzKfSM/KkQmgRyUro +GDT+TI5VhgvQyNLmkWNRhnI30DnlsQ1Bc0MEQ1hismOYxD4mCqufCOS3BmakDRNK +QBz0xl0i5Dbf+e4o3rEaCGW/rzKkL1mm1TXqpDEy3UAwj+jIOZu5yByw5djfgSIX +OEVuqklUASYAPeVdSyf/VAflLV9nGKI= -----END CERTIFICATE----- diff --git a/integration_tests/generic/auth/certificate/assets/client.key b/integration_tests/generic/auth/certificate/assets/client.key index 96a88abbc..a31f753cf 100644 --- a/integration_tests/generic/auth/certificate/assets/client.key +++ b/integration_tests/generic/auth/certificate/assets/client.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnXzsxf+w8aVOX -xaSXJNKHz6RhRHLbNaanYWCUy+jTnw1qcIOuNgm9LYU0zZnBFkfnaoGvqumoixrx -AMWOeUUkrnLgU2vXThmMOvVhcN9fl+1B55fAJq8VEN0f+zjeQbI2M15IfvwSGqYL -fPsvEIlPmTytdEDILTa4FGpBOtEd825W8Yglnn2PERLNEm3z2Ra5jKaQCPXosd98 -y9yrocvCv8obwjKsklGPkpwPNiisWidM972vAGUBtTculDhmUQ/WZADY90KDe3zm -nPef7nsjDzf1tavBuN+KNcdjfKNup8QBqpqZ3TycHJODxNmW31TONaCPQrUNcMwM -OQkqqLf1AgMBAAECggEAEZiXdorGYUuJeElVFnbOk2ynEXrKwHURgkNgjgQqBCWS -mYAet/ACchsZCAYdhgk7of62h6tmSUvmlzPHkUT6mfKlLSRYEBir4uxH4+ij8z7b -uLQKZi8q9QIC3VviDKvHep9H6ENBaP3YOxj2p2oLpYysrmesb98hA5VR3m26knVb -sfYwLdsrqQ5Tiuzm1Mdaca2HEYq0iXoZqjQHi31h33rw4HFBKcGX6w6PJOon+i2m -eSSaCAJMYFkoS6NafJYwQDuwIwp+IyrYdI/vaAR6s2ufJ4doIjJC7YuPO5jpLpJc -IBBl5e7pu4/rlwJCDARfBWuXFbjtoMAvmM1MojdNOQKBgQDYCIl97ef19LmeYfwk -RW2xhQTVCWwwBBhSl06iB9SeFkzCCRb+AFjhVhcwe4xIWh9GXWZawZC2XSaoTXte -hxcZRFbnpjcpZ6sYKiP4fB/GXGqYDUWovxu8gmXKrXtfwqJMXklfFE7WblGgK5gG -l7OfbuczaRaIQHoIQYzfmeCwmwKBgQDGVhCSNFGiRaDG7k2VfXElaIyL7m7FlsXf -EptolOeoGv5GvVr3CB5TvUqXN3haLZvUbBKRpWrDbP1n1i+77VICOLtq1qf6SEog -1p2PAccGhXXvrL7LJLUr9Hk831D4fSX9TqVzdxwfMdGFepYoS4vm8fkGVCuaytDa -fniJl8TarwKBgQCpTreCrAsY5bz7dcuIGamIcLmCxKm3T95IDDEiJ4ToiI2LnFga -pOcDYtc1tf4RTiAoo1ZuVjk10vdS+7ZuNO1Tbg216rxchNTAUXZzbcPxT8hydiRb -xbrVGFTybNe+CunrdBGIpH/M6hSqtL+mmwm5L8+eqQNxsSZyhf0D2LMRdQKBgD8J -CXk+MZfOY1v2Tygs1zIZeVnb7M7VrYvJYSUq9jliYuBevDN5HBJnPfazhYe7qSQp -OPmbRkRYNm2zEDa9JWxZVY+OK5MLOKwZKbhSy0uSTTpgf78WqpIOwB2NqDFhrRpF -zaXV/FUZw0qV/HVQFWXQD+JoC/fFb/2RZoPsfX83AoGBAM4VPDUa1I16MuuW+FNZ -cVENZK/qsXFMvm3xutezvMSgmCxVSnXy9GP8QbqkfMrDJd3v/HnwrC4ORTlU7rim -AWSvC6CYO1c2RantleA46T90uWW8kP24TK1yWOrRGKuaQYvfokiiLVExq1nA1iSR -/QPLg6vEoPMOLhB7BQBpsFkq +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCp8/P9JAyE90Zr +E1LZcJ/B24f79aazY8S/eeRRZsTvUP73NrOznv1zhvJ9TKHUNcOouZ/NPQanNOiq +koigQwP7L2FA2bPOPAPIWBPWGdjSkeyZ8MYbA7Or+16k2ZYvKsCarG/PgGeL0UFL +e6INvZRMnk1s+iF0upcHv5BhjIfBwzh4o2pLY1d9bbnEsuSNagOzIkQS3mI22d1Y +bJKxXP0m+tBk1gTqhUhwEAXNaIBCRscsxyv9pW7ZSjPabf/L0Md2yMcVs0+oK6rk +QbAWrTTjN1lJ603BHh+keIDMwQnbMB0UAStJdyQpwa7hZ+5767+GxR7n85Twe1rS +exmTl9/fAgMBAAECggEAPZzaVGhQPZgjqEfeHkQtNqtuthJNd/Vwa3Y2JqiaNqRn +epoTNcgq3EoM+Q3iETvYjf+VhmNcWRveSZBMBcWl2NdJa6hA/kBVorkDn/fI2jXa +z8gxGbQS3AOKQTs8ribSooBnHJPRdifLgyD0FAUpkUlGin53yIionj99iU/YG48g +4dwkBIFHRcxertQyhu3YQ+XleJ35n7mNFwGzC7curRBPUHMImPASzVYQhVdN8OBt +TZEoJw+2lmH4fIJYult27hcl2/pLs1FPvQFSLTIoqzaEzRhKdANkclmnhJjCBXzB +7RLUpKOv1Q28u+P5KH1nFBV/UuuxXrjFt4jhRdji2QKBgQDvv+W0GJWX5POfyRHT +pAROclgVPEgS5vXQIelMdR76a72L/4Vm2/xeolWW1h5qmJF479V/+P+ppxb1IrUy +6+yGtkMiQE4CizhFGWivfXUTPZbdeeSpHMUl9tRZdBZWi3aXzJ/8DfCzD+ZVS4Vx ++y62V4ymQyAqBWv2ast/ElEbowKBgQC1ePQgR+MNfz7/BaatCcLPwFG/kkqPVuzH +//6HB+gAYTyuZsbLrYhCQsbsTjvQz0ExmTnNSeCjHTntQ+pZ8Tnuet9bHxKTRbvG +9Ol/J402EnY2tO/b8jKXHNNyLNImvWpJ4PpaLRKQVxLPei+JcEHyz4MVMrhIjX1b +EhhDCZ6ulQKBgEUy+jX1MphY+QiRnJedq7CIyGu4roTmLOUaJKBw3bQiDN+vrO13 +aWxXJqUWwEi8KKDjeJgrYn+xPqsajXpZJjfru4zTSrDpRiCLqO+eIoCfMkBSwnEd +YLoIeFopa5knP9+orDSwQV0tpanQ1n+DpIP02R/UGCCI2BST1pCi1M5JAoGAC/+E +PIIkO+c21gucmoIztCKmBQF6FoILw6lkPa9DIotLRMicyiieAquBlWwSvlqFl+7m +iHEi/gXXp50+6FVvnBnZnJ+wTbZllODqczK9Pl74G+PYm/UmbSFFxZ27Az6wwVOz +mbSzLoHjR35vmCmo4pHfu84PqxRXvmay3fPL3wUCgYEA0yZcvQqiTs+f4S/mZbhp +fyPgurmowXUNgdijyeFoH+DMkwdWUJeBrinelQaXADUSXkKiA8gaoNGOIkDIBcve +gdUhrY204MeoTYxnIb1dw6/KReya4YdRSMlYiX2hYEURIxdaJV5HcwW5ySMOzP87 +t2+YVr4faAv4AS8k21pBGrc= -----END PRIVATE KEY----- diff --git a/integration_tests/generic/auth/certificate/assets/server.crt b/integration_tests/generic/auth/certificate/assets/server.crt index aa818acb4..b406cf943 100644 --- a/integration_tests/generic/auth/certificate/assets/server.crt +++ b/integration_tests/generic/auth/certificate/assets/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDETCCAfkCFHA1RpGfOY5p/vQmeMQ1oRFqH+CGMA0GCSqGSIb3DQEBCwUAMEUx +MIIDEzCCAfsCFC21Zw7U0tGDyLyMalwfo9cWbL6dMA0GCSqGSIb3DQEBCwUAMEUx CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MjQ2WhcNMjMwNzIxMDA0 -MjQ2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA3VdrKR5hmZ+vyvg6NB2dOL5vEIQ/9DevivnKWqX5mserYLMj -Wq0knVfogewZnrDe+zVC3kOogBQvYk8Z53kTY9qpJT85dMCuW4xDx0JU+cWHul9a -pzF+bvws4paCWIcsGONyocPAx5g07LbPU9civC80QkQqELo1zYiRU1bX8vRJJqbN -TW2mzl9MN3AnCAYTwq8WhVG/1QR3LPQhPR68/1LWrFefQaEWaXT2s+Xv7K7NDXro -WSba4SgKdFd6fyUVMVr/ioT1KT45TP5jbRrW5JJUTdpkiXaIucrZg39f6F5gTZGA -U7bNROUMkqrJJngN9+Hp+YH1GpkKgu9EKA30EQIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQAw91bxiAi7DIVsKL3k4B0I+50ZKq9VMVNE3YCTPygpfuRiGQvlITZ5I8I5 -3Ok2wWltgKx6EnicHIlLg42yRj7j3mdgOLMFMrUCfJmdogwnS+k6veG3G1RHUs9r -ATfX49u/hEX2pe7Rvx2VYVIugwrQESgQ21iaf6uUMsrq6W8eYZ31as1nJKpqIGbu -W1fZMSi0RIUJP+mpVBE82IW+gJRi3uKU4HKPqyrU3dviBFdBxb3lNbh34/vdNkIw -4H2CfBxEvdwLYAhWDerlm4wWCmjkMiHfBHPBhhOICTkR25a7NFy27h/UDHjVC/6m -fGshVSBtxVPJP7kcvZ1scIctvFZZ +cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwNzI4MTAwNzI3WhgPMzAwMzA5Mjkx +MDA3MjdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCjMlvOKQX9yn9SOYPJ8p+jeDUU/JWPwT4LRfqaxvvKSnS7 +NZzd7lS4AR0YTjyjiRj3+t0QnEDHVKBD8cMh9kMXkQ2S0r7psCURLvvZOYt4v6KM +CyZpBbp8b/pG3aJQHDZjRDOApQrXhx62XJDIs64YKA8NybYOLqNisrWGrfqF4uEz +RMgVGlthuQcXo3n2HzobuYN7RsHBzCWGLn9fRMDC2j3IAnQLf4YOznOJ57CjMd2W +mn/yhHK8h9s4iU5zw3+PK+X/IM4GeAfeJMx8c5uq2A8A24uzMidyhxJCK7VUprjK +/ckdNYya6dkG2De+LR7W82ygfWbFDOnZKM26cPG/AgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAH5+Wdb/1jgBhihN6Pb6SWJmDvwkOEP3t00E3fBao4TDqdDOhPsLYrAm +8gt16OcGrrXDQA3bi79mAVqAqCvaf4hk0vSI0L4rNcCSP4D3fUBjRO3fY3fM4Qw8 +xg9AusF5hRrvzFbEak7lPJ01kLOJEgBA1l457HrLnXcpDTml8Y46WqdWa6yVM33l +7tNaXWrPwYZYMTcRumIytsYtIJXp/sMLBIT0AO/QR4yarvVOeMSJ1va459PjKLBG +JGGmf2rigaT050e71QOrGyMXgT6xsNjJgzeVhUgPO422mPT692kDi2oB5DA0Fau0 +4qm5CMFgmYcC3zQoN53aDs1mHyWeroc= -----END CERTIFICATE----- diff --git a/integration_tests/headless/headless-header-status-test.yaml b/integration_tests/headless/headless-header-status-test.yaml deleted file mode 100644 index 1b53d2252..000000000 --- a/integration_tests/headless/headless-header-status-test.yaml +++ /dev/null @@ -1,24 +0,0 @@ -id: headless-header-status-test - -info: - name: headless header + status test - author: pdteam - severity: info - -headless: - - steps: - - args: - url: "{{BaseURL}}" - action: navigate - - action: waitload - - matchers-condition: and - matchers: - - type: word - part: header - words: - - text/plain - - - type: status - status: - - 200 diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index d886a7f5a..0e5c9c288 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4a0a004730450220782aba8c09f28370a95ad26563a89b6a0602de41d60577e07f16b356ed383f32022100c4aaeaaded87e5a66b29df00196382f7405edaa84e7ace9fe229b17faa1f9f49 \ No newline at end of file +# digest: 4b0a00483046022100cd2b9d34169cdb716caee25976fed763880435f2f1e2979c9d7c9d2bd7b8e409022100dd0ba8bd3fa6a6be5f964ca3b0ce8bdbb20d865553133cf494ef64fbeebff345 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 93d077a42..8fae1b051 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a004730450220669770a131431f19616155e6f984627773d33798780cf5ce9f40bdbb0947c718022100e87c841ac3e9082d261b420681d4b4bd5443a23dff7080cd95d4282c27d43ec8 \ No newline at end of file +# digest: 4b0a00483046022100f663e5afaf5c118b21b9c5918cba12d7cc83edc2a3ee0f338c07e3cd1fe40e20022100b46193e3275c490a4ad3897c6e2ca51ce09f408538b17d041e0063d40f4df833 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index f9fa4f706..de4abdac0 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 490a004630440220528b4720e39820f5ce36b77c75c04a26ba672c8f67ceb43214541e1d4cafabfd0220456698a9ba7d52f3a6833774fa1c2ed0a2e1dbcf116611c255db8d3061e16560 \ No newline at end of file +# digest: 4b0a00483046022100c45cd27b9d49879663e1ea3c877dc362d06b8a0aea64b1ab06be3af5aa9a32ee0221008f5ee347245a2c1e04c46528e4c70a5a851f95c6ba49d2834ef7c3784bca47a9 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 989064895..9b0187dd8 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022100a38f338f224d5c4953bb4624352ad6ea0f0bb9000f00edcb40907b2c9806250002201cdc272cd895e85ff07a437997a844180ddc8baf4b9fc7d258fe90bd0ab2b8a8 \ No newline at end of file +# digest: 4a0a00473045022100df57bf446d6d8e73ff9424b1055faebcea9038e5d5934834ed8e619b77bdfd5e02201754c1cebe9f65883315b3830755a0689999f33db7102cd8d5469e4c01cc6a66 \ No newline at end of file diff --git a/integration_tests/protocols/http/dsl-functions.yaml b/integration_tests/protocols/http/dsl-functions.yaml index 72dffbfc5..5afce178e 100644 --- a/integration_tests/protocols/http/dsl-functions.yaml +++ b/integration_tests/protocols/http/dsl-functions.yaml @@ -98,6 +98,7 @@ requests: 85: {{split("ab,cd,efg", ",", 2)}} 86: {{ip_format('127.0.0.1', 3)}} 87: {{ip_format('127.0.1.0', 11)}} + 88: {{jarm('scanme.sh:443')}} extractors: - type: regex name: results diff --git a/integration_tests/protocols/http/http-paths.yaml b/integration_tests/protocols/http/http-paths.yaml index a43725dfc..ae389a379 100644 --- a/integration_tests/protocols/http/http-paths.yaml +++ b/integration_tests/protocols/http/http-paths.yaml @@ -16,6 +16,7 @@ info: - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "/test/..;/..;/" - "/xyz/%25u2s/%25invalid" + - "//CFIDE/wizards/common/utils.cfc" # duplicating here because same results are expected even if http request is written in different format - "/1337?with=param" - "/some%0A/%0D" @@ -24,6 +25,7 @@ info: - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "/test/..;/..;/" - "/xyz/%25u2s/%25invalid" + - "//CFIDE/wizards/common/utils.cfc" # Test all templates with FullURLs requests: @@ -62,6 +64,10 @@ requests: - |+ GET /xyz/%u2s/%invalid HTTP/1.1 Host: scanme.sh + # test relative path start with // + - |+ + GET //CFIDE/wizards/common/utils.cfc HTTP/1.1 + Host: scanme.sh matchers: - type: status @@ -77,6 +83,7 @@ requests: - "{{BaseURL}}/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "{{BaseURL}}/test/..;/..;/" - "{{BaseURL}}/xyz/%u2s/%invalid" + - "{{BaseURL}}//CFIDE/wizards/common/utils.cfc" matchers: - type: status diff --git a/integration_tests/protocols/http/matcher-status.yaml b/integration_tests/protocols/http/matcher-status.yaml new file mode 100644 index 000000000..5704c2a3d --- /dev/null +++ b/integration_tests/protocols/http/matcher-status.yaml @@ -0,0 +1,40 @@ +id: matchet-status + +info: + name: Test Matcher Status + author: pdteam + severity: critical + +variables: + username: test + password: admin + date: 2023-05-31 + +http: + - method: GET + path: + - "{{RootURL}}/login?username={{username}}&password={{password}}" + - "{{BaseURL}}/admin-pannel" + + - method: GET + path: + - "{{BaseURL}}/dashboard?date={{date}}" + - "{{BaseURL}}/signup" + + - method: POST + path: + - "{{BaseURL}}/filemanager/upload.php" + body: "fldr=&url=file:///etc/passwd" + + + stop-at-first-match: true + matchers-condition: and + matchers: + - type: word + part: body + words: + - "matcher status" + + - type: status + status: + - 200 diff --git a/integration_tests/protocols/http/raw-unsafe-path.yaml b/integration_tests/protocols/http/raw-unsafe-path.yaml index 4c86dcfd9..a10721eb3 100644 --- a/integration_tests/protocols/http/raw-unsafe-path.yaml +++ b/integration_tests/protocols/http/raw-unsafe-path.yaml @@ -16,6 +16,7 @@ info: - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "/test/..;/..;/" - "/xyz/%u2s/%invalid" + - "//CFIDE/wizards/common/utils.cfc" # Test all unsafe URL Handling Edgecases @@ -50,6 +51,10 @@ requests: - |+ GET /xyz/%u2s/%invalid HTTP/1.1 Host: scanme.sh + # test relative path start with // (should not be removed) + - |+ + GET //CFIDE/wizards/common/utils.cfc HTTP/1.1 + Host: scanme.sh unsafe: true matchers: diff --git a/integration_tests/multi/dynamic-values.yaml b/integration_tests/protocols/multi/dynamic-values.yaml similarity index 100% rename from integration_tests/multi/dynamic-values.yaml rename to integration_tests/protocols/multi/dynamic-values.yaml diff --git a/integration_tests/multi/evaluate-variables.yaml b/integration_tests/protocols/multi/evaluate-variables.yaml similarity index 100% rename from integration_tests/multi/evaluate-variables.yaml rename to integration_tests/protocols/multi/evaluate-variables.yaml diff --git a/integration_tests/multi/exported-response-vars.yaml b/integration_tests/protocols/multi/exported-response-vars.yaml similarity index 100% rename from integration_tests/multi/exported-response-vars.yaml rename to integration_tests/protocols/multi/exported-response-vars.yaml diff --git a/integration_tests/protocols/network/same-address.yaml b/integration_tests/protocols/network/same-address.yaml new file mode 100644 index 000000000..5cad9d21f --- /dev/null +++ b/integration_tests/protocols/network/same-address.yaml @@ -0,0 +1,29 @@ +id: same-target + +info: + name: same-target + author: pdteam + severity: info + description: Riak is a distributed NoSQL key-value data store that offers high availability, fault tolerance, operational simplicity, and scalability. + +network: + - host: + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + inputs: + - data: "PING\r\n" + read-size: 4 + matchers: + - type: word + part: data + words: + - "PONG" diff --git a/integration_tests/workflow/shared-cookie.yaml b/integration_tests/workflow/shared-cookie.yaml index f2d68be2c..802be7818 100644 --- a/integration_tests/workflow/shared-cookie.yaml +++ b/integration_tests/workflow/shared-cookie.yaml @@ -11,5 +11,5 @@ workflows: - template: workflow/http-2.yaml # store cookie in native browser context - template: workflow/headless-1.yaml - # retrive 2 standard library cookies + headless cookie + # retrieve 2 standard library cookies + headless cookie - template: workflow/http-3.yaml \ No newline at end of file diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 51f91820d..0c0bf492c 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -32,6 +32,11 @@ "title": "epss score for the template", "description": "EPSS Score for the template" }, + "epss-percentile": { + "type": "number", + "title": "epss percentile for the template", + "description": "EPSS Percentile for the template" + }, "cpe": { "type": "string", "title": "cpe for the template", @@ -1293,7 +1298,7 @@ } }, "type": "object", - "title": "payloads for the webosocket request", + "title": "payloads for the websocket request", "description": "Payloads contains any payloads for the current request" } }, diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 83b95e801..df77d595c 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -37,13 +37,7 @@ archives: - format: zip id: nuclei builds: [nuclei-cli] - replacements: - darwin: macOS - -- format: zip - id: annotate - builds: [annotate] - name_template: "{{ .Binary }}" + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: algorithm: sha256 diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index f047b8ec7..bd03245fa 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -13,13 +13,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) -var codeTestCases = map[string]testutils.TestCase{ - "protocols/code/py-snippet.yaml": &codeSnippet{}, - "protocols/code/py-file.yaml": &codeFile{}, - "protocols/code/py-env-var.yaml": &codeEnvVar{}, - "protocols/code/unsigned.yaml": &unsignedCode{}, - "protocols/code/rsa-signed.yaml": &rsaSignedCode{}, - "protocols/code/py-interactsh.yaml": &codeSnippet{}, +var codeTestCases = []TestCaseInfo{ + {Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}}, + {Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}}, + {Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}}, + {Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}}, + {Path: "protocols/code/rsa-signed.yaml", TestCase: &rsaSignedCode{}}, + {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}}, + {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() }}, } var ( @@ -50,10 +51,6 @@ func init() { panic(err) } - if osutils.IsWindows() { - codeTestCases["protocols/code/ps1-snippet.yaml"] = &codeSnippet{} - } - signTemplates() } @@ -69,7 +66,15 @@ func signTemplates() { log.Fatalf("couldn't create crypto engine: %s\n", err) } - for templatePath, testCase := range codeTestCases { + for _, v := range codeTestCases { + templatePath := v.Path + testCase := v.TestCase + + if v.DisableOn != nil && v.DisableOn() { + // skip ps1 test case on non-windows platforms + continue + } + templatePath, err := filepath.Abs(templatePath) if err != nil { panic(err) diff --git a/v2/cmd/integration-test/custom-dir.go b/v2/cmd/integration-test/custom-dir.go index 5ee811acd..4670e561b 100644 --- a/v2/cmd/integration-test/custom-dir.go +++ b/v2/cmd/integration-test/custom-dir.go @@ -8,8 +8,8 @@ import ( type customConfigDirTest struct{} -var customConfigDirTestCases = map[string]testutils.TestCase{ - "protocols/dns/cname-fingerprint.yaml": &customConfigDirTest{}, +var customConfigDirTestCases = []TestCaseInfo{ + {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &customConfigDirTest{}}, } // Execute executes a test case and returns an error if occurred diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index a4e3cbc5e..d0035e8a9 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -4,14 +4,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var dnsTestCases = map[string]testutils.TestCase{ - "protocols/dns/basic.yaml": &dnsBasic{}, - "protocols/dns/ptr.yaml": &dnsPtr{}, - "protocols/dns/caa.yaml": &dnsCAA{}, - "protocols/dns/tlsa.yaml": &dnsTLSA{}, - "protocols/dns/variables.yaml": &dnsVariables{}, - "protocols/dns/payload.yaml": &dnsPayload{}, - "protocols/dns/dsl-matcher-variable.yaml": &dnsDSLMatcherVariable{}, +var dnsTestCases = []TestCaseInfo{ + {Path: "protocols/dns/basic.yaml", TestCase: &dnsBasic{}}, + {Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}}, + {Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}}, + {Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}}, + {Path: "protocols/dns/variables.yaml", TestCase: &dnsVariables{}}, + {Path: "protocols/dns/payload.yaml", TestCase: &dnsPayload{}}, + {Path: "protocols/dns/dsl-matcher-variable.yaml", TestCase: &dnsDSLMatcherVariable{}}, } type dnsBasic struct{} diff --git a/v2/cmd/integration-test/dsl.go b/v2/cmd/integration-test/dsl.go new file mode 100644 index 000000000..6a1666d7e --- /dev/null +++ b/v2/cmd/integration-test/dsl.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var dslTestcases = []TestCaseInfo{ + {Path: "dsl/hide-version-warning.yaml", TestCase: &dslVersionWarning{}}, + {Path: "dsl/show-version-warning.yaml", TestCase: &dslShowVersionWarning{}}, +} + +var defaultDSLEnvs = []string{"HIDE_TEMPLATE_SIG_WARNING=true"} + +type dslVersionWarning struct{} + +func (d *dslVersionWarning) Execute(templatePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "DSL version parsing warning test") + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiArgsAndGetErrors(debug, defaultDSLEnvs, "-t", templatePath, "-target", ts.URL, "-v") + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + +type dslShowVersionWarning struct{} + +func (d *dslShowVersionWarning) Execute(templatePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "DSL version parsing warning test") + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiArgsAndGetErrors(debug, append(defaultDSLEnvs, "SHOW_DSL_ERRORS=true"), "-t", templatePath, "-target", ts.URL, "-v") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/v2/cmd/integration-test/file.go b/v2/cmd/integration-test/file.go index f33f07f05..c7c4b6696 100644 --- a/v2/cmd/integration-test/file.go +++ b/v2/cmd/integration-test/file.go @@ -4,10 +4,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var fileTestcases = map[string]testutils.TestCase{ - "protocols/file/matcher-with-or.yaml": &fileWithOrMatcher{}, - "protocols/file/matcher-with-and.yaml": &fileWithAndMatcher{}, - "protocols/file/extract.yaml": &fileWithExtractor{}, +var fileTestcases = []TestCaseInfo{ + {Path: "protocols/file/matcher-with-or.yaml", TestCase: &fileWithOrMatcher{}}, + {Path: "protocols/file/matcher-with-and.yaml", TestCase: &fileWithAndMatcher{}}, + {Path: "protocols/file/matcher-with-nested-and.yaml", TestCase: &fileWithAndMatcher{}}, + {Path: "protocols/file/extract.yaml", TestCase: &fileWithExtractor{}}, } type fileWithOrMatcher struct{} diff --git a/v2/cmd/integration-test/fuzz.go b/v2/cmd/integration-test/fuzz.go index 2a9573d23..5dd048d32 100644 --- a/v2/cmd/integration-test/fuzz.go +++ b/v2/cmd/integration-test/fuzz.go @@ -12,11 +12,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var fuzzingTestCases = map[string]testutils.TestCase{ - "fuzz/fuzz-mode.yaml": &fuzzModeOverride{}, - "fuzz/fuzz-type.yaml": &fuzzTypeOverride{}, - "fuzz/fuzz-query.yaml": &httpFuzzQuery{}, - "fuzz/fuzz-headless.yaml": &HeadlessFuzzingQuery{}, +var fuzzingTestCases = []TestCaseInfo{ + {Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}}, + {Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}}, + {Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}}, + {Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}}, } type httpFuzzQuery struct{} diff --git a/v2/cmd/integration-test/generic.go b/v2/cmd/integration-test/generic.go index dfd24b165..8b66257a0 100644 --- a/v2/cmd/integration-test/generic.go +++ b/v2/cmd/integration-test/generic.go @@ -13,59 +13,59 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var genericTestcases = map[string]testutils.TestCase{ - "generic/auth/certificate/http-get.yaml": &clientCertificate{}, +var genericTestcases = []TestCaseInfo{ + {Path: "generic/auth/certificate/http-get.yaml", TestCase: &clientCertificate{}}, } var ( serverCRT = `-----BEGIN CERTIFICATE----- -MIIDETCCAfkCFHA1RpGfOY5p/vQmeMQ1oRFqH+CGMA0GCSqGSIb3DQEBCwUAMEUx +MIIDEzCCAfsCFC21Zw7U0tGDyLyMalwfo9cWbL6dMA0GCSqGSIb3DQEBCwUAMEUx CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MjQ2WhcNMjMwNzIxMDA0 -MjQ2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA3VdrKR5hmZ+vyvg6NB2dOL5vEIQ/9DevivnKWqX5mserYLMj -Wq0knVfogewZnrDe+zVC3kOogBQvYk8Z53kTY9qpJT85dMCuW4xDx0JU+cWHul9a -pzF+bvws4paCWIcsGONyocPAx5g07LbPU9civC80QkQqELo1zYiRU1bX8vRJJqbN -TW2mzl9MN3AnCAYTwq8WhVG/1QR3LPQhPR68/1LWrFefQaEWaXT2s+Xv7K7NDXro -WSba4SgKdFd6fyUVMVr/ioT1KT45TP5jbRrW5JJUTdpkiXaIucrZg39f6F5gTZGA -U7bNROUMkqrJJngN9+Hp+YH1GpkKgu9EKA30EQIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQAw91bxiAi7DIVsKL3k4B0I+50ZKq9VMVNE3YCTPygpfuRiGQvlITZ5I8I5 -3Ok2wWltgKx6EnicHIlLg42yRj7j3mdgOLMFMrUCfJmdogwnS+k6veG3G1RHUs9r -ATfX49u/hEX2pe7Rvx2VYVIugwrQESgQ21iaf6uUMsrq6W8eYZ31as1nJKpqIGbu -W1fZMSi0RIUJP+mpVBE82IW+gJRi3uKU4HKPqyrU3dviBFdBxb3lNbh34/vdNkIw -4H2CfBxEvdwLYAhWDerlm4wWCmjkMiHfBHPBhhOICTkR25a7NFy27h/UDHjVC/6m -fGshVSBtxVPJP7kcvZ1scIctvFZZ ------END CERTIFICATE----- +cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwNzI4MTAwNzI3WhgPMzAwMzA5Mjkx +MDA3MjdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCjMlvOKQX9yn9SOYPJ8p+jeDUU/JWPwT4LRfqaxvvKSnS7 +NZzd7lS4AR0YTjyjiRj3+t0QnEDHVKBD8cMh9kMXkQ2S0r7psCURLvvZOYt4v6KM +CyZpBbp8b/pG3aJQHDZjRDOApQrXhx62XJDIs64YKA8NybYOLqNisrWGrfqF4uEz +RMgVGlthuQcXo3n2HzobuYN7RsHBzCWGLn9fRMDC2j3IAnQLf4YOznOJ57CjMd2W +mn/yhHK8h9s4iU5zw3+PK+X/IM4GeAfeJMx8c5uq2A8A24uzMidyhxJCK7VUprjK +/ckdNYya6dkG2De+LR7W82ygfWbFDOnZKM26cPG/AgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAH5+Wdb/1jgBhihN6Pb6SWJmDvwkOEP3t00E3fBao4TDqdDOhPsLYrAm +8gt16OcGrrXDQA3bi79mAVqAqCvaf4hk0vSI0L4rNcCSP4D3fUBjRO3fY3fM4Qw8 +xg9AusF5hRrvzFbEak7lPJ01kLOJEgBA1l457HrLnXcpDTml8Y46WqdWa6yVM33l +7tNaXWrPwYZYMTcRumIytsYtIJXp/sMLBIT0AO/QR4yarvVOeMSJ1va459PjKLBG +JGGmf2rigaT050e71QOrGyMXgT6xsNjJgzeVhUgPO422mPT692kDi2oB5DA0Fau0 +4qm5CMFgmYcC3zQoN53aDs1mHyWeroc= +-----END CERTIFICATE----- ` serverKey = `-----BEGIN PRIVATE KEY----- -MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDdV2spHmGZn6/K -+Do0HZ04vm8QhD/0N6+K+cpapfmax6tgsyNarSSdV+iB7BmesN77NULeQ6iAFC9i -TxnneRNj2qklPzl0wK5bjEPHQlT5xYe6X1qnMX5u/CziloJYhywY43Khw8DHmDTs -ts9T1yK8LzRCRCoQujXNiJFTVtfy9Ekmps1NbabOX0w3cCcIBhPCrxaFUb/VBHcs -9CE9Hrz/UtasV59BoRZpdPaz5e/srs0NeuhZJtrhKAp0V3p/JRUxWv+KhPUpPjlM -/mNtGtbkklRN2mSJdoi5ytmDf1/oXmBNkYBTts1E5QySqskmeA334en5gfUamQqC -70QoDfQRAgMBAAECggEBALtPsHMSr9vW5Giq2m6iJRwRJGJg2NJukZLVwuYlkW7n -zGNAFgo1fkfdTfks+Z1u5rTGJPl9XkpNSrAyaqSVtNALCptnvtLMAIGe2Pj2bH0X -Kb6R1WCqJOn9ZGq4nkQW2D2Ttb2psCn458jvB9NWu6FvfRUbJFIVk1SFXx6c3pFN -kPCUudAiscaldUDCiz4FccKGXdRjq6HIeeWqvdErteb6JPTs9QXCHfBql9Esl4rK -SHt9RmAFNY+CLExHiFPBR15hHZRtiVkAVrgnPg1CPGAyVG0hGXj7YMMWpAyfFWpn -8gWVt7XJ4UX2knUwfU8p8dWe6qwf+AMrhravYJyccoUCgYEA8Ts0kHFnLga8Ewao -nyDQs5uYGG0PWkbXqnFVYnMeSbXzyC4ouInIk/eOQABCxdjy3NF9QuYvVLpfLJ+9 -a97q1Vyg6lZ4PPuK8ZcPrHFSNNaj4eWNTOMo/Qdzz4bfflTsv8vjeeMxsqb6woXV -+E23UKCPlQPf86jugZVdaMtvZKsCgYEA6uR7glji70pVoG/f3soX1vllmVTtiLnh -zYMmwPyTRDvoGgg/nGK+GCq//Xyn8D900hbX8KKqGX7ca5FGk5pOpW/QE9uLcuWK -xcy8KAc05k1u4VaS5loWKnPGWreIpj3RbCfbPs5X/jBC+fPIA4Q8Qor5ZGdqVBvW -IKejnNqasjMCgYEAqltPUbpkTWLAKweGyWnZOR3mmUlbkDt7Toje7bmyaAew82t1 -omzbU3N958DHZwVA7aSbu0TnpARB9jeRA77XRHo3wYXzP828X8R4cyVMEriJ35vG -38eESLyckrAC4SqETyZjrM4/aJT3fawaYVIw5SWegHPOEjr4xFaBMuKH9iUCgYEA -wFpC2kc374UMAcobpjIQu7aYAKyPqDuwMb+I6NjtMB9uvoKqtMIXsWqwtkBytkcA -v1p9k01hxmcg0eWxygW/CbM6zkgnNfvLXJeALbdZFo+qkVV4DrMPG8ybToalnJ1a -9hrda91GKZ4T+uQrktWjE0sDV7loVWBGRY+CaFyL+gkCgYEA3Z0j8VOLJnAKdCDp -3N74460pykwJ2suEYSJG6glXfU3fZ5VwAYjimxgD0S2VU4qK8PYBfa/oFH2vRX5p -11dWQWbfBdREO70UmJD4Pr6g3q9AF6DXLXb7dVm4y+hX065Xshk8oIuITVyO/XVK -wWqBD5GScI+Q7PLMes7aqtsDDJI= ------END PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCjMlvOKQX9yn9S +OYPJ8p+jeDUU/JWPwT4LRfqaxvvKSnS7NZzd7lS4AR0YTjyjiRj3+t0QnEDHVKBD +8cMh9kMXkQ2S0r7psCURLvvZOYt4v6KMCyZpBbp8b/pG3aJQHDZjRDOApQrXhx62 +XJDIs64YKA8NybYOLqNisrWGrfqF4uEzRMgVGlthuQcXo3n2HzobuYN7RsHBzCWG +Ln9fRMDC2j3IAnQLf4YOznOJ57CjMd2Wmn/yhHK8h9s4iU5zw3+PK+X/IM4GeAfe +JMx8c5uq2A8A24uzMidyhxJCK7VUprjK/ckdNYya6dkG2De+LR7W82ygfWbFDOnZ +KM26cPG/AgMBAAECggEAFtRko2J5xBcf2JDTLt0SF/wo8Nak1Ydi9pDDjgNoFdR0 +n/vQBfvhPhxpxYysTvRO2eHuKvSw2zGredXIRmf82r8f9vokWuyZQt4fvTOfnzSv +uIeWx/pVLDM9/8vhePN5aEmSKtzrt1rfoQMx/eGk6RwxfuxI25MKqDP30O9lrHTn +Y0lW7dthgdDMlQnSpOqUm2ldDsykYCBFteh4i5RDzAhiGx1ryaz3FMg+/y0VTTk0 +BM43qW6H9PD8P4iOau3DGIPNqtIlFSnWoYaM6Ta2osfzzdsnFbe5F7JbdMrf5MBc +Jq3VMUqffRmHubz7di03qRsRqGYQn2cJeiuVC+y6gQKBgQDYpq3MfMjwzPeoB1Ay +ZQdzx+T290XRxFZwkiv3uugsYMlFGEabdAMFx5oIIOdjWSBLI92RvXbg7qMd/xMC +ya/GzbKQd+5GbRLW+TZ0odGkMFkTo+DEkt07yEM8mrPJ6XePUndHbiNFSdpVKx4g +KdmiRHinm3R8Lr5/puvISrOdcwKBgQDA1kln9aD1mvIdObI6MubPitb+NuNcpVDo +myc1UrEJbcn8nBbLb+0Q+7gckjau2C8GN7Olnd8RCYLc7kU1On2pY+f19Ru/PdZX +cjCCTcxqCJvWkNWOzw14ag6UrDTF5nxtoVl/eXbHxWqFjdt0a211sa1mp3Gn3ZNq +m/teImYHhQKBgQCzWUA1MPPzi+pU2kEEhugla8xauha9cUiRhiAJw1uiKTlVDqSc +2ewKo9MaeYqzjruSGI26sVqxGDxGf7tQKoBuFiiFOhMxj+fxuHrhEHiI8FE9VgOj +F2U3sTAgAn1lX/VO21jM9BsUp++rY7dbrulwUDiFn8ZNazDeYeN8eoK4iwKBgQCb +cqJN+YW9NyCBSqdPnwTMvSE+YES7xFAKkjfzFiu8bBJtXe5KJHm4PRJXhc4q9/5A +Rtq8YR0WgNJLApArrnDqAa1Vajbp3RFSAKz1/X0Q5MurFanxqxsyvFvwoTkRZxFa +1rxstB96Prv12TrVCFx+ibI8lDJcnZNeV0s0wQn6eQKBgQDXkfPuX5TFBpNe1bWI +KUFmw9R1ynmUlIOaU3ITLv9C+w8zaJSpxFDZgJdv3uT8PfnXrsHm+lWjaOunvjri +quZSc06mLlEbggYoIFQNPeNPRyN0+GLvefMS3mCotzanZTmD5GrH9XG451tVPiH9 +G/lpNA1ccRCCsLslcG/aaa5PQw== +-----END PRIVATE KEY----- ` ) diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go index 20e9cfc46..dcb647b9d 100644 --- a/v2/cmd/integration-test/headless.go +++ b/v2/cmd/integration-test/headless.go @@ -10,14 +10,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var headlessTestcases = map[string]testutils.TestCase{ - "protocols/headless/headless-basic.yaml": &headlessBasic{}, - "protocols/headless/headless-header-action.yaml": &headlessHeaderActions{}, - "protocols/headless/headless-extract-values.yaml": &headlessExtractValues{}, - "protocols/headless/headless-payloads.yaml": &headlessPayloads{}, - "protocols/headless/variables.yaml": &headlessVariables{}, - "protocols/headless/file-upload.yaml": &headlessFileUpload{}, - "protocols/headless/headless-header-status-test.yaml": &headlessHeaderStatus{}, +var headlessTestcases = []TestCaseInfo{ + {Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessBasic{}}, + {Path: "protocols/headless/headless-header-action.yaml", TestCase: &headlessHeaderActions{}}, + {Path: "protocols/headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}}, + {Path: "protocols/headless/headless-payloads.yaml", TestCase: &headlessPayloads{}}, + {Path: "protocols/headless/variables.yaml", TestCase: &headlessVariables{}}, + {Path: "protocols/headless/file-upload.yaml", TestCase: &headlessFileUpload{}}, + {Path: "protocols/headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}}, } type headlessBasic struct{} diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 954b09fca..17d06acbd 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -26,57 +26,60 @@ import ( stringsutil "github.com/projectdiscovery/utils/strings" ) -var httpTestcases = map[string]testutils.TestCase{ - "protocols/http/get-headers.yaml": &httpGetHeaders{}, - "protocols/http/get-query-string.yaml": &httpGetQueryString{}, - "protocols/http/get-redirects.yaml": &httpGetRedirects{}, - "protocols/http/get-host-redirects.yaml": &httpGetHostRedirects{}, - "protocols/http/disable-redirects.yaml": &httpDisableRedirects{}, - "protocols/http/get.yaml": &httpGet{}, - "protocols/http/post-body.yaml": &httpPostBody{}, - "protocols/http/post-json-body.yaml": &httpPostJSONBody{}, - "protocols/http/post-multipart-body.yaml": &httpPostMultipartBody{}, - "protocols/http/raw-cookie-reuse.yaml": &httpRawCookieReuse{}, - "protocols/http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{}, - "protocols/http/raw-get-query.yaml": &httpRawGetQuery{}, - "protocols/http/raw-get.yaml": &httpRawGet{}, - "protocols/http/raw-with-params.yaml": &httpRawWithParams{}, - "protocols/http/raw-unsafe-with-params.yaml": &httpRawWithParams{}, // Not a typo, functionality is same as above - "protocols/http/raw-path-trailing-slash.yaml": &httpRawPathTrailingSlash{}, - "protocols/http/raw-payload.yaml": &httpRawPayload{}, - "protocols/http/raw-post-body.yaml": &httpRawPostBody{}, - "protocols/http/raw-unsafe-path.yaml": &httpRawUnsafePath{}, - "protocols/http/http-paths.yaml": &httpPaths{}, - "protocols/http/request-condition.yaml": &httpRequestCondition{}, - "protocols/http/request-condition-new.yaml": &httpRequestCondition{}, - "protocols/http/self-contained.yaml": &httpRequestSelfContained{}, - "protocols/http/self-contained-with-path.yaml": &httpRequestSelfContained{}, // Not a typo, functionality is same as above - "protocols/http/self-contained-with-params.yaml": &httpRequestSelfContainedWithParams{}, - "protocols/http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{}, - "protocols/http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, - "protocols/http/get.yaml,protocols/http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, - "protocols/http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, - "protocols/http/dsl-matcher-variable.yaml": &httpDSLVariable{}, - "protocols/http/dsl-functions.yaml": &httpDSLFunctions{}, - "protocols/http/race-simple.yaml": &httpRaceSimple{}, - "protocols/http/race-multiple.yaml": &httpRaceMultiple{}, - "protocols/http/stop-at-first-match.yaml": &httpStopAtFirstMatch{}, - "protocols/http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{}, - "protocols/http/variables.yaml": &httpVariables{}, - "protocols/http/variable-dsl-function.yaml": &httpVariableDSLFunction{}, - "protocols/http/get-override-sni.yaml": &httpSniAnnotation{}, - "protocols/http/get-sni.yaml": &customCLISNI{}, - "protocols/http/redirect-match-url.yaml": &httpRedirectMatchURL{}, - "protocols/http/get-sni-unsafe.yaml": &customCLISNIUnsafe{}, - "protocols/http/annotation-timeout.yaml": &annotationTimeout{}, - "protocols/http/custom-attack-type.yaml": &customAttackType{}, - "protocols/http/get-all-ips.yaml": &scanAllIPS{}, - "protocols/http/get-without-scheme.yaml": &httpGetWithoutScheme{}, - "protocols/http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{}, - "protocols/http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, - "protocols/http/save-extractor-values-to-file.yaml": &httpSaveExtractorValuesToFile{}, - "protocols/http/cli-with-constants.yaml": &ConstantWithCliVar{}, - "protocols/http/disable-path-automerge.yaml": &httpDisablePathAutomerge{}, +var httpTestcases = []TestCaseInfo{ + // TODO: excluded due to parsing errors with console + // "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, + {Path: "protocols/http/get-headers.yaml", TestCase: &httpGetHeaders{}}, + {Path: "protocols/http/get-query-string.yaml", TestCase: &httpGetQueryString{}}, + {Path: "protocols/http/get-redirects.yaml", TestCase: &httpGetRedirects{}}, + {Path: "protocols/http/get-host-redirects.yaml", TestCase: &httpGetHostRedirects{}}, + {Path: "protocols/http/disable-redirects.yaml", TestCase: &httpDisableRedirects{}}, + {Path: "protocols/http/get.yaml", TestCase: &httpGet{}}, + {Path: "protocols/http/post-body.yaml", TestCase: &httpPostBody{}}, + {Path: "protocols/http/post-json-body.yaml", TestCase: &httpPostJSONBody{}}, + {Path: "protocols/http/post-multipart-body.yaml", TestCase: &httpPostMultipartBody{}}, + {Path: "protocols/http/raw-cookie-reuse.yaml", TestCase: &httpRawCookieReuse{}}, + {Path: "protocols/http/raw-dynamic-extractor.yaml", TestCase: &httpRawDynamicExtractor{}}, + {Path: "protocols/http/raw-get-query.yaml", TestCase: &httpRawGetQuery{}}, + {Path: "protocols/http/raw-get.yaml", TestCase: &httpRawGet{}}, + {Path: "protocols/http/raw-with-params.yaml", TestCase: &httpRawWithParams{}}, + {Path: "protocols/http/raw-unsafe-with-params.yaml", TestCase: &httpRawWithParams{}}, // Not a typo, functionality is same as above + {Path: "protocols/http/raw-path-trailing-slash.yaml", TestCase: &httpRawPathTrailingSlash{}}, + {Path: "protocols/http/raw-payload.yaml", TestCase: &httpRawPayload{}}, + {Path: "protocols/http/raw-post-body.yaml", TestCase: &httpRawPostBody{}}, + {Path: "protocols/http/raw-unsafe-path.yaml", TestCase: &httpRawUnsafePath{}}, + {Path: "protocols/http/http-paths.yaml", TestCase: &httpPaths{}}, + {Path: "protocols/http/request-condition.yaml", TestCase: &httpRequestCondition{}}, + {Path: "protocols/http/request-condition-new.yaml", TestCase: &httpRequestCondition{}}, + {Path: "protocols/http/self-contained.yaml", TestCase: &httpRequestSelfContained{}}, + {Path: "protocols/http/self-contained-with-path.yaml", TestCase: &httpRequestSelfContained{}}, // Not a typo, functionality is same as above + {Path: "protocols/http/self-contained-with-params.yaml", TestCase: &httpRequestSelfContainedWithParams{}}, + {Path: "protocols/http/self-contained-file-input.yaml", TestCase: &httpRequestSelfContainedFileInput{}}, + {Path: "protocols/http/get-case-insensitive.yaml", TestCase: &httpGetCaseInsensitive{}}, + {Path: "protocols/http/get.yaml,protocols/http/get-case-insensitive.yaml", TestCase: &httpGetCaseInsensitiveCluster{}}, + {Path: "protocols/http/get-redirects-chain-headers.yaml", TestCase: &httpGetRedirectsChainHeaders{}}, + {Path: "protocols/http/dsl-matcher-variable.yaml", TestCase: &httpDSLVariable{}}, + {Path: "protocols/http/dsl-functions.yaml", TestCase: &httpDSLFunctions{}}, + {Path: "protocols/http/race-simple.yaml", TestCase: &httpRaceSimple{}}, + {Path: "protocols/http/race-multiple.yaml", TestCase: &httpRaceMultiple{}}, + {Path: "protocols/http/stop-at-first-match.yaml", TestCase: &httpStopAtFirstMatch{}}, + {Path: "protocols/http/stop-at-first-match-with-extractors.yaml", TestCase: &httpStopAtFirstMatchWithExtractors{}}, + {Path: "protocols/http/variables.yaml", TestCase: &httpVariables{}}, + {Path: "protocols/http/variable-dsl-function.yaml", TestCase: &httpVariableDSLFunction{}}, + {Path: "protocols/http/get-override-sni.yaml", TestCase: &httpSniAnnotation{}}, + {Path: "protocols/http/get-sni.yaml", TestCase: &customCLISNI{}}, + {Path: "protocols/http/redirect-match-url.yaml", TestCase: &httpRedirectMatchURL{}}, + {Path: "protocols/http/get-sni-unsafe.yaml", TestCase: &customCLISNIUnsafe{}}, + {Path: "protocols/http/annotation-timeout.yaml", TestCase: &annotationTimeout{}}, + {Path: "protocols/http/custom-attack-type.yaml", TestCase: &customAttackType{}}, + {Path: "protocols/http/get-all-ips.yaml", TestCase: &scanAllIPS{}}, + {Path: "protocols/http/get-without-scheme.yaml", TestCase: &httpGetWithoutScheme{}}, + {Path: "protocols/http/cl-body-without-header.yaml", TestCase: &httpCLBodyWithoutHeader{}}, + {Path: "protocols/http/cl-body-with-header.yaml", TestCase: &httpCLBodyWithHeader{}}, + {Path: "protocols/http/save-extractor-values-to-file.yaml", TestCase: &httpSaveExtractorValuesToFile{}}, + {Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}}, + {Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}}, + {Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}}, } type httpInteractshRequest struct{} @@ -169,7 +172,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error { if err != nil { return err } - // polling is asyncronous, so the interactions may be retrieved after the first request + // polling is asynchronous, so the interactions may be retrieved after the first request return expectResultsCount(results, 1) } @@ -359,7 +362,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error { resultPart = stringsutil.TrimPrefixAny(resultPart, "/", " ", "[") extracted := strings.Split(resultPart, ",") - numberOfDslFunctions := 87 + numberOfDslFunctions := 88 if len(extracted) != numberOfDslFunctions { return errors.New("incorrect number of results") } @@ -1423,6 +1426,24 @@ func (h *ConstantWithCliVar) Execute(filePath string) error { return expectResultsCount(got, 1) } +type matcherStatusTest struct{} + +// Execute executes a test case and returns an error if occurred +func (h *matcherStatusTest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-ms") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + // disable path automerge in raw request type httpDisablePathAutomerge struct{} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index f43c74efc..66d562ccf 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -4,7 +4,7 @@ import ( "flag" "fmt" "os" - "sort" + "runtime" "strings" "github.com/logrusorgru/aurora" @@ -13,6 +13,12 @@ import ( sliceutil "github.com/projectdiscovery/utils/slice" ) +type TestCaseInfo struct { + Path string + TestCase testutils.TestCase + DisableOn func() bool +} + var ( debug = os.Getenv("DEBUG") == "true" githubAction = os.Getenv("GH_ACTION") == "true" @@ -21,7 +27,7 @@ var ( success = aurora.Green("[✓]").String() failed = aurora.Red("[✘]").String() - protocolTests = map[string]map[string]testutils.TestCase{ + protocolTests = map[string][]TestCaseInfo{ "http": httpTestcases, "interactsh": interactshTestCases, "network": networkTestcases, @@ -42,6 +48,7 @@ var ( "code": codeTestCases, "multi": multiProtoTestcases, "generic": genericTestcases, + "dsl": dslTestcases, } // For debug purposes @@ -100,18 +107,18 @@ func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCou } func debugTests() { - keys := getMapKeys(protocolTests[runProtocol]) - for _, tpath := range keys { - testcase := protocolTests[runProtocol][tpath] - if runTemplate != "" && !strings.Contains(tpath, runTemplate) { + testCaseInfos := protocolTests[runProtocol] + for _, testCaseInfo := range testCaseInfos { + if (runTemplate != "" && !strings.Contains(testCaseInfo.Path, runTemplate)) || + (testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn()) { continue } if runProtocol == "interactsh" { - if _, err := executeWithRetry(testcase, tpath, interactshRetryCount); err != nil { + if _, err := executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount); err != nil { fmt.Printf("\n%v", err.Error()) } } else { - if _, err := execute(testcase, tpath); err != nil { + if _, err := execute(testCaseInfo.TestCase, testCaseInfo.Path); err != nil { fmt.Printf("\n%v", err.Error()) } } @@ -121,21 +128,22 @@ func debugTests() { func runTests(customTemplatePaths []string) []string { var failedTestTemplatePaths []string - for proto, testCases := range protocolTests { + for proto, testCaseInfos := range protocolTests { if len(customTemplatePaths) == 0 { fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto)) } - keys := getMapKeys(testCases) - - for _, templatePath := range keys { - testCase := testCases[templatePath] - if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, templatePath) { + for _, testCaseInfo := range testCaseInfos { + if testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn() { + fmt.Printf("skipping test case %v. disabled on %v.\n", aurora.Blue(testCaseInfo.Path), runtime.GOOS) + continue + } + if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, testCaseInfo.Path) { var failedTemplatePath string var err error - if proto == "interactsh" || strings.Contains(templatePath, "interactsh") { - failedTemplatePath, err = executeWithRetry(testCase, templatePath, interactshRetryCount) + if proto == "interactsh" || strings.Contains(testCaseInfo.Path, "interactsh") { + failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount) } else { - failedTemplatePath, err = execute(testCase, templatePath) + failedTemplatePath, err = execute(testCaseInfo.TestCase, testCaseInfo.Path) } if err != nil { failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath) @@ -170,12 +178,3 @@ func normalizeSplit(str string) []string { return r == ',' }) } - -func getMapKeys[T any](testcases map[string]T) []string { - keys := make([]string, 0, len(testcases)) - for k := range testcases { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} diff --git a/v2/cmd/integration-test/interactsh.go b/v2/cmd/integration-test/interactsh.go index eab5693e2..7b45be6fe 100644 --- a/v2/cmd/integration-test/interactsh.go +++ b/v2/cmd/integration-test/interactsh.go @@ -1,10 +1,10 @@ package main -import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +import osutils "github.com/projectdiscovery/utils/os" // All Interactsh related testcases -var interactshTestCases = map[string]testutils.TestCase{ - "protocols/http/interactsh.yaml": &httpInteractshRequest{}, - "protocols/http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, - "protocols/http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{}, +var interactshTestCases = []TestCaseInfo{ + {Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, + {Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, + {Path: "protocols/http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, } diff --git a/v2/cmd/integration-test/library.go b/v2/cmd/integration-test/library.go index 551580716..0dbce8724 100644 --- a/v2/cmd/integration-test/library.go +++ b/v2/cmd/integration-test/library.go @@ -34,9 +34,9 @@ import ( "github.com/projectdiscovery/ratelimit" ) -var libraryTestcases = map[string]testutils.TestCase{ - "library/test.yaml": &goIntegrationTest{}, - "library/test.json": &goIntegrationTest{}, +var libraryTestcases = []TestCaseInfo{ + {Path: "library/test.yaml", TestCase: &goIntegrationTest{}}, + {Path: "library/test.json", TestCase: &goIntegrationTest{}}, } type goIntegrationTest struct{} diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 4f4c06056..6c0ccae61 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -12,13 +12,13 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var loaderTestcases = map[string]testutils.TestCase{ - "loader/template-list.yaml": &remoteTemplateList{}, - "loader/workflow-list.yaml": &remoteWorkflowList{}, - "loader/excluded-template.yaml": &excludedTemplate{}, - "loader/nonexistent-template-list.yaml": &nonExistentTemplateList{}, - "loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{}, - "loader/template-list-not-allowed.yaml": &remoteTemplateListNotAllowed{}, +var loaderTestcases = []TestCaseInfo{ + {Path: "loader/template-list.yaml", TestCase: &remoteTemplateList{}}, + {Path: "loader/workflow-list.yaml", TestCase: &remoteWorkflowList{}}, + {Path: "loader/excluded-template.yaml", TestCase: &excludedTemplate{}}, + {Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}}, + {Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}}, + {Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}}, } type remoteTemplateList struct{} diff --git a/v2/cmd/integration-test/multi.go b/v2/cmd/integration-test/multi.go index 2868f3e5b..a7998ab35 100644 --- a/v2/cmd/integration-test/multi.go +++ b/v2/cmd/integration-test/multi.go @@ -2,10 +2,10 @@ package main import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" -var multiProtoTestcases = map[string]testutils.TestCase{ - "multi/dynamic-values.yaml": &multiProtoDynamicExtractor{}, - "multi/evaluate-variables.yaml": &multiProtoDynamicExtractor{}, // Not a typo execution is same as above testcase - "multi/exported-response-vars.yaml": &multiProtoDynamicExtractor{}, // Not a typo execution is same as above testcase +var multiProtoTestcases = []TestCaseInfo{ + {Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}}, + {Path: "protocols/multi/evaluate-variables.yaml", TestCase: &multiProtoDynamicExtractor{}}, + {Path: "protocols/multi/exported-response-vars.yaml", TestCase: &multiProtoDynamicExtractor{}}, } type multiProtoDynamicExtractor struct{} diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 809073389..c0b660e97 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -4,14 +4,16 @@ import ( "net" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + osutils "github.com/projectdiscovery/utils/os" ) -var networkTestcases = map[string]testutils.TestCase{ - "protocols/network/basic.yaml": &networkBasic{}, - "protocols/network/hex.yaml": &networkBasic{}, - "protocols/network/multi-step.yaml": &networkMultiStep{}, - "protocols/network/self-contained.yaml": &networkRequestSelContained{}, - "protocols/network/variables.yaml": &networkVariables{}, +var networkTestcases = []TestCaseInfo{ + {Path: "protocols/network/basic.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }}, + {Path: "protocols/network/hex.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }}, + {Path: "protocols/network/multi-step.yaml", TestCase: &networkMultiStep{}}, + {Path: "protocols/network/self-contained.yaml", TestCase: &networkRequestSelContained{}}, + {Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}}, + {Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}}, } const defaultStaticPort = 5431 diff --git a/v2/cmd/integration-test/offline-http.go b/v2/cmd/integration-test/offline-http.go index 6a4ed3988..0a9500ca3 100644 --- a/v2/cmd/integration-test/offline-http.go +++ b/v2/cmd/integration-test/offline-http.go @@ -6,10 +6,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var offlineHttpTestcases = map[string]testutils.TestCase{ - "protocols/offlinehttp/rfc-req-resp.yaml": &RfcRequestResponse{}, - "protocols/offlinehttp/offline-allowed-paths.yaml": &RequestResponseWithAllowedPaths{}, - "protocols/offlinehttp/offline-raw.yaml": &RawRequestResponse{}, +var offlineHttpTestcases = []TestCaseInfo{ + {Path: "protocols/offlinehttp/rfc-req-resp.yaml", TestCase: &RfcRequestResponse{}}, + {Path: "protocols/offlinehttp/offline-allowed-paths.yaml", TestCase: &RequestResponseWithAllowedPaths{}}, + {Path: "protocols/offlinehttp/offline-raw.yaml", TestCase: &RawRequestResponse{}}, } type RfcRequestResponse struct{} diff --git a/v2/cmd/integration-test/ssl.go b/v2/cmd/integration-test/ssl.go index a821cc77b..ecb790665 100644 --- a/v2/cmd/integration-test/ssl.go +++ b/v2/cmd/integration-test/ssl.go @@ -7,12 +7,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var sslTestcases = map[string]testutils.TestCase{ - "protocols/ssl/basic.yaml": &sslBasic{}, - "protocols/ssl/basic-ztls.yaml": &sslBasicZtls{}, - "protocols/ssl/custom-cipher.yaml": &sslCustomCipher{}, - "protocols/ssl/custom-version.yaml": &sslCustomVersion{}, - "protocols/ssl/ssl-with-vars.yaml": &sslWithVars{}, +var sslTestcases = []TestCaseInfo{ + {Path: "protocols/ssl/basic.yaml", TestCase: &sslBasic{}}, + {Path: "protocols/ssl/basic-ztls.yaml", TestCase: &sslBasicZtls{}}, + {Path: "protocols/ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}}, + {Path: "protocols/ssl/custom-version.yaml", TestCase: &sslCustomVersion{}}, + {Path: "protocols/ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}}, } type sslBasic struct{} diff --git a/v2/cmd/integration-test/template-dir.go b/v2/cmd/integration-test/template-dir.go index 65dc6cfcf..fb2f74c08 100644 --- a/v2/cmd/integration-test/template-dir.go +++ b/v2/cmd/integration-test/template-dir.go @@ -7,8 +7,8 @@ import ( errorutil "github.com/projectdiscovery/utils/errors" ) -var templatesDirTestCases = map[string]testutils.TestCase{ - "protocols/dns/cname-fingerprint.yaml": &templateDirWithTargetTest{}, +var templatesDirTestCases = []TestCaseInfo{ + {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templateDirWithTargetTest{}}, } type templateDirWithTargetTest struct{} diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 935f60ea6..773f91925 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -12,15 +12,15 @@ func getTemplatePath() string { return config.DefaultConfig.TemplatesDirectory } -var templatesPathTestCases = map[string]testutils.TestCase{ +var templatesPathTestCases = []TestCaseInfo{ //template folder path issue - "protocols/http/get.yaml": &folderPathTemplateTest{}, + {Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd - "./protocols/dns/cname-fingerprint.yaml": &cwdTemplateTest{}, + {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, //relative path - "protocols/dns/cname-fingerprint.yaml": &relativePathTemplateTest{}, + {Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path - fmt.Sprintf("%v/dns/cname-fingerprint.yaml", getTemplatePath()): &absolutePathTemplateTest{}, + {Path: fmt.Sprintf("%v/dns/dns-saas-service-detection.yaml", getTemplatePath()), TestCase: &absolutePathTemplateTest{}}, } type cwdTemplateTest struct{} diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index e9b66d0a2..fc6508e05 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -9,11 +9,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var websocketTestCases = map[string]testutils.TestCase{ - "protocols/websocket/basic.yaml": &websocketBasic{}, - "protocols/websocket/cswsh.yaml": &websocketCswsh{}, - "protocols/websocket/no-cswsh.yaml": &websocketNoCswsh{}, - "protocols/websocket/path.yaml": &websocketWithPath{}, +var websocketTestCases = []TestCaseInfo{ + {Path: "protocols/websocket/basic.yaml", TestCase: &websocketBasic{}}, + {Path: "protocols/websocket/cswsh.yaml", TestCase: &websocketCswsh{}}, + {Path: "protocols/websocket/no-cswsh.yaml", TestCase: &websocketNoCswsh{}}, + {Path: "protocols/websocket/path.yaml", TestCase: &websocketWithPath{}}, } type websocketBasic struct{} diff --git a/v2/cmd/integration-test/whois.go b/v2/cmd/integration-test/whois.go index d8feb3247..8e3954e50 100644 --- a/v2/cmd/integration-test/whois.go +++ b/v2/cmd/integration-test/whois.go @@ -4,8 +4,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var whoisTestCases = map[string]testutils.TestCase{ - "protocols/whois/basic.yaml": &whoisBasic{}, +var whoisTestCases = []TestCaseInfo{ + {Path: "protocols/whois/basic.yaml", TestCase: &whoisBasic{}}, } type whoisBasic struct{} diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index fc22d8d8f..4cfb95b16 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -11,14 +11,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var workflowTestcases = map[string]testutils.TestCase{ - "workflow/basic.yaml": &workflowBasic{}, - "workflow/condition-matched.yaml": &workflowConditionMatched{}, - "workflow/condition-unmatched.yaml": &workflowConditionUnmatch{}, - "workflow/matcher-name.yaml": &workflowMatcherName{}, - "workflow/http-value-share-workflow.yaml": &workflowHttpKeyValueShare{}, - "workflow/dns-value-share-workflow.yaml": &workflowDnsKeyValueShare{}, - "workflow/shared-cookie.yaml": &workflowSharedCookies{}, +var workflowTestcases = []TestCaseInfo{ + {Path: "workflow/basic.yaml", TestCase: &workflowBasic{}}, + {Path: "workflow/condition-matched.yaml", TestCase: &workflowConditionMatched{}}, + {Path: "workflow/condition-unmatched.yaml", TestCase: &workflowConditionUnmatch{}}, + {Path: "workflow/matcher-name.yaml", TestCase: &workflowMatcherName{}}, + {Path: "workflow/http-value-share-workflow.yaml", TestCase: &workflowHttpKeyValueShare{}}, + {Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}}, + {Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}}, } type workflowBasic struct{} diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 70380cea4..1554eea1d 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -204,7 +204,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVar(&options.Tags, "tags", nil, "templates to run based on tags (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", nil, "templates to exclude based on tags (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.IncludeTags, "include-tags", "itags", nil, "tags to be executed even if they are excluded either by default or configuration", goflags.FileNormalizedStringSliceOptions), // TODO show default deny list - flagSet.StringSliceVarP(&options.IncludeIds, "template-id", "id", nil, "templates to run based on template ids (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.IncludeIds, "template-id", "id", nil, "templates to run based on template ids (comma-separated, file, allow-wildcard)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", nil, "templates to exclude based on template ids (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", nil, "templates to be executed even if they are excluded either by default or configuration", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", nil, "template or template directory to exclude (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), @@ -223,7 +223,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"), - flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), + flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"), + flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"), flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"), flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"), @@ -253,9 +254,10 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"), - flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), + flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"), - flagSet.BoolVar(&options.Sandbox, "sandbox", false, "sandbox nuclei for safe templates execution"), + flagSet.BoolVarP(&options.AllowLocalFileAccess, "allow-local-file-access", "lfa", false, "allows file (payload) access anywhere on the system"), + flagSet.BoolVarP(&options.RestrictLocalNetworkAccess, "restrict-local-network-access", "lna", false, "blocks connections to the local / private network"), flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"), flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"), flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"), @@ -323,6 +325,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support (root user on Linux will disable sandbox)"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), + flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "use local installed Chrome browser instead of nuclei installed"), flagSet.BoolVarP(&options.ShowActions, "list-headless-action", "lha", false, "list available headless actions"), ) @@ -472,11 +475,11 @@ func printTemplateVersion() { if fileutil.FolderExists(cfg.CustomS3TemplatesDirectory) { gologger.Info().Msgf("Custom S3 templates location: %s\n", cfg.CustomS3TemplatesDirectory) } - if fileutil.FolderExists(cfg.CustomGithubTemplatesDirectory) { - gologger.Info().Msgf("Custom Github templates location: %s ", cfg.CustomGithubTemplatesDirectory) + if fileutil.FolderExists(cfg.CustomGitHubTemplatesDirectory) { + gologger.Info().Msgf("Custom GitHub templates location: %s ", cfg.CustomGitHubTemplatesDirectory) } if fileutil.FolderExists(cfg.CustomGitLabTemplatesDirectory) { - gologger.Info().Msgf("Custom Gitlab templates location: %s ", cfg.CustomGitLabTemplatesDirectory) + gologger.Info().Msgf("Custom GitLab templates location: %s ", cfg.CustomGitLabTemplatesDirectory) } if fileutil.FolderExists(cfg.CustomAzureTemplatesDirectory) { gologger.Info().Msgf("Custom Azure templates location: %s ", cfg.CustomAzureTemplatesDirectory) diff --git a/v2/cmd/nuclei/test.yaml b/v2/cmd/nuclei/test.yaml deleted file mode 100644 index ecb62d2ec..000000000 --- a/v2/cmd/nuclei/test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: basic-example - -info: - name: Test HTTP Template - author: pdteam - severity: info - -http: - - raw: - - |+ - GET / HTTP/1.1 - Host: {{Hostname}} - - unsafe: true - matchers: - - type: dsl - dsl: - - true \ No newline at end of file diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go index 97f8d6122..850ef5529 100644 --- a/v2/cmd/tmc/main.go +++ b/v2/cmd/tmc/main.go @@ -76,8 +76,6 @@ func init() { } } -var idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)") - type options struct { input string errorLogFile string @@ -92,7 +90,7 @@ type options struct { func main() { opts := options{} flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`TemplateMan CLI is baisc utility built on the TemplateMan API to standardize nuclei templates.`) + flagSet.SetDescription(`TemplateMan CLI is basic utility built on the TemplateMan API to standardize nuclei templates.`) flagSet.CreateGroup("Input", "input", flagSet.StringVarP(&opts.input, "input", "i", "", "Templates to annotate"), @@ -206,11 +204,6 @@ func process(opts options) error { } if opts.enhance { - // currently enhance api only supports cve-id's - matches := idRegex.FindAllStringSubmatch(dataString, 1) - if len(matches) == 0 { - continue - } enhancedTemplateData, isEnhanced, err := enhanceTemplate(dataString) if err != nil { gologger.Info().Label("enhance").Msg(logErrMsg(path, err, opts.debug, errFile)) diff --git a/v2/examples/simple.go b/v2/examples/simple.go index ea7fe130c..e1f9a4cd4 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -5,12 +5,13 @@ import ( "fmt" "log" "os" - "path" + "path/filepath" "time" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" @@ -47,7 +48,7 @@ func main() { protocolstate.Init(defaultOpts) protocolinit.Init(defaultOpts) - defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"} + defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) @@ -58,7 +59,7 @@ func main() { defer interactClient.Close() home, _ := os.UserHomeDir() - catalog := disk.NewCatalog(path.Join(home, "nuclei-templates")) + catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates")) executerOpts := protocols.ExecutorOptions{ Output: outputWriter, Options: defaultOpts, @@ -86,9 +87,20 @@ func main() { } store.Load() + // flat input without probe inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}} - input := &inputs.SimpleInputProvider{Inputs: inputArgs} + + httpxOptions := httpx.DefaultOptions + httpxOptions.Timeout = 5 * time.Second + httpxClient, err := httpx.New(&httpxOptions) + if err != nil { + log.Fatal(err) + } + + // use httpx to probe the URL => https://scanme.sh + input.SetWithProbe("scanme.sh", httpxClient) + _ = engine.Execute(store.Templates(), input) engine.WorkPool().Wait() // Wait for the scan to finish } diff --git a/v2/gh_retry.sh b/v2/gh_retry.sh index b8516ec92..bf212e129 100755 --- a/v2/gh_retry.sh +++ b/v2/gh_retry.sh @@ -17,7 +17,7 @@ LIMIT=30 BEFORE="30 mins ago" WORKFLOW="Build Test" -# You can add multiple patterns seperated by | +# You can add multiple patterns separated by | GREP_ERROR_PATTERN='Test "http/interactsh.yaml" failed' #Set fonts for Help. @@ -81,7 +81,7 @@ function retry_failed_jobs() { select ( .conclusion=="failure" ) | select ( .updatedAt > $date) ' --arg date "$date" --arg branch "$BRANCH" --arg workflow "$WORKFLOW" | jq .databaseId) - # convert line seperated by space to array + # convert line separated by space to array eval "arr=($workflowIds)" if [[ -z $arr ]] diff --git a/v2/go.mod b/v2/go.mod index 28ddc1ded..053960407 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/go-playground/validator/v10 v10.14.1 - github.com/go-rod/rod v0.113.0 + github.com/go-rod/rod v0.114.0 github.com/gobwas/ws v1.2.1 github.com/google/go-github v17.0.0+incompatible github.com/itchyny/gojq v0.12.13 @@ -20,18 +20,18 @@ require ( github.com/miekg/dns v1.1.55 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/clistats v0.0.18 - github.com/projectdiscovery/fastdialer v0.0.31 + github.com/projectdiscovery/clistats v0.0.19 + github.com/projectdiscovery/fastdialer v0.0.35 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 - github.com/projectdiscovery/rawhttp v0.1.13 - github.com/projectdiscovery/retryabledns v1.0.30 - github.com/projectdiscovery/retryablehttp-go v1.0.18 + github.com/projectdiscovery/rawhttp v0.1.18 + github.com/projectdiscovery/retryabledns v1.0.32 + github.com/projectdiscovery/retryablehttp-go v1.0.20 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.23.5 // indirect + github.com/shirou/gopsutil/v3 v3.23.6 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 @@ -39,26 +39,26 @@ require ( github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.11.0 - golang.org/x/oauth2 v0.9.0 - golang.org/x/text v0.10.0 + golang.org/x/net v0.12.0 + golang.org/x/oauth2 v0.10.0 + golang.org/x/text v0.11.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/DataDog/gostackparse v0.6.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 - github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 - github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 + github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go-v2/config v1.18.28 + github.com/aws/aws-sdk-go-v2/credentials v1.13.27 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 + github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/docker/go-units v0.5.0 github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 @@ -66,20 +66,20 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd + github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.10 - github.com/projectdiscovery/gologger v1.1.10 + github.com/projectdiscovery/goflags v0.1.12 + github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 - github.com/projectdiscovery/httpx v1.3.0 + github.com/projectdiscovery/httpx v1.3.4 github.com/projectdiscovery/mapcidr v1.1.2 - github.com/projectdiscovery/ratelimit v0.0.8 + github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.0 + github.com/projectdiscovery/tlsx v1.1.1 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.39 - github.com/projectdiscovery/wappalyzergo v0.0.102 + github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 + github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 @@ -92,21 +92,21 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bits-and-blooms/bitset v1.3.1 // indirect github.com/bits-and-blooms/bloom/v3 v3.4.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect - github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaukas/godicttls v0.0.3 // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect @@ -126,10 +126,10 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/projectdiscovery/asnmap v1.0.4 // indirect - github.com/projectdiscovery/cdncheck v1.0.6 // indirect + github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/refraction-networking/utls v1.3.2 // indirect - github.com/sashabaranov/go-openai v1.11.2 // indirect + github.com/sashabaranov/go-openai v1.14.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect @@ -191,7 +191,7 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mholt/acmez v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.24 // indirect + github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -212,18 +212,18 @@ require ( github.com/ysmood/leakless v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect + github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.10.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/mod v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/crypto v0.11.0 + golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 + golang.org/x/mod v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect + golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.29.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect ) @@ -233,14 +233,14 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/alecthomas/chroma v0.10.0 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect diff --git a/v2/go.sum b/v2/go.sum index 5aa2d42bc..082cecc25 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -9,8 +9,9 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybI github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1vBI4o= @@ -51,8 +52,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= @@ -69,57 +70,42 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= +github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= -github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 h1:fI9/5BDEaAv/pv1VO1X1n3jfP9it+IGqWsCuuBQI8wM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67/go.mod h1:zQClPRIwQZfJlZq6WZve+s4Tb4JW+3V6eS+4+KrYeP8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= +github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= +github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27/go.mod h1:syOqAek45ZXZp29HlnRS/BNgMIW6uiRmeuQsz4Qh2UE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5/go.mod h1:Gj7tm95r+QsDoN2Fhuz/3npQvcZbkEf5mL70n3Xfluc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 h1:m0MmP89v1B0t3b8W8rtATU76KNsodak69QtiokHyEvo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72/go.mod h1:ylOTxIuoTL+XjH46Omv2iPjHdeGUk3SQ4hxYho4EHMA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27/go.mod h1:ZdjYvJpDlefgh8/hWelJhqgqJeodxu4SmbVsSdBlL7E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o= -github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30/go.mod h1:qQtIBl5OVMfmeQkz8HaVyh5DzFmmFXyvK27UgIgOr4c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4/go.mod h1:JniVpqvw90sVjNqanGLufrVapWySL28fhBlYgl96Q/w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0/go.mod h1:PwyKKVL0cNkC37QwLcrhyeCrAk+5bY8O2ou7USyAS2A= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13/go.mod h1:BzqsVVFduubEmzrVtUFQQIQdFqvUItF8XUq2EnS8Wog= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2OZgH1ypC114H04U= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= @@ -140,8 +126,8 @@ github.com/caddyserver/certmagic v0.17.2 h1:o30seC1T/dBqBCNNGNHWwj2i5/I/FMjBbTAh github.com/caddyserver/certmagic v0.17.2/go.mod h1:ouWUuC490GOLJzkyN35eXfV8bSbwMwSf4bdhkIxtdQE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= -github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= -github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= +github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -172,8 +158,8 @@ github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3O github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -202,8 +188,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-rod/rod v0.113.0 h1:E7+GLjYVZnScewIB2u8+66joQLaDGbOLzSOT4orNHms= -github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= +github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -353,8 +339,8 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= -github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= +github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= +github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= @@ -370,6 +356,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= @@ -405,55 +392,55 @@ github.com/projectdiscovery/asnmap v1.0.4 h1:dmXrls7Y0Sdeb6cLlKGwdYX8h1K9q2iYOGX github.com/projectdiscovery/asnmap v1.0.4/go.mod h1:iTLDyYsblEwYHcLiKZCRx8Et+xV7NlvgGLusANpgegc= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6uGfPofVpC4c= -github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= -github.com/projectdiscovery/clistats v0.0.18 h1:WLQNqLXsKvjoieDwXJO/1jlnxR0x9vdFaRUAR3gXfKQ= -github.com/projectdiscovery/clistats v0.0.18/go.mod h1:YUnUrMHFw+FHwUTIKr1KDUwz81x+SFjPU3xfLqXfzf0= -github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd h1:16DMjd4HeACrC9CkWJkkLeSh+LYPDorwNx11BlTbonU= -github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd/go.mod h1:S72Cq/lfxzkldf64Sul1G2KFbGKNgpRFFCF/FazpznM= -github.com/projectdiscovery/fastdialer v0.0.31 h1:eu0wTBCWjT8dXChmBtnQaAxoFpkLdvq0VroRxZoe/M8= -github.com/projectdiscovery/fastdialer v0.0.31/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= +github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= +github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= +github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= +github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= +github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= +github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= +github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= +github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.10 h1:Gompf8JDy8y+5c4eWlc70KKtPuDH/hqFB3tMeHcMiKk= -github.com/projectdiscovery/goflags v0.1.10/go.mod h1:MHEkqm3XgxBf5fK4gr3IXsj6VeLTq4qJYGC/4JRYQ74= -github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoii4P2yDovqck= -github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= +github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0= +github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= +github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= +github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6/go.mod h1:jCpXNvLUCPMzm5AhJv8wtnUt/7rz0TY2SsqvKQ8tn2E= github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= -github.com/projectdiscovery/httpx v1.3.0 h1:x7XbAUd1i3SL/SqVBZFM5EXv7YRXL58jdwonwxR4bvE= -github.com/projectdiscovery/httpx v1.3.0/go.mod h1:tglYwSUBz1zl2cC3NtjgURrI3cZsGepH1DLyQZZY2/M= +github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= +github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= github.com/projectdiscovery/interactsh v1.1.4/go.mod h1:rM8IEm6AAm68fWWExzBHjhBWfRhFYzR9gY5emOFiZCY= github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4tQsLPCrsA= github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= -github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= -github.com/projectdiscovery/rawhttp v0.1.13 h1:Xn3NY3SYIk0151K5Qfuvx3tayl2UOoxMuVyYvGT95BA= -github.com/projectdiscovery/rawhttp v0.1.13/go.mod h1:AjZUYdPCx4xqeWYPqFPLGCxQsVFeUrobxidnU6Nta8M= +github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8= +github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I= +github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt31kQO6P+41s= +github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0= -github.com/projectdiscovery/retryabledns v1.0.30/go.mod h1:+Aqc0TjKGcTtP0HtXE8o1GzrjAHhSno6hSF+L63TBtI= -github.com/projectdiscovery/retryablehttp-go v1.0.18 h1:3IUxyIOOUVSGEBm4pV0cQSk1i/DausZdHePdGDip0Lg= -github.com/projectdiscovery/retryablehttp-go v1.0.18/go.mod h1:oE3dmYWMadFWzaIfG1IqINsYAzUWYUtdI4PJ2xo7cXg= +github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo= +github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c= +github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE= +github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpWBJBG6I= -github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= +github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= +github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.39 h1:iyi5qPilENRmFyt16qtd58pb65fUu0wAU2C0Lq5t6zo= -github.com/projectdiscovery/utils v0.0.39/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= -github.com/projectdiscovery/wappalyzergo v0.0.102 h1:ABjZghof2U2yzGNL+q5ouWHEardLd2o53Ukgrf8CZzE= -github.com/projectdiscovery/wappalyzergo v0.0.102/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 h1:bgTXdrA/yFhFGfjhMIsczVNhnsMEHFidgS/FD2Tq5Js= +github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= +github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= @@ -469,15 +456,15 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.11.2 h1:HuMf+18eldSKbqVblyeCQbtcqSpGVfqTshvi8Bn6zes= -github.com/sashabaranov/go-openai v1.11.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.14.1 h1:jqfkdj8XHnBF84oi2aNtT8Ktp3EJ0MfuVjvcMkfI0LA= +github.com/sashabaranov/go-openai v1.14.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= +github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -485,6 +472,7 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -511,7 +499,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -554,11 +541,9 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= +github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/xanzy/go-gitlab v0.84.0 h1:PdpCaskQSgcVDsx21c6ikf8Rfyo7SNtFAJwP9PrbCFE= github.com/xanzy/go-gitlab v0.84.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= @@ -596,8 +581,12 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= -github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= +github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= +github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= +github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -622,6 +611,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -629,17 +619,17 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -648,9 +638,9 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -663,19 +653,19 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= -golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -690,6 +680,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -715,8 +706,9 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -725,18 +717,19 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -748,8 +741,8 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -761,8 +754,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -790,7 +783,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go index 12dd85a55..22caa5d7f 100644 --- a/v2/internal/installer/template.go +++ b/v2/internal/installer/template.go @@ -57,7 +57,9 @@ func (t *templateUpdateResults) String() string { // TemplateManager is a manager for templates. // It downloads / updates / installs templates. type TemplateManager struct { - CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates + CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates + DisablePublicTemplates bool // if true, + // public templates are not downloaded from the GitHub nuclei-templates repository } // FreshInstallIfNotExists installs templates if they are not already installed @@ -78,7 +80,7 @@ func (t *TemplateManager) FreshInstallIfNotExists() error { // UpdateIfOutdated updates templates if they are outdated func (t *TemplateManager) UpdateIfOutdated() error { - // if folder does not exist, its a fresh install and not update + // if the templates folder does not exist, it's a fresh installation and do not update if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { return t.FreshInstallIfNotExists() } @@ -95,12 +97,16 @@ func (t *TemplateManager) installTemplatesAt(dir string) error { return errorutil.NewWithErr(err).Msgf("failed to create directory at %s", dir) } } + if t.DisablePublicTemplates { + gologger.Info().Msgf("Skipping installation of public nuclei-templates") + return nil + } ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName) if err != nil { return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) } // write templates to disk - if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir) } gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir) @@ -109,10 +115,14 @@ func (t *TemplateManager) installTemplatesAt(dir string) error { // updateTemplatesAt updates templates at given directory func (t *TemplateManager) updateTemplatesAt(dir string) error { - // firstly read checksums from .checksum file these are used to generate stats + if t.DisablePublicTemplates { + gologger.Info().Msgf("Skipping update of public nuclei-templates") + return nil + } + // firstly, read checksums from .checksum file these are used to generate stats oldchecksums, err := t.getChecksumFromDir(dir) if err != nil { - // if something went wrong overwrite all files + // if something went wrong, overwrite all files oldchecksums = make(map[string]string) } @@ -124,7 +134,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error { gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName()) // write templates to disk - if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { return err } @@ -173,10 +183,10 @@ func (t *TemplateManager) summarizeChanges(old, new map[string]string) *template return results } -// getAbsoluteFilePath returns absolute path where a file should be written based on given uri(i.e files in zip) -// if returned path is empty, it means that file should not be written and skipped -func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.FileInfo) string { - // overwrite .nuclei-ignore everytime nuclei-templates are downloaded +// getAbsoluteFilePath returns an absolute path where a file should be written based on given uri(i.e., files in zip) +// if a returned path is empty, it means that file should not be written and skipped +func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.FileInfo) string { + // overwrite .nuclei-ignore every time nuclei-templates are downloaded if f.Name() == config.NucleiIgnoreFileName { return config.DefaultConfig.GetIgnoreFilePath() } @@ -194,9 +204,9 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File if index == -1 { // zip files does not have directory at all , in this case log error but continue gologger.Warning().Msgf("failed to get directory name from uri: %s", uri) - return filepath.Join(templatedir, uri) + return filepath.Join(templateDir, uri) } - // seperator is also included in rootDir + // separator is also included in rootDir rootDirectory := uri[:index+1] relPath := strings.TrimPrefix(uri, rootDirectory) @@ -205,14 +215,14 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File return "" } - newPath := filepath.Clean(filepath.Join(templatedir, relPath)) + newPath := filepath.Clean(filepath.Join(templateDir, relPath)) - if !strings.HasPrefix(newPath, templatedir) { + if !strings.HasPrefix(newPath, templateDir) { // we don't allow LFI return "" } - if newPath == templatedir || newPath == templatedir+string(os.PathSeparator) { + if newPath == templateDir || newPath == templateDir+string(os.PathSeparator) { // skip writing the folder itself since it already exists return "" } @@ -228,12 +238,12 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File } // writeChecksumFileInDir is actual method responsible for writing all templates to directory -func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { - LocaltemplatesIndex, err := config.GetNucleiTemplatesIndex() +func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { + localTemplatesIndex, err := config.GetNucleiTemplatesIndex() if err != nil { gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err) - if LocaltemplatesIndex == nil { - LocaltemplatesIndex = map[string]string{} // no-op + if localTemplatesIndex == nil { + localTemplatesIndex = map[string]string{} // no-op } } @@ -253,10 +263,10 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo // instead of creating it from scratch id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri) if id != "" { - // based on template id, check if we are updating path of official nuclei template - if oldPath, ok := LocaltemplatesIndex[id]; ok { + // based on template id, check if we are updating a path of official nuclei template + if oldPath, ok := localTemplatesIndex[id]; ok { if oldPath != writePath { - // write new template at new path and delete old template + // write new template at a new path and delete old template if err := os.WriteFile(writePath, bin, f.Mode()); err != nil { return errorutil.NewWithErr(err).Msgf("failed to write file %s", uri) } @@ -303,12 +313,12 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index") } - // after installation create and write checksums to .checksum file + // after installation, create and write checksums to .checksum file return t.writeChecksumFileInDir(dir) } // getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) -// if .checksum file does not exist checksums are calculated and returned +// if .checksum file does not exist, checksums are calculated and returned func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, error) { checksumFilePath := config.DefaultConfig.GetChecksumFilePath() if fileutil.FileExists(checksumFilePath) { diff --git a/v2/internal/installer/template_test.go b/v2/internal/installer/template_test.go index e6b0345e2..95f44f856 100644 --- a/v2/internal/installer/template_test.go +++ b/v2/internal/installer/template_test.go @@ -53,7 +53,7 @@ func TestTemplateInstallation(t *testing.T) { // we should have at least 1000 templates require.Greater(t, counter, 1000) - // everytime we install templates, it should override the ignore file with latest one + // every time we install templates, it should override the ignore file with latest one require.FileExists(t, config.DefaultConfig.GetIgnoreFilePath()) t.Logf("Installed %d templates", counter) } diff --git a/v2/internal/installer/util.go b/v2/internal/installer/util.go index c9b1b7c4b..024202a34 100644 --- a/v2/internal/installer/util.go +++ b/v2/internal/installer/util.go @@ -36,7 +36,7 @@ func GetNewTemplatesInVersions(versions ...string) []string { continue } - arr, err := getNewAdditionsFileFromGithub(v) + arr, err := getNewAdditionsFileFromGitHub(v) if err != nil { gologger.Error().Msgf("failed to fetch new additions for %v got: %v", v, err) continue @@ -46,7 +46,7 @@ func GetNewTemplatesInVersions(versions ...string) []string { return allTemplates } -func getNewAdditionsFileFromGithub(version string) ([]string, error) { +func getNewAdditionsFileFromGitHub(version string) ([]string, error) { resp, err := retryableHttpClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version)) if err != nil { return nil, err diff --git a/v2/internal/runner/cloud.go b/v2/internal/runner/cloud.go index 707652227..4d173bf28 100644 --- a/v2/internal/runner/cloud.go +++ b/v2/internal/runner/cloud.go @@ -375,8 +375,8 @@ func (r *Runner) addCloudDataSource(source string) error { return err } case "github": - for _, repo := range r.options.GithubTemplateRepo { - if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil { + for _, repo := range r.options.GitHubTemplateRepo { + if _, err := r.processDataSourceItem(repo, r.options.GitHubToken, "github"); err != nil { return err } } diff --git a/v2/internal/runner/inputs.go b/v2/internal/runner/inputs.go index 1bce3f879..191972a03 100644 --- a/v2/internal/runner/inputs.go +++ b/v2/internal/runner/inputs.go @@ -1,18 +1,16 @@ package runner import ( - "fmt" - "net/http" - "strings" "sync/atomic" "time" - "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/remeh/sizedwaitgroup" ) @@ -25,7 +23,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { if err != nil { return nil, errors.Wrap(err, "could not create temporary input file") } - gologger.Info().Msgf("Running httpx on input host") var bulkSize = probeBulkSize @@ -45,7 +42,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { swg := sizedwaitgroup.New(bulkSize) count := int32(0) r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { - if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") { + if stringsutil.HasPrefixAny(value.Input, "http://", "https://") { return true } @@ -53,7 +50,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { go func(input *contextargs.MetaInput) { defer swg.Done() - if result := probeURL(input.Input, httpxClient); result != "" { + if result := utils.ProbeURL(input.Input, httpxClient); result != "" { atomic.AddInt32(&count, 1) _ = hm.Set(input.Input, []byte(result)) } @@ -65,27 +62,3 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count)) return hm, nil } - -var ( - httpSchemes = []string{"https", "http"} -) - -// probeURL probes the scheme for a URL. first HTTPS is tried -// and if any errors occur http is tried. If none succeeds, probing -// is abandoned for such URLs. -func probeURL(input string, httpxclient *httpx.HTTPX) string { - for _, scheme := range httpSchemes { - formedURL := fmt.Sprintf("%s://%s", scheme, input) - req, err := httpxclient.NewRequest(http.MethodHead, formedURL) - if err != nil { - continue - } - req.Header.Set("User-Agent", uarand.GetRandom()) - - if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { - continue - } - return formedURL - } - return "" -} diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 8481a5da6..831842c72 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -31,7 +31,7 @@ import ( func ConfigureOptions() error { // with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions - // if file has extension `.yaml,.json` we consider those as strings and not files to be read + // if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read isFromFileFunc := func(s string) bool { return !config.IsTemplate(s) } @@ -78,7 +78,7 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) - if err := loadTemplateSignaturesKeys(options); err != nil { + if err := loadTemplateSignaturesKeys(options); err != nil && !getBoolEnvValue("HIDE_TEMPLATE_SIG_WARNING") { gologger.Warning().Msgf("Could not initialize code template verifier: %s\n", err) } @@ -87,9 +87,9 @@ func ParseOptions(options *types.Options) { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } - // Set Github token in env variable. runner.getGHClientWithToken() reads token from env - if options.GithubToken != "" && os.Getenv("GITHUB_TOKEN") != options.GithubToken { - os.Setenv("GITHUB_TOKEN", options.GithubToken) + // Set GitHub token in env variable. runner.getGHClientWithToken() reads token from env + if options.GitHubToken != "" && os.Getenv("GITHUB_TOKEN") != options.GitHubToken { + os.Setenv("GITHUB_TOKEN", options.GitHubToken) } if options.UncoverQuery != nil { @@ -121,6 +121,10 @@ func validateOptions(options *types.Options) error { return errors.New("both verbose and silent mode specified") } + if (options.HeadlessOptionalArguments != nil || options.ShowBrowser || options.UseInstalledChrome) && !options.Headless { + return errors.New("headless mode (-headless) is required if -ho, -sb, -sc or -lha are set") + } + if options.FollowHostRedirects && options.FollowRedirects { return errors.New("both follow host redirects and follow redirects specified") } @@ -143,7 +147,7 @@ func validateOptions(options *types.Options) error { validateCertificatePaths(options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile) } // Verify AWS secrets are passed if a S3 template bucket is passed - if options.AwsBucketName != "" && options.UpdateTemplates { + if options.AwsBucketName != "" && options.UpdateTemplates && !options.AwsTemplateDisableDownload { missing := validateMissingS3Options(options) if missing != nil { return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ",")) @@ -151,7 +155,7 @@ func validateOptions(options *types.Options) error { } // Verify Azure connection configuration is passed if the Azure template bucket is passed - if options.AzureContainerName != "" && options.UpdateTemplates { + if options.AzureContainerName != "" && options.UpdateTemplates && !options.AzureTemplateDisableDownload { missing := validateMissingAzureOptions(options) if missing != nil { return fmt.Errorf("azure connection details are missing. Please provide %s", strings.Join(missing, ",")) @@ -159,7 +163,7 @@ func validateOptions(options *types.Options) error { } // Verify that all GitLab options are provided if the GitLab server or token is provided - if len(options.GitLabTemplateRepositoryIDs) != 0 && options.UpdateTemplates { + if len(options.GitLabTemplateRepositoryIDs) != 0 && options.UpdateTemplates && !options.GitLabTemplateDisableDownload { missing := validateMissingGitLabOptions(options) if missing != nil { return fmt.Errorf("gitlab server details are missing. Please provide %s", strings.Join(missing, ",")) @@ -206,7 +210,7 @@ func validateCloudOptions(options *types.Options) error { case "s3": missing = validateMissingS3Options(options) case "github": - missing = validateMissingGithubOptions(options) + missing = validateMissingGitHubOptions(options) case "gitlab": missing = validateMissingGitLabOptions(options) case "azure": @@ -256,12 +260,12 @@ func validateMissingAzureOptions(options *types.Options) []string { return missing } -func validateMissingGithubOptions(options *types.Options) []string { +func validateMissingGitHubOptions(options *types.Options) []string { var missing []string - if options.GithubToken == "" { + if options.GitHubToken == "" { missing = append(missing, "GITHUB_TOKEN") } - if len(options.GithubTemplateRepo) == 0 { + if len(options.GitHubTemplateRepo) == 0 { missing = append(missing, "GITHUB_TEMPLATE_REPO") } return missing @@ -299,7 +303,7 @@ func configureOutput(options *types.Options) { logutil.DisableDefaultLogger() } -// loadResolvers loads resolvers from both user provided flag and file +// loadResolvers loads resolvers from both user-provided flags and file func loadResolvers(options *types.Options) { if options.ResolversFile == "" { return @@ -366,10 +370,10 @@ func readEnvInputVars(options *types.Options) { } options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_API") - options.GithubToken = os.Getenv("GITHUB_TOKEN") + options.GitHubToken = os.Getenv("GITHUB_TOKEN") repolist := os.Getenv("GITHUB_TEMPLATE_REPO") if repolist != "" { - options.GithubTemplateRepo = append(options.GithubTemplateRepo, stringsutil.SplitAny(repolist, ",")...) + options.GitHubTemplateRepo = append(options.GitHubTemplateRepo, stringsutil.SplitAny(repolist, ",")...) } // GitLab options for downloading templates from a repository @@ -410,6 +414,24 @@ func readEnvInputVars(options *types.Options) { // Custom public keys for template verification options.CodeTemplateSignaturePublicKey = os.Getenv("NUCLEI_SIGNATURE_PUBLIC_KEY") options.CodeTemplateSignatureAlgorithm = os.Getenv("NUCLEI_SIGNATURE_ALGORITHM") + + // General options to disable the template download locations from being used. + // This will override the default behavior of downloading templates from the default locations as well as the + // custom locations. + // The primary use-case is when the user wants to use custom templates only and does not want to download any + // templates from the default locations or is unable to connect to the public internet. + options.PublicTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_PUBLIC_DOWNLOAD") + options.GitHubTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITHUB_DOWNLOAD") + options.GitLabTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITLAB_DOWNLOAD") + options.AwsTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AWS_DOWNLOAD") + options.AzureTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AZURE_DOWNLOAD") + + // Options to modify the behavior of exporters + options.MarkdownExportSortMode = strings.ToLower(os.Getenv("MARKDOWN_EXPORT_SORT_MODE")) + // If the user has not specified a valid sort mode, use the default + if options.MarkdownExportSortMode != "template" && options.MarkdownExportSortMode != "severity" && options.MarkdownExportSortMode != "host" { + options.MarkdownExportSortMode = "" + } } func loadTemplateSignaturesKeys(options *types.Options) error { @@ -440,3 +462,8 @@ func loadTemplateSignaturesKeys(options *types.Options) error { return signer.AddToDefault(verifier) } + +func getBoolEnvValue(key string) bool { + value := os.Getenv(key) + return strings.EqualFold(value, "true") +} diff --git a/v2/internal/runner/options_test.go b/v2/internal/runner/options_test.go new file mode 100644 index 000000000..72a9adb38 --- /dev/null +++ b/v2/internal/runner/options_test.go @@ -0,0 +1,61 @@ +package runner + +import ( + "strings" + "testing" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/stretchr/testify/require" +) + +func TestParseHeadlessOptionalArguments(t *testing.T) { + tests := []struct { + name string + input string + want map[string]string + }{ + { + name: "single value", + input: "a=b", + want: map[string]string{"a": "b"}, + }, + { + name: "empty string", + input: "", + want: map[string]string{}, + }, + { + name: "empty key", + input: "=b", + want: map[string]string{}, + }, + { + name: "empty value", + input: "a=", + want: map[string]string{}, + }, + { + name: "double input", + input: "a=b,c=d", + want: map[string]string{"a": "b", "c": "d"}, + }, + { + name: "duplicated input", + input: "a=b,a=b", + want: map[string]string{"a": "b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strsl := goflags.StringSlice{} + for _, v := range strings.Split(tt.input, ",") { + //nolint + strsl.Set(v) + } + opt := types.Options{HeadlessOptionalArguments: strsl} + got := opt.ParseHeadlessOptionalArguments() + require.Equal(t, tt.want, got) + }) + } +} diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index 5bf464b7a..a4323eeba 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -14,7 +14,7 @@ import ( proxyutils "github.com/projectdiscovery/utils/proxy" ) -// loadProxyServers load list of proxy servers from file or comma seperated +// loadProxyServers load list of proxy servers from file or comma separated func loadProxyServers(options *types.Options) error { if len(options.Proxy) == 0 { return nil diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8b396a783..01c49c9e9 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -112,7 +112,10 @@ func New(options *types.Options) (*Runner, error) { // Check for template updates and update if available. // If the custom templates manager is not nil, we will install custom templates if there is a fresh installation - tm := &installer.TemplateManager{CustomTemplates: ctm} + tm := &installer.TemplateManager{ + CustomTemplates: ctm, + DisablePublicTemplates: options.PublicTemplateDisableDownload, + } if err := tm.FreshInstallIfNotExists(); err != nil { gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err) } @@ -340,10 +343,18 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } if options.MarkdownExportDirectory != "" { if reportingOptions != nil { - reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{ + Directory: options.MarkdownExportDirectory, + IncludeRawPayload: !options.OmitRawRequests, + SortMode: options.MarkdownExportSortMode, + } } else { reportingOptions = &reporting.Options{} - reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{ + Directory: options.MarkdownExportDirectory, + IncludeRawPayload: !options.OmitRawRequests, + SortMode: options.MarkdownExportSortMode, + } } } if options.SarifExport != "" { @@ -356,18 +367,30 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } if options.JSONExport != "" { if reportingOptions != nil { - reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport} + reportingOptions.JSONExporter = &jsonexporter.Options{ + File: options.JSONExport, + IncludeRawPayload: !options.OmitRawRequests, + } } else { reportingOptions = &reporting.Options{} - reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport} + reportingOptions.JSONExporter = &jsonexporter.Options{ + File: options.JSONExport, + IncludeRawPayload: !options.OmitRawRequests, + } } } if options.JSONLExport != "" { if reportingOptions != nil { - reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport} + reportingOptions.JSONLExporter = &jsonl.Options{ + File: options.JSONLExport, + IncludeRawPayload: !options.OmitRawRequests, + } } else { reportingOptions = &reporting.Options{} - reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport} + reportingOptions.JSONLExporter = &jsonl.Options{ + File: options.JSONLExport, + IncludeRawPayload: !options.OmitRawRequests, + } } } diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index de667f745..559448732 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -57,7 +57,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { path = aurora.Cyan(tpl.Path).String() tplBody, err = r.highlightTemplate(&tplBody) if err != nil { - gologger.Error().Msgf("Could not hihglight the template %s: %s", tpl.Path, err) + gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err) continue } @@ -74,7 +74,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { func (r *Runner) highlightTemplate(body *[]byte) ([]byte, error) { var buf bytes.Buffer - // YAML lexer, true color terminar formatter and monokai style + // YAML lexer, true color terminal formatter and monokai style err := quick.Highlight(&buf, string(*body), "yaml", "terminal16m", "monokai") if err != nil { return nil, err diff --git a/v2/pkg/catalog/aws/catalog.go b/v2/pkg/catalog/aws/catalog.go index 1303a4f49..5dfa86a56 100644 --- a/v2/pkg/catalog/aws/catalog.go +++ b/v2/pkg/catalog/aws/catalog.go @@ -129,7 +129,7 @@ func (c Catalog) ResolvePath(templateName, second string) (string, error) { // if c second path is given, it's c folder and we join the two and check against keys if second != "" { - // Note: Do not replace `path` with `filepath` since filepath is aware of Os path seperator + // Note: Do not replace `path` with `filepath` since filepath is aware of Os path separator // and we only see `/` in s3 paths changing it to filepath cause build fail and other errors target := path.Join(path.Dir(second), templateName) for _, key := range keys { diff --git a/v2/pkg/catalog/aws/catalog_test.go b/v2/pkg/catalog/aws/catalog_test.go index fa19a1423..57dac391a 100644 --- a/v2/pkg/catalog/aws/catalog_test.go +++ b/v2/pkg/catalog/aws/catalog_test.go @@ -141,7 +141,7 @@ func TestCatalog_OpenFile(t *testing.T) { false, }, { - "non-existent key", + "nonexistent key", "something/that-doesnt-exist.yaml", true, }, diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 6be99aeda..8b945c4ad 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -14,22 +14,23 @@ const ( NucleiTemplatesIndexFileName = ".templates-index" // contains index of official nuclei templates NucleiTemplatesCheckSumFileName = ".checksum" NewTemplateAdditionsFileName = ".new-additions" - CLIConifgFileName = "config.yaml" + CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei Version = `v3.0.0` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" - CustomGithubTemplatesDirName = "github" + CustomGitHubTemplatesDirName = "github" CustomAzureTemplatesDirName = "azure" CustomGitLabTemplatesDirName = "gitlab" ) -// IsOutdatedVersion compares two versions and returns true if the current version is outdated +// IsOutdatedVersion compares two versions and returns true +// if the current version is outdated func IsOutdatedVersion(current, latest string) bool { if latest == "" { - // if pdtm api call failed it's assumed that current version is outdated - // and it will be confirmed while updating from github + // if pdtm api call failed it's assumed that the current version is outdated + // and it will be confirmed while updating from GitHub // this fixes `version string empty` errors return true } diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index de2e5044a..7dafbb8dc 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -23,16 +23,16 @@ type Config struct { TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` // customtemplates exists in templates directory with the name of custom-templates provider - // below custom paths are absolute paths to respecitive custom-templates directories + // below custom paths are absolute paths to respective custom-templates directories CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` - CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` + CustomGitHubTemplatesDirectory string `json:"custom-github-templates-directory"` CustomGitLabTemplatesDirectory string `json:"custom-gitlab-templates-directory"` CustomAzureTemplatesDirectory string `json:"custom-azure-templates-directory"` TemplateVersion string `json:"nuclei-templates-version,omitempty"` NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` - // Latestxxx are not meant to be used directly and is used as + // LatestXXX are not meant to be used directly and is used as // local cache of nuclei version check endpoint // these fields are only update during nuclei version check // TODO: move these fields to a separate unexported struct as they are not meant to be used directly @@ -83,7 +83,7 @@ func (c *Config) NeedsTemplateUpdate() bool { return !c.disableUpdates && (c.TemplateVersion == "" || IsOutdatedVersion(c.TemplateVersion, c.LatestNucleiTemplatesVersion) || !fileutil.FolderExists(c.TemplatesDirectory)) } -// NeedsIngoreFileUpdate returns true if Ignore file hash is different (aka ignore file is outdated) +// NeedsIgnoreFileUpdate returns true if Ignore file hash is different (aka ignore file is outdated) func (c *Config) NeedsIgnoreFileUpdate() bool { return c.NucleiIgnoreHash == "" || c.NucleiIgnoreHash != c.LatestNucleiIgnoreHash } @@ -111,7 +111,7 @@ func (c *Config) GetConfigDir() string { // GetAllCustomTemplateDirs returns all custom template directories func (c *Config) GetAllCustomTemplateDirs() []string { - return []string{c.CustomS3TemplatesDirectory, c.CustomGithubTemplatesDirectory, c.CustomGitLabTemplatesDirectory, c.CustomAzureTemplatesDirectory} + return []string{c.CustomS3TemplatesDirectory, c.CustomGitHubTemplatesDirectory, c.CustomGitLabTemplatesDirectory, c.CustomAzureTemplatesDirectory} } // GetReportingConfigFilePath returns the nuclei reporting config file path @@ -135,7 +135,7 @@ func (c *Config) GetChecksumFilePath() string { // GetCLIOptsConfigFilePath returns the nuclei cli config file path func (c *Config) GetFlagsConfigFilePath() string { - return filepath.Join(c.configDir, CLIConifgFileName) + return filepath.Join(c.configDir, CLIConfigFileName) } // GetNewAdditions returns new template additions in current template release @@ -188,7 +188,7 @@ func (c *Config) SetTemplatesDir(dirPath string) { } c.TemplatesDirectory = dirPath // Update the custom templates directory - c.CustomGithubTemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) + c.CustomGitHubTemplatesDirectory = filepath.Join(dirPath, CustomGitHubTemplatesDirName) c.CustomS3TemplatesDirectory = filepath.Join(dirPath, CustomS3TemplatesDirName) c.CustomGitLabTemplatesDirectory = filepath.Join(dirPath, CustomGitLabTemplatesDirName) c.CustomAzureTemplatesDirectory = filepath.Join(dirPath, CustomAzureTemplatesDirName) diff --git a/v2/pkg/catalog/disk/find.go b/v2/pkg/catalog/disk/find.go index 77ff0cb24..98b8cf5be 100644 --- a/v2/pkg/catalog/disk/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -48,7 +48,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri } } } - // purge all falsepositivies + // purge all false positives filteredTemplates := []string{} for _, v := range allTemplates { // TODO: this is a temporary fix to avoid treating these files as templates @@ -129,7 +129,7 @@ func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) { // findGlobPathMatches returns the matched files from a glob path func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { - // to support globbing on old paths we use bruteforce to find matches with exit on first match + // to support globbing on old paths we use brute force to find matches with exit on first match // trim templateDir if any relPath := strings.TrimPrefix(absPath, c.templatesDirectory) // trim leading slash if any diff --git a/v2/pkg/catalog/disk/path.go b/v2/pkg/catalog/disk/path.go index 1fbef23ce..18673db29 100644 --- a/v2/pkg/catalog/disk/path.go +++ b/v2/pkg/catalog/disk/path.go @@ -89,7 +89,7 @@ func BackwardsCompatiblePaths(templateDir string, oldPath string) string { // trim the template directory from the path return newPathCallback(tmp) case strings.Contains(oldPath, urlutil.SchemeSeparator): - // scheme seperator is used to identify the path as url + // scheme separator is used to identify the path as url // TBD: add support for url directories ?? return oldPath case strings.Contains(oldPath, "*"): diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index 571c212af..638da945b 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "strings" + "path/filepath" "github.com/Knetic/govaluate" "github.com/projectdiscovery/gologger" @@ -170,9 +171,18 @@ func isIdMatch(tagFilter *TagFilter, templateId string) bool { if len(tagFilter.excludeIds) == 0 && len(tagFilter.allowedIds) == 0 { return true } - included := true - if len(tagFilter.allowedIds) > 0 { - _, included = tagFilter.allowedIds[templateId] + + included := len(tagFilter.allowedIds) == 0 + for id := range tagFilter.allowedIds { + match, err := filepath.Match(id, templateId) + if err != nil { + continue + } + + if match { + included = true + break + } } excluded := false @@ -207,6 +217,7 @@ func tryCollectConditionsMatchinfo(template *templates.Template) map[string]inte parameters["cwe_id"] = template.Info.Classification.CWEID.ToSlice() parameters["cpe"] = template.Info.Classification.CPE parameters["epss_score"] = template.Info.Classification.EPSSScore + parameters["epss_percentile"] = template.Info.Classification.EPSSPercentile } if template.Type() == types.HTTPProtocol { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 703149320..dc65b393d 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -188,7 +188,7 @@ func (store *Store) ValidateTemplates() error { if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) { return nil } - return errors.New("errors occured during template validation") + return errors.New("errors occurred during template validation") } func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool { diff --git a/v2/pkg/core/execute_options.go b/v2/pkg/core/execute_options.go index 6055d9cb6..4654e7928 100644 --- a/v2/pkg/core/execute_options.go +++ b/v2/pkg/core/execute_options.go @@ -29,7 +29,7 @@ func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target return e.ExecuteScanWithOpts(templatesList, target, false) } -// ExecuteScanWithOpts executes scan with given scanStatergy +// ExecuteScanWithOpts executes scan with given scanStrategy func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { results := &atomic.Bool{} selfcontainedWg := &sync.WaitGroup{} diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 0d40b9768..1fb2d73ad 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -277,7 +277,7 @@ func (i *Input) setItem(metaInput *contextargs.MetaInput) { } } -// setHostMapStream sets iteam in stream mode +// setHostMapStream sets item in stream mode func (i *Input) setHostMapStream(data string) { if _, err := i.hostMapStream.Merge([][]byte{[]byte(data)}); err != nil { gologger.Warning().Msgf("%s\n", err) diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go index bb75b6e9c..5e04d7128 100644 --- a/v2/pkg/core/inputs/inputs.go +++ b/v2/pkg/core/inputs/inputs.go @@ -1,6 +1,10 @@ package inputs -import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +import ( + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" +) type SimpleInputProvider struct { Inputs []*contextargs.MetaInput @@ -24,3 +28,12 @@ func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) b func (s *SimpleInputProvider) Set(value string) { s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) } + +// SetWithProbe adds item to input provider with http probing +func (s *SimpleInputProvider) SetWithProbe(value string, httpxClient *httpx.HTTPX) { + valueToAppend := value + if result := utils.ProbeURL(value, httpxClient); result != "" { + valueToAppend = result + } + s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: valueToAppend}) +} diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index b873ed6c1..d1734a580 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -25,14 +25,23 @@ func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Work ctxArgs.MetaInput = input ctxArgs.CookieJar = workflowCookieJar - swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) + // we can know the nesting level only at runtime, so the best we can do here is increase template threads by one unit in case it's equal to 1 to allow + // at least one subtemplate to go through, which it's idempotent to one in-flight template as the parent one is in an idle state + templateThreads := w.Options.Options.TemplateThreads + if templateThreads == 1 { + templateThreads++ + } + swg := sizedwaitgroup.New(templateThreads) + for _, template := range w.Workflows { swg.Add() + func(template *workflows.WorkflowTemplate) { + defer swg.Done() + if err := e.runWorkflowStep(template, ctxArgs, results, &swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } - swg.Done() }(template) } swg.Wait() @@ -126,10 +135,11 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co swg.Add() go func(subtemplate *workflows.WorkflowTemplate) { + defer swg.Done() + if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) } - swg.Done() }(subtemplate) } } diff --git a/v2/pkg/external/customtemplates/azure_blob.go b/v2/pkg/external/customtemplates/azure_blob.go index 4e405caf5..442d03746 100644 --- a/v2/pkg/external/customtemplates/azure_blob.go +++ b/v2/pkg/external/customtemplates/azure_blob.go @@ -25,7 +25,7 @@ type customTemplateAzureBlob struct { // NewAzureProviders creates a new Azure Blob Storage provider for downloading custom templates func NewAzureProviders(options *types.Options) ([]*customTemplateAzureBlob, error) { providers := []*customTemplateAzureBlob{} - if options.AzureContainerName != "" { + if options.AzureContainerName != "" && !options.AzureTemplateDisableDownload { // Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL) if err != nil { diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index e0cd75d9c..dca234383 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -17,9 +17,9 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) -var _ Provider = &customTemplateGithubRepo{} +var _ Provider = &customTemplateGitHubRepo{} -type customTemplateGithubRepo struct { +type customTemplateGitHubRepo struct { owner string reponame string gitCloneURL string @@ -27,8 +27,8 @@ type customTemplateGithubRepo struct { } // This function download the custom github template repository -func (customTemplate *customTemplateGithubRepo) Download(ctx context.Context) { - clonePath := customTemplate.getLocalRepoClonePath(config.DefaultConfig.CustomGithubTemplatesDirectory) +func (customTemplate *customTemplateGitHubRepo) Download(ctx context.Context) { + clonePath := customTemplate.getLocalRepoClonePath(config.DefaultConfig.CustomGitHubTemplatesDirectory) if !fileutil.FolderExists(clonePath) { err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken) @@ -41,8 +41,8 @@ func (customTemplate *customTemplateGithubRepo) Download(ctx context.Context) { } } -func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) { - downloadPath := config.DefaultConfig.CustomGithubTemplatesDirectory +func (customTemplate *customTemplateGitHubRepo) Update(ctx context.Context) { + downloadPath := config.DefaultConfig.CustomGitHubTemplatesDirectory clonePath := customTemplate.getLocalRepoClonePath(downloadPath) // If folder does not exits then clone/download the repo @@ -58,27 +58,31 @@ func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) { } } -// NewGithubProviders returns new instance of github providers for downloading custom templates -func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, error) { - providers := []*customTemplateGithubRepo{} +// NewGitHubProviders returns new instance of GitHub providers for downloading custom templates +func NewGitHubProviders(options *types.Options) ([]*customTemplateGitHubRepo, error) { + providers := []*customTemplateGitHubRepo{} gitHubClient := getGHClientIncognito() - for _, repoName := range options.GithubTemplateRepo { + if options.GitHubTemplateDisableDownload { + return providers, nil + } + + for _, repoName := range options.GitHubTemplateRepo { owner, repo, err := getOwnerAndRepo(repoName) if err != nil { gologger.Error().Msgf("%s", err) continue } - githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken) + githubRepo, err := getGitHubRepo(gitHubClient, owner, repo, options.GitHubToken) if err != nil { gologger.Error().Msgf("%s", err) continue } - customTemplateRepo := &customTemplateGithubRepo{ + customTemplateRepo := &customTemplateGitHubRepo{ owner: owner, reponame: repo, gitCloneURL: githubRepo.GetCloneURL(), - githubToken: options.GithubToken, + githubToken: options.GitHubToken, } providers = append(providers, customTemplateRepo) } @@ -86,8 +90,8 @@ func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, er } // getOwnerAndRepo returns the owner, repo, err from the given string -// eg. it takes input projectdiscovery/nuclei-templates and -// returns owner=> projectdiscovery , repo => nuclei-templates +// e.g., it takes input projectdiscovery/nuclei-templates and +// returns owner => projectdiscovery, repo => nuclei-templates func getOwnerAndRepo(reponame string) (owner string, repo string, err error) { s := strings.Split(reponame, "/") if len(s) != 2 { @@ -100,7 +104,7 @@ func getOwnerAndRepo(reponame string) (owner string, repo string, err error) { } // returns *github.Repository if passed github repo name -func getGithubRepo(gitHubClient *github.Client, repoOwner, repoName, githubToken string) (*github.Repository, error) { +func getGitHubRepo(gitHubClient *github.Client, repoOwner, repoName, githubToken string) (*github.Repository, error) { var retried bool getRepo: repo, _, err := gitHubClient.Repositories.Get(context.Background(), repoOwner, repoName) @@ -118,8 +122,8 @@ getRepo: return repo, nil } -// download the git repo to given path -func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) error { +// download the git repo to a given path +func (ctr *customTemplateGitHubRepo) cloneRepo(clonePath, githubToken string) error { r, err := git.PlainClone(clonePath, false, &git.CloneOptions{ URL: ctr.gitCloneURL, Auth: getAuth(ctr.owner, githubToken), @@ -127,14 +131,14 @@ func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) er if err != nil { return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error()) } - // Add the user as well in the config. By default user is not set + // Add the user as well in the config. By default, user is not set config, _ := r.Storer.Config() config.User.Name = ctr.owner return r.SetConfig(config) } // performs the git pull on given repo -func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) error { +func (ctr *customTemplateGitHubRepo) pullChanges(repoPath, githubToken string) error { r, err := git.PlainOpen(repoPath) if err != nil { return err @@ -150,21 +154,9 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e return nil } -// getLocalRepoClonePath returns the clone path. -// if same name repo directory exists from another owner then it appends the owner then and returns the path -// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep -func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string { - if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) { - return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) - } - return filepath.Join(downloadPath, ctr.reponame) -} - -// isRepoDirExists take the path and checks if the same repo or not -func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool { - r, _ := git.PlainOpen(repoPath) - local, _ := r.Config() - return local.User.Name == ctr.owner // repo already cloned no need to rename and clone +// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness +func (ctr *customTemplateGitHubRepo) getLocalRepoClonePath(downloadPath string) string { + return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) } // returns the auth object with username and github token as password diff --git a/v2/pkg/external/customtemplates/github_test.go b/v2/pkg/external/customtemplates/github_test.go index 60d469565..6c812f376 100644 --- a/v2/pkg/external/customtemplates/github_test.go +++ b/v2/pkg/external/customtemplates/github_test.go @@ -22,14 +22,14 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { config.DefaultConfig.SetTemplatesDir(templatesDirectory) options := testutils.DefaultOptions - options.GithubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"} - options.GithubToken = os.Getenv("GITHUB_TOKEN") + options.GitHubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"} + options.GitHubToken = os.Getenv("GITHUB_TOKEN") ctm, err := NewCustomTemplatesManager(options) require.Nil(t, err, "could not create custom templates manager") ctm.Download(context.Background()) - require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists") + require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists") require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists") } diff --git a/v2/pkg/external/customtemplates/gitlab.go b/v2/pkg/external/customtemplates/gitlab.go index 20e7c41d8..61dbca0a3 100644 --- a/v2/pkg/external/customtemplates/gitlab.go +++ b/v2/pkg/external/customtemplates/gitlab.go @@ -21,10 +21,10 @@ type customTemplateGitLabRepo struct { projectIDs []int } -// NewGitlabProviders returns a new list of GitLab providers for downloading custom templates -func NewGitlabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) { +// NewGitLabProviders returns a new list of GitLab providers for downloading custom templates +func NewGitLabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) { providers := []*customTemplateGitLabRepo{} - if options.GitLabToken != "" { + if options.GitLabToken != "" && !options.GitLabTemplateDisableDownload { // Establish a connection to GitLab and build a client object with which to download templates from GitLab gitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken) if err != nil { diff --git a/v2/pkg/external/customtemplates/s3.go b/v2/pkg/external/customtemplates/s3.go index 0804590d2..f5421672f 100644 --- a/v2/pkg/external/customtemplates/s3.go +++ b/v2/pkg/external/customtemplates/s3.go @@ -61,7 +61,7 @@ func (bk *customTemplateS3Bucket) Update(ctx context.Context) { // NewS3Providers returns a new instances of a s3 providers for downloading custom templates func NewS3Providers(options *types.Options) ([]*customTemplateS3Bucket, error) { providers := []*customTemplateS3Bucket{} - if options.AwsBucketName != "" { + if options.AwsBucketName != "" && !options.AwsTemplateDisableDownload { s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("error downloading s3 bucket %s", options.AwsBucketName) diff --git a/v2/pkg/external/customtemplates/templates_provider.go b/v2/pkg/external/customtemplates/templates_provider.go index 02a9794f5..6ae01e3bd 100644 --- a/v2/pkg/external/customtemplates/templates_provider.go +++ b/v2/pkg/external/customtemplates/templates_provider.go @@ -41,7 +41,7 @@ func NewCustomTemplatesManager(options *types.Options) (*CustomTemplatesManager, } // Add GitHub providers - githubProviders, err := NewGithubProviders(options) + githubProviders, err := NewGitHubProviders(options) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create github providers for custom templates") } @@ -68,7 +68,7 @@ func NewCustomTemplatesManager(options *types.Options) (*CustomTemplatesManager, } // Add GitLab providers - gitlabProviders, err := NewGitlabProviders(options) + gitlabProviders, err := NewGitLabProviders(options) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create gitlab providers for custom templates") } diff --git a/v2/pkg/input/input.go b/v2/pkg/input/input.go index 5dadb2bf1..eb379a860 100644 --- a/v2/pkg/input/input.go +++ b/v2/pkg/input/input.go @@ -18,7 +18,7 @@ type Helper struct { InputsHTTP *hybrid.HybridMap } -// NewHelper returns a new inpt helper instance +// NewHelper returns a new input helper instance func NewHelper() *Helper { helper := &Helper{} return helper diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index 93e92f966..c3cf98c43 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -47,7 +47,7 @@ type Info struct { // examples: // - value: > // []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"} - Reference stringslice.RawStringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` + Reference *stringslice.RawStringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` // description: | // Severity of the template. SeverityHolder severity.Holder `json:"severity,omitempty" yaml:"severity,omitempty"` @@ -101,6 +101,11 @@ type Classification struct { // - value: "\"0.42509\"" EPSSScore float64 `json:"epss-score,omitempty" yaml:"epss-score,omitempty" jsonschema:"title=epss score for the template,description=EPSS Score for the template,example=0.42509"` // description: | + // EPSS Percentile for the template. + // examples: + // - value: "\"0.42509\"" + EPSSPercentile float64 `json:"epss-percentile,omitempty" yaml:"epss-percentile,omitempty" jsonschema:"title=epss percentile for the template,description=EPSS Percentile for the template,example=0.42509"` + // description: | // CPE for the template. // examples: // - value: "\"cpe:/a:vendor:product:version\"" diff --git a/v2/pkg/model/model_test.go b/v2/pkg/model/model_test.go index aba1cf1c9..7a0ca188e 100644 --- a/v2/pkg/model/model_test.go +++ b/v2/pkg/model/model_test.go @@ -18,7 +18,7 @@ func TestInfoJsonMarshal(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.NewRaw("Reference1"), + Reference: stringslice.NewRawStringSlice("Reference1"), Metadata: map[string]interface{}{ "string_key": "string_value", "array_key": []string{"array_value1", "array_value2"}, @@ -42,7 +42,7 @@ func TestInfoYamlMarshal(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.NewRaw("Reference1"), + Reference: stringslice.NewRawStringSlice("Reference1"), Metadata: map[string]interface{}{ "string_key": "string_value", "array_key": []string{"array_value1", "array_value2"}, diff --git a/v2/pkg/model/types/stringslice/stringslice_raw.go b/v2/pkg/model/types/stringslice/stringslice_raw.go index 8ca38c5e8..494731ca5 100644 --- a/v2/pkg/model/types/stringslice/stringslice_raw.go +++ b/v2/pkg/model/types/stringslice/stringslice_raw.go @@ -4,10 +4,10 @@ type RawStringSlice struct { StringSlice } -func NewRaw(value interface{}) RawStringSlice { - return RawStringSlice{StringSlice: StringSlice{Value: value}} +func NewRawStringSlice(value interface{}) *RawStringSlice { + return &RawStringSlice{StringSlice: StringSlice{Value: value}} } -func (rawStringSlice RawStringSlice) Normalize(value string) string { +func (rawStringSlice *RawStringSlice) Normalize(value string) string { return value } diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index abd8bf336..aae6f3a94 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -1,13 +1,21 @@ package matchers import ( + "os" "strings" "github.com/Knetic/govaluate" + dslRepo "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + stringsutil "github.com/projectdiscovery/utils/strings" +) + +var ( + // showDSLErr controls whether to show hidden DSL errors or not + showDSLErr = strings.EqualFold(os.Getenv("SHOW_DSL_ERRORS"), "true") ) // MatchStatusCode matches a status code check against a corpus @@ -185,10 +193,8 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { if matcher.condition == ANDCondition { return false } - if strings.Contains(err.Error(), "No parameter") { + if !matcher.ignoreErr(err) { gologger.Warning().Msgf("[%s] %s", data["template-id"], err.Error()) - } else { - gologger.Error().Label("WRN").Msgf("[%s] %s", data["template-id"], err.Error()) } continue } @@ -219,3 +225,15 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { } return false } + +// ignoreErr checks if the error is to be ignored or not +// Reference: https://github.com/projectdiscovery/nuclei/issues/3950 +func (m *Matcher) ignoreErr(err error) bool { + if showDSLErr { + return false + } + if stringsutil.ContainsAny(err.Error(), "No parameter", dslRepo.ErrParsingArg.Error()) { + return true + } + return false +} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 9d2895bc7..3d3b8e847 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -144,7 +144,7 @@ type ResultEvent struct { // MatcherStatus is the status of the match MatcherStatus bool `json:"matcher-status"` // Lines is the line count for the specified match - Lines []int `json:"matched-line"` + Lines []int `json:"matched-line,omitempty"` FileToIndexPosition map[string]int `json:"-"` } @@ -187,9 +187,10 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err) } } + writer := &StandardWriter{ json: options.JSONL, - jsonReqResp: options.JSONRequests, + jsonReqResp: !options.OmitRawRequests, noMetadata: options.NoMeta, matcherStatus: options.MatcherStatus, timestamp: options.Timestamp, diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index f8d1246f2..507eb367a 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -70,7 +70,7 @@ func TestLoadTemplate(t *testing.T) { expectedErr: errors.New("field 'severity' is missing"), }, { - name: "template-without-serverity-with-correct-filter-id", + name: "template-without-severity-with-correct-filter-id", template: &templates.Template{ ID: "CVE-2021-27330", Info: model.Info{ @@ -84,7 +84,7 @@ func TestLoadTemplate(t *testing.T) { filter: filter.Config{IncludeIds: []string{"CVE-2021-27330"}}, }, { - name: "template-without-serverity-with-diff-filter-id", + name: "template-without-severity-with-diff-filter-id", template: &templates.Template{ ID: "CVE-2021-27330", Info: model.Info{ diff --git a/v2/pkg/protocols/common/automaticscan/doc.go b/v2/pkg/protocols/common/automaticscan/doc.go index b9756af76..02ff42f2e 100644 --- a/v2/pkg/protocols/common/automaticscan/doc.go +++ b/v2/pkg/protocols/common/automaticscan/doc.go @@ -14,5 +14,5 @@ // detection. // // The logic is very simple and can be further improved to increase the coverage of -// this mode of nuclei exection. +// this mode of nuclei execution. package automaticscan diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index 787f9583d..aa7dfa39b 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -2,9 +2,10 @@ package contextargs import ( "net/http/cookiejar" + "sync/atomic" "github.com/projectdiscovery/gologger" - maputils "github.com/projectdiscovery/utils/maps" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Context implements a shared context struct to share information across multiple templates within a workflow @@ -14,8 +15,9 @@ type Context struct { // CookieJar shared within workflow's http templates CookieJar *cookiejar.Jar + // Args is a workflow shared key-value store - args maputils.SyncLockMap[string, interface{}] + args *mapsutil.SyncLockMap[string, interface{}] } // Create a new contextargs instance @@ -27,45 +29,54 @@ func New() *Context { func NewWithInput(input string) *Context { jar, err := cookiejar.New(nil) if err != nil { - gologger.Error().Msgf("Could not create cookie jar: %s\n", err) + gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err) + } + return &Context{ + MetaInput: &MetaInput{Input: input}, + CookieJar: jar, + args: &mapsutil.SyncLockMap[string, interface{}]{ + Map: make(map[string]interface{}), + ReadOnly: atomic.Bool{}, + }, } - return &Context{MetaInput: &MetaInput{Input: input}, CookieJar: jar, args: maputils.SyncLockMap[string, interface{}]{ - Map: make(map[string]interface{}), - }} } // Set the specific key-value pair func (ctx *Context) Set(key string, value interface{}) { - if err := ctx.args.Set(key, value); err != nil { - gologger.Error().Msgf("contextargs: could not set key: %s\n", err) - } + _ = ctx.args.Set(key, value) +} + +func (ctx *Context) hasArgs() bool { + return !ctx.args.IsEmpty() } // Get the value with specific key if exists func (ctx *Context) Get(key string) (interface{}, bool) { + if !ctx.hasArgs() { + return nil, false + } + return ctx.args.Get(key) } -func (ctx *Context) GetAll() maputils.Map[string, interface{}] { - return ctx.args.GetAll() +func (ctx *Context) GetAll() map[string]interface{} { + if !ctx.hasArgs() { + return nil + } + + return ctx.args.Clone().Map } -func (ctx *Context) ForEach(f func(string, interface{}) error) { - if err := ctx.args.Iterate(f); err != nil { - gologger.Error().Msgf("contextargs: could not iterate: %s\n", err) - } -} - -// Merge merges the map into the contextargs -func (ctx *Context) Merge(m map[string]interface{}) { - if err := ctx.args.Merge(m); err != nil { - gologger.Error().Msgf("contextargs: could not merge: %s\n", err) - } +func (ctx *Context) ForEach(f func(string, interface{})) { + _ = ctx.args.Iterate(func(k string, v interface{}) error { + f(k, v) + return nil + }) } // Has check if the key exists func (ctx *Context) Has(key string) bool { - return ctx.args.Has(key) + return ctx.hasArgs() && ctx.args.Has(key) } func (ctx *Context) HasArgs() bool { @@ -75,7 +86,7 @@ func (ctx *Context) HasArgs() bool { func (ctx *Context) Clone() *Context { newCtx := &Context{ MetaInput: ctx.MetaInput.Clone(), - args: *ctx.args.Clone(), + args: ctx.args, CookieJar: ctx.CookieJar, } return newCtx diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index c25733a4a..4983ae1ed 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -65,12 +65,22 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { dynamicValues := make(map[string]interface{}) if input.HasArgs() { - input.ForEach(func(key string, value interface{}) error { + input.ForEach(func(key string, value interface{}) { dynamicValues[key] = value - return nil }) } previous := make(map[string]interface{}) + + var lastMatcherEvent *output.InternalWrappedEvent + writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { + if !results.Load() && matcherStatus { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + results.CompareAndSwap(false, true) + } + } + for _, req := range e.requests { inputItem := input.Clone() if e.options.InputHelper != nil && input.MetaInput.Input != "" { @@ -99,16 +109,12 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. if !event.HasOperatorResult() && !event.UsesInteractsh { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + lastMatcherEvent = event } else { if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { results.CompareAndSwap(false, true) } else { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + lastMatcherEvent = event } } }) @@ -123,6 +129,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { break } } + if lastMatcherEvent != nil { + writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) + } return results.Load(), nil } @@ -130,9 +139,8 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) if input.HasArgs() { - input.ForEach(func(key string, value interface{}) error { + input.ForEach(func(key string, value interface{}) { dynamicValues[key] = value - return nil }) } previous := make(map[string]interface{}) diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 7b8f0b595..c4d1cb61f 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -120,7 +120,7 @@ func hasLiteralsOnly(data string) bool { if err != nil { return false } - if err == nil && expr != nil { + if expr != nil { _, err = expr.Evaluate(nil) return err == nil } diff --git a/v2/pkg/protocols/common/fuzz/execute.go b/v2/pkg/protocols/common/fuzz/execute.go index a16c7107e..eca6d8d90 100644 --- a/v2/pkg/protocols/common/fuzz/execute.go +++ b/v2/pkg/protocols/common/fuzz/execute.go @@ -75,7 +75,7 @@ func (rule *Rule) isExecutable(input *contextargs.Context) bool { if err != nil { return false } - if len(parsed.Query()) > 0 && rule.partType == queryPartType { + if !parsed.Query().IsEmpty() && rule.partType == queryPartType { return true } return false diff --git a/v2/pkg/protocols/common/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go index 0576f302f..0e09aaf80 100644 --- a/v2/pkg/protocols/common/fuzz/parts.go +++ b/v2/pkg/protocols/common/fuzz/parts.go @@ -2,14 +2,15 @@ package fuzz import ( "context" - "io" "net/http" "strings" "github.com/corpix/uarand" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/retryablehttp-go" + sliceutil "github.com/projectdiscovery/utils/slice" urlutil "github.com/projectdiscovery/utils/url" ) @@ -29,32 +30,32 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) return err } origRequestURL := requestURL.Clone() - temp := urlutil.Params{} - for k, v := range origRequestURL.Query() { - // this has to be a deep copy - x := []string{} - x = append(x, v...) - temp[k] = x - } + // clone the params to avoid modifying the original + temp := origRequestURL.Params.Clone() - for key, values := range origRequestURL.Query() { + origRequestURL.Query().Iterate(func(key string, values []string) bool { + cloned := sliceutil.Clone(values) for i, value := range values { if !rule.matchKeyOrValue(key, value) { continue } var evaluated string evaluated, input.InteractURLs = rule.executeEvaluate(input, key, value, payload, input.InteractURLs) - temp[key][i] = evaluated + cloned[i] = evaluated if rule.modeType == singleModeType { + temp.Update(key, cloned) requestURL.Params = temp - if err := rule.buildQueryInput(input, requestURL, input.InteractURLs); err != nil { - return err + if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil { + err = qerr + return false } - temp[key][i] = value // change back to previous value for temp + cloned[i] = value // change back to previous value for temp } } - } + temp.Update(key, cloned) + return true + }) if rule.modeType == multipleModeType { requestURL.Params = temp @@ -62,7 +63,8 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) return err } } - return nil + + return err } // buildQueryInput returns created request for a Query Input @@ -85,7 +87,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL, DynamicValues: input.Values, } if !input.Callback(request) { - return io.EOF + return types.ErrNoMoreRequests } return nil } diff --git a/v2/pkg/protocols/common/fuzz/parts_test.go b/v2/pkg/protocols/common/fuzz/parts_test.go index e1855a3ca..a8f6e141b 100644 --- a/v2/pkg/protocols/common/fuzz/parts_test.go +++ b/v2/pkg/protocols/common/fuzz/parts_test.go @@ -32,9 +32,9 @@ func TestExecuteQueryPartRule(t *testing.T) { }, "1337'") require.NoError(t, err, "could not execute part rule") require.ElementsMatch(t, []string{ - "http://localhost:8080/?file=passwdfile&mode=multiple&url=localhost1337'", - "http://localhost:8080/?file=passwdfile&mode=multiple1337'&url=localhost", - "http://localhost:8080/?file=passwdfile1337'&mode=multiple&url=localhost", + "http://localhost:8080/?url=localhost1337'&mode=multiple&file=passwdfile", + "http://localhost:8080/?url=localhost&mode=multiple1337'&file=passwdfile", + "http://localhost:8080/?url=localhost&mode=multiple&file=passwdfile1337'", }, generatedURL, "could not get generated url") }) t.Run("multiple", func(t *testing.T) { @@ -54,7 +54,7 @@ func TestExecuteQueryPartRule(t *testing.T) { }, }, "1337'") require.NoError(t, err, "could not execute part rule") - require.Equal(t, "http://localhost:8080/?file=passwdfile1337'&mode=multiple1337'&url=localhost1337'", generatedURL, "could not get generated url") + require.Equal(t, "http://localhost:8080/?url=localhost1337'&mode=multiple1337'&file=passwdfile1337'", generatedURL, "could not get generated url") }) } diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 9cc8a9761..98fc6aa13 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -17,7 +17,7 @@ type PayloadGenerator struct { } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, sandbox bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, allowLocalFileAccess bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -43,7 +43,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, sandbox) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, allowLocalFileAccess) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index de419488e..92ec93201 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,16 +2,16 @@ package generators import ( "bufio" - "io" "path/filepath" "strings" "github.com/pkg/errors" + pkgTypes "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/spf13/cast" ) // loadPayloads loads the input payloads from a map to a data map -func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, sandbox bool) (map[string][]string, error) { +func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, allowLocalFileAccess bool) (map[string][]string, error) { loadedPayloads := make(map[string][]string) for name, payload := range payloads { @@ -22,9 +22,13 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, if len(elements) >= 2 { loadedPayloads[name] = elements } else { - if sandbox { + if !allowLocalFileAccess { pt = filepath.Clean(pt) - templatePathDir := filepath.Dir(templatePath) + templateAbsPath, err := filepath.Abs(templatePath) + if err != nil { + return nil, errors.Wrap(err, "could not get absolute path") + } + templatePathDir := filepath.Dir(templateAbsPath) if !(templatePathDir != "/" && strings.HasPrefix(pt, templatePathDir)) && !strings.HasPrefix(pt, templateDirectory) { return nil, errors.New("denied payload file path specified") } @@ -60,7 +64,7 @@ func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]stri } lines = append(lines, text) } - if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { + if err := scanner.Err(); err != nil && !errors.Is(err, pkgTypes.ErrNoMoreRequests) { return lines, scanner.Err() } return lines, nil diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index cecdde9b5..28803b097 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -25,14 +25,20 @@ func TestLoadPayloads(t *testing.T) { t.Run("templates-directory", func(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/test", tempdir, true) + }, "/test", tempdir, false) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) + t.Run("templates-path-relative", func(t *testing.T) { + _, err := generator.loadPayloads(map[string]interface{}{ + "new": "../../../../../../../../../etc/passwd", + }, ".", tempdir, false) + require.Error(t, err, "could load payloads") + }) t.Run("template-directory", func(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, filepath.Join(tempdir, "test.yaml"), "/test", true) + }, filepath.Join(tempdir, "test.yaml"), "/test", false) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) @@ -42,19 +48,19 @@ func TestLoadPayloads(t *testing.T) { } _, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", false) + }, "/random", "/test", true) require.NoError(t, err, "could load payloads") }) t.Run("invalid", func(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", true) + }, "/random", "/test", false) require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") values, err = generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/random", "/test", true) + }, "/random", "/test", false) require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") }) diff --git a/v2/pkg/protocols/common/helpers/deserialization/java.go b/v2/pkg/protocols/common/helpers/deserialization/java.go index 68dda623d..cde50d547 100644 --- a/v2/pkg/protocols/common/helpers/deserialization/java.go +++ b/v2/pkg/protocols/common/helpers/deserialization/java.go @@ -75,7 +75,7 @@ func urlsafeBase64Encode(data []byte) string { } // generateCommonsCollections40Payload generates org.apache.commons:commons-collections4:4.0 -// deserialization paylaod for a command. +// deserialization payload for a command. func generateCommonsCollections40Payload(cmd string) []byte { buffer := &bytes.Buffer{} @@ -90,7 +90,7 @@ func generateCommonsCollections40Payload(cmd string) []byte { } // generateCommonsCollections440PPayload generates commons-collections 3.1 -// deserialization paylaod for a command. +// deserialization payload for a command. func generateCommonsCollections31Payload(cmd string) []byte { buffer := &bytes.Buffer{} @@ -105,7 +105,7 @@ func generateCommonsCollections31Payload(cmd string) []byte { } // generateGroovy1Payload generates org.codehaus.groovy:groovy:2.3.9 -// deserialization paylaod for a command. +// deserialization payload for a command. func generateGroovy1Payload(cmd string) []byte { buffer := &bytes.Buffer{} @@ -119,7 +119,7 @@ func generateGroovy1Payload(cmd string) []byte { return buffer.Bytes() } -// generateDNSPayload generates DNS interaction deserialization paylaod for a DNS Name. +// generateDNSPayload generates DNS interaction deserialization payload for a DNS Name. // Taken from ysoserial DNS gadget. func generateDNSPayload(URL string) []byte { parsed, err := url.Parse(URL) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 3c006f661..6539915a7 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -128,7 +128,7 @@ func (c *Client) poll() error { return nil } -// requestShouldStopAtFirstmatch checks if furthur interactions should be stopped +// requestShouldStopAtFirstmatch checks if further interactions should be stopped // note: extra care should be taken while using this function since internalEvent is // synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that // we could use `TryLock()` but that may over complicate things and need to differentiate diff --git a/v2/pkg/protocols/common/interactsh/options.go b/v2/pkg/protocols/common/interactsh/options.go index 22a745d37..48d536803 100644 --- a/v2/pkg/protocols/common/interactsh/options.go +++ b/v2/pkg/protocols/common/interactsh/options.go @@ -43,7 +43,7 @@ type Options struct { DisableHttpFallback bool // NoInteractsh disables the engine NoInteractsh bool - // NoColor dissbles printing colors for matches + // NoColor disables printing colors for matches NoColor bool StopAtFirstMatch bool diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index b4662aeda..829c2ce34 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -91,12 +91,12 @@ func Init(options *types.Options) error { if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } - if options.Sandbox { + if options.RestrictLocalNetworkAccess { opts.Deny = append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...) } opts.WithDialerHistory = true - opts.WithZTLS = options.ZTLS opts.SNIName = options.SNI + // fastdialer now by default fallbacks to ztls when there are tls related errors dialer, err := fastdialer.NewDialer(opts) if err != nil { return errors.Wrap(err, "could not create dialer") @@ -106,14 +106,14 @@ func Init(options *types.Options) error { } // isIpAssociatedWithInterface checks if the given IP is associated with the given interface. -func isIpAssociatedWithInterface(souceIP, interfaceName string) (bool, error) { +func isIpAssociatedWithInterface(sourceIP, interfaceName string) (bool, error) { addrs, err := interfaceAddresses(interfaceName) if err != nil { return false, err } for _, addr := range addrs { if ipnet, ok := addr.(*net.IPNet); ok { - if ipnet.IP.String() == souceIP { + if ipnet.IP.String() == sourceIP { return true, nil } } diff --git a/v2/pkg/protocols/common/randomip/randomip_test.go b/v2/pkg/protocols/common/randomip/randomip_test.go index 2fcda223d..9cc9eb7b3 100644 --- a/v2/pkg/protocols/common/randomip/randomip_test.go +++ b/v2/pkg/protocols/common/randomip/randomip_test.go @@ -45,7 +45,7 @@ func TestGetRandomIp(t *testing.T) { valid: true, }, { - name: "Negativ CIDR length", + name: "Negative CIDR length", cidr: []string{"10.11.12.13/-1"}, valid: false, errorMsg: "10.11.12.13/-1 is not a valid CIDR", diff --git a/v2/pkg/protocols/common/uncover/uncover.go b/v2/pkg/protocols/common/uncover/uncover.go index 7ba2fa905..9a30590b1 100644 --- a/v2/pkg/protocols/common/uncover/uncover.go +++ b/v2/pkg/protocols/common/uncover/uncover.go @@ -53,7 +53,7 @@ func GetTargetsFromUncover(ctx context.Context, outputFormat string, opts *uncov return outputChan, nil } -// processUncoverOutput returns output strign depending on uncover field +// processUncoverOutput returns output string depending on uncover field func processUncoverOutput(result sources.Result, outputFormat string) string { if (result.IP == "" || result.Port == 0) && stringsutil.ContainsAny(outputFormat, "ip", "port") { // if ip or port is not present, fallback to using host diff --git a/v2/pkg/protocols/common/updatecheck/client.go b/v2/pkg/protocols/common/updatecheck/client.go deleted file mode 100644 index a921034f4..000000000 --- a/v2/pkg/protocols/common/updatecheck/client.go +++ /dev/null @@ -1,83 +0,0 @@ -package updatecheck - -import ( - "context" - "io" - "net/http" - "net/url" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" - "github.com/projectdiscovery/retryablehttp-go" -) - -const ( - RegisterServer = "https://version-check.nuclei.sh/" - VersionsCall = "versions" - IgnoreCall = "ignore" -) - -var nucleiVersion string - -// LatestVersion is the latest version info for nuclei and templates repos -type LatestVersion struct { - Nuclei string - Templates string - IgnoreHash string -} - -func InitNucleiVersion(version string) { - nucleiVersion = version -} - -// GetLatestNucleiTemplatesVersion returns the latest version info for nuclei and templates repos -func GetLatestNucleiTemplatesVersion() (*LatestVersion, error) { - body, err := callRegisterServer(VersionsCall) - if err != nil { - return nil, err - } - defer body.Close() - - data := make(map[string]string) - if err := jsoniter.NewDecoder(body).Decode(&data); err != nil { - return nil, err - } - return &LatestVersion{Nuclei: data["nuclei"], Templates: data["templates"], IgnoreHash: data["ignore-hash"]}, nil -} - -// GetLatestIgnoreFile returns the latest version of nuclei ignore -func GetLatestIgnoreFile() ([]byte, error) { - body, err := callRegisterServer(IgnoreCall) - if err != nil { - return nil, err - } - defer body.Close() - - data, err := io.ReadAll(body) - if err != nil { - return nil, err - } - return data, nil -} - -// callRegisterServer makes a request to RegisterServer with a call. -func callRegisterServer(call string) (io.ReadCloser, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, RegisterServer+call, nil) - if err != nil { - return nil, errors.Wrap(err, "could not make request") - } - if nucleiVersion != "" { - query := make(url.Values, 1) - query.Set("v", nucleiVersion) - req.URL.RawQuery = query.Encode() - } - resp, err := retryablehttp.DefaultClient().Do(req) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - return resp.Body, nil -} diff --git a/v2/pkg/protocols/common/variables/doc.go b/v2/pkg/protocols/common/variables/doc.go index 74e7d9c31..b0b54a87e 100644 --- a/v2/pkg/protocols/common/variables/doc.go +++ b/v2/pkg/protocols/common/variables/doc.go @@ -22,6 +22,6 @@ package variables // Non-Linear Sources (need to re-evaluate) // 1. VariablesMap // 2. PayloadsMap -// Everytime Linear Sources are updated , Non-Linear Sources need to be re-evaluated +// Every time Linear Sources are updated , Non-Linear Sources need to be re-evaluated // Constants (no need to re-evaluate, should contain only scalars) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index dad2ef750..1cc8302e5 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -172,7 +172,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 85e7af7b0..de43211d5 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -8,6 +8,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/launcher/flags" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -75,6 +76,11 @@ func New(options *types.Options) (*Browser, error) { if types.ProxyURL != "" { chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } + + for k, v := range options.ParseHeadlessOptionalArguments() { + chromeLauncher.Set(flags.Flag(k), v) + } + launcherURL, err := chromeLauncher.Launch() if err != nil { return nil, err diff --git a/v2/pkg/protocols/headless/engine/instance.go b/v2/pkg/protocols/headless/engine/instance.go index 90ffccac2..f0dae7828 100644 --- a/v2/pkg/protocols/headless/engine/instance.go +++ b/v2/pkg/protocols/headless/engine/instance.go @@ -17,6 +17,7 @@ type Instance struct { // redundant due to dependency cycle interactsh *interactsh.Client + requestLog map[string]string // contains actual request that was sent } // NewInstance creates a new instance for the current browser. @@ -35,7 +36,14 @@ func (b *Browser) NewInstance() (*Instance, error) { // We use a custom sleeper that sleeps from 100ms to 500 ms waiting // for an interaction. Used throughout rod for clicking, etc. browser = browser.Sleeper(func() utils.Sleeper { return maxBackoffSleeper(10) }) - return &Instance{browser: b, engine: browser}, nil + return &Instance{browser: b, engine: browser, requestLog: map[string]string{}}, nil +} + +// returns a map of [template-defined-urls] -> [actual-request-sent] +// Note: this does not include CORS or other requests while rendering that were not explicitly +// specified in template +func (i *Instance) GetRequestLog() map[string]string { + return i.requestLog } // Close closes all the tabs and pages for a browser instance diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index e23d2eb25..a566ab37a 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -134,7 +134,7 @@ func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads m } } - data, err := createdPage.ExecuteActions(input, actions) + data, err := createdPage.ExecuteActions(input, actions, payloads) if err != nil { return nil, nil, err } diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index b96d4f264..93cd5018c 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -2,11 +2,8 @@ package engine import ( "context" - "net" - "net/url" "os" "path/filepath" - "regexp" "strconv" "strings" "sync" @@ -19,17 +16,21 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" "github.com/segmentio/ksuid" ) var ( errinvalidArguments = errors.New("invalid arguments provided") - reUrlWithPort = regexp.MustCompile(`{{BaseURL}}:(\d+)`) ) const ( @@ -39,17 +40,13 @@ const ( ) // ExecuteActions executes a list of actions on a page. -func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (map[string]string, error) { - baseURL, err := url.Parse(input.MetaInput.Input) - if err != nil { - return nil, err - } - +func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (map[string]string, error) { outData := make(map[string]string) + var err error for _, act := range actions { switch act.ActionType.ActionType { case ActionNavigate: - err = p.NavigateURL(act, outData, baseURL) + err = p.NavigateURL(act, outData, variables) case ActionScript: err = p.RunScript(act, outData) case ActionClick: @@ -237,25 +234,57 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { } // NavigateURL executes an ActionLoadURL actions loading a URL for the page. -func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error { - URL := p.getActionArgWithDefaultValues(action, "url") - if URL == "" { +func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[string]interface{}) error { + // input <- is input url from cli + // target <- is the url from template (ex: {{BaseURL}}/test) + input, err := urlutil.Parse(p.input.MetaInput.Input) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not parse url %s", p.input.MetaInput.Input) + } + target := p.getActionArgWithDefaultValues(action, "url") + if target == "" { return errinvalidArguments } - // Handle the dynamic value substitution here. - URL, parsed = baseURLWithTemplatePrefs(URL, parsed) - if strings.HasSuffix(parsed.Path, "/") && strings.Contains(URL, "{{BaseURL}}/") { - parsed.Path = strings.TrimSuffix(parsed.Path, "/") - } - parsedString := parsed.String() - final := replaceWithValues(URL, map[string]interface{}{ - "Hostname": parsed.Hostname(), - "BaseURL": parsedString, - }) + // if target contains port ex: {{BaseURL}}:8080 use port specified in input + input, target = httputil.UpdateURLPortFromPayload(input, target) + hasTrailingSlash := httputil.HasTrailingSlash(target) - if err := p.page.Navigate(final); err != nil { - return errors.Wrap(err, "could not navigate") + // create vars from input url + defaultReqVars := protocolutils.GenerateVariables(input, hasTrailingSlash, contextargs.GenerateVariables(p.input)) + // merge all variables + // Note: ideally we should evaluate all available variables with reqvars + // but due to cyclic dependency between packages `engine` and `protocols` + // allvars are evaluated,merged and passed from headless package itself + // TODO: remove cyclic dependency between packages `engine` and `protocols` + allvars = generators.MergeMaps(allvars, defaultReqVars) + + if vardump.EnableVarDump { + gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) + } + + // Evaluate the target url with all variables + target, err = expressions.Evaluate(target, allvars) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not evaluate url %s", target) + } + + reqURL, err := urlutil.ParseURL(target, true) + if err != nil { + return errorutil.NewWithTag("http", "failed to parse url %v while creating http request", target) + } + + // ===== parameter automerge ===== + // while merging parameters first preference is given to target params + finalparams := input.Params.Clone() + finalparams.Merge(reqURL.Params.Encode()) + reqURL.Params = finalparams + + // log all navigated requests + p.instance.requestLog[action.GetArg("url")] = reqURL.String() + + if err := p.page.Navigate(reqURL.String()); err != nil { + return errorutil.NewWithErr(err).Msgf("could not navigate to url %s", reqURL.String()) } return nil } @@ -609,23 +638,6 @@ func selectorBy(selector string) rod.SelectorType { } } -// baseURLWithTemplatePrefs returns the url for BaseURL keeping -// the template port and path preference over the user provided one. -func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { - // template port preference over input URL port if template has a port - matches := reUrlWithPort.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return data, parsed - } - port := matches[0][1] - parsed.Host = net.JoinHostPort(parsed.Hostname(), port) - data = strings.ReplaceAll(data, ":"+port, "") - if parsed.Path == "" { - parsed.Path = "/" - } - return data, parsed -} - func (p *Page) getActionArg(action *Action, arg string) string { return p.getActionArgWithValues(action, arg, nil) } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 06203d3ed..0d1d09e5e 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -106,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { if len(request.Payloads) > 0 { var err error - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 5d6b33c82..049cfb5a5 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -1,7 +1,7 @@ package headless import ( - "io" + "fmt" "net/url" "strings" "time" @@ -22,6 +22,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v2/pkg/types" urlutil "github.com/projectdiscovery/utils/url" ) @@ -120,21 +121,31 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p } defer page.Close() + reqLog := instance.GetRequestLog() + navigatedURL := request.getLastNavigationURLWithLog(reqLog) // also known as matchedURL if there is a match + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil) request.options.Progress.IncrementRequests() - gologger.Verbose().Msgf("Sent Headless request to %s", input.MetaInput.Input) + gologger.Verbose().Msgf("Sent Headless request to %s", navigatedURL) reqBuilder := &strings.Builder{} if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input.MetaInput.Input) + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, navigatedURL) for _, act := range request.Steps { - actStepStr := act.String() - actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", input.MetaInput.Input) - reqBuilder.WriteString("\t" + actStepStr + "\n") + if act.ActionType.ActionType == engine.ActionNavigate { + value := act.GetArg("url") + if reqLog[value] != "" { + reqBuilder.WriteString(fmt.Sprintf("\tnavigate => %v\n", reqLog[value])) + } else { + reqBuilder.WriteString(fmt.Sprintf("%v not found in %v\n", value, reqLog)) + } + } else { + actStepStr := act.String() + reqBuilder.WriteString("\t" + actStepStr + "\n") + } } gologger.Debug().Msgf(reqBuilder.String()) - } var responseBody string @@ -210,7 +221,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads Values: payloads, BaseRequest: nil, }) - if err == io.EOF { + if err == types.ErrNoMoreRequests { return nil } if err != nil { @@ -219,3 +230,16 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads } return nil } + +// getLastNavigationURL returns last successfully navigated URL +func (request *Request) getLastNavigationURLWithLog(reqLog map[string]string) string { + for i := len(request.Steps) - 1; i >= 0; i-- { + if request.Steps[i].ActionType.ActionType == engine.ActionNavigate { + templateURL := request.Steps[i].GetArg("url") + if reqLog[templateURL] != "" { + return reqLog[templateURL] + } + } + } + return "" +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index c82c13d40..13a45f7cd 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -17,8 +17,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/utils" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" @@ -65,7 +65,7 @@ func (r *requestGenerator) Total() int { } // Make creates a http request for the provided input. -// It returns io.EOF as error when all the requests have been exhausted. +// It returns ErrNoMoreRequests as error when all the requests have been exhausted. func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { // value of `reqData` depends on the type of request specified in template // 1. If request is raw request = reqData contains raw request (i.e http request dump) @@ -100,8 +100,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, hasTrailingSlash := false if !isRawRequest { // if path contains port ex: {{BaseURL}}:8080 use port specified in reqData - parsed, reqData = utils.UpdateURLPortFromPayload(parsed, reqData) - hasTrailingSlash = utils.HasTrailingSlash(reqData) + parsed, reqData = httputil.UpdateURLPortFromPayload(parsed, reqData) + hasTrailingSlash = httputil.HasTrailingSlash(reqData) } // defaultreqvars are vars generated from request/input ex: {{baseURL}}, {{Host}} etc @@ -152,7 +152,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, } // while merging parameters first preference is given to target params finalparams := parsed.Params - finalparams.Merge(reqURL.Params) + finalparams.Merge(reqURL.Params.Encode()) reqURL.Params = finalparams return r.generateHttpRequest(ctx, reqURL, finalVars, payloads) } @@ -256,7 +256,7 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalVars, interactshURLs: r.interactshURLs}, nil } -// generateRawRequest generates Raw Request from from request data from template and variables +// generateRawRequest generates Raw Request from request data from template and variables // finalVars = contains all variables including generator and protocol specific variables // generatorValues = contains variables used in fuzzing or other generator specific values func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest string, baseURL *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) { @@ -365,13 +365,13 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st req.Body = bodyReader } if !r.request.Unsafe { - utils.SetHeader(req, "User-Agent", uarand.GetRandom()) + httputil.SetHeader(req, "User-Agent", uarand.GetRandom()) } // Only set these headers on non-raw requests if len(r.request.Raw) == 0 && !r.request.Unsafe { - utils.SetHeader(req, "Accept", "*/*") - utils.SetHeader(req, "Accept-Language", "en") + httputil.SetHeader(req, "Accept", "*/*") + httputil.SetHeader(req, "Accept-Language", "en") } if !LeaveDefaultPorts { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 99afe8c49..552d9229b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -353,7 +353,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index b2892ceac..58aae6cb8 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -53,7 +53,7 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b return nil, errorutil.NewWithTag("raw", "could not automergepath for template path %v", urlx.GetRelativePath()).Wrap(parseErr) } rawrequest.Path = cloned.GetRelativePath() - // If unsafe changes must be made in raw request string iteself + // If unsafe changes must be made in raw request string itself case unsafe: prevPath := rawrequest.Path cloned := inputURL.Clone() @@ -61,13 +61,13 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b if (cloned.Path == "" || cloned.Path == "/") && !strings.HasPrefix(prevPath, "/") { // Edgecase if raw unsafe request is // GET 1337?with=param HTTP/1.1 - if tmpurl, err := urlutil.ParseRelativePath(prevPath, true); err == nil && len(tmpurl.Params) > 0 { + if tmpurl, err := urlutil.ParseRelativePath(prevPath, true); err == nil && !tmpurl.Params.IsEmpty() { // if raw request contains parameters - cloned.Params.Merge(tmpurl.Params) + cloned.Params.Merge(tmpurl.Params.Encode()) unsafeRelativePath = strings.TrimPrefix(tmpurl.Path, "/") + "?" + cloned.Params.Encode() } else { // if raw request does not contain param - if len(cloned.Params) > 0 { + if !cloned.Params.IsEmpty() { unsafeRelativePath = prevPath + "?" + cloned.Params.Encode() } else { unsafeRelativePath = prevPath @@ -83,6 +83,7 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b } unsafeRelativePath = cloned.GetRelativePath() } + rawrequest.Path = cloned.GetRelativePath() rawrequest.UnsafeRawBytes = bytes.Replace(rawrequest.UnsafeRawBytes, []byte(prevPath), []byte(unsafeRelativePath), 1) default: diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index b004ff8f3..a44664d48 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -31,17 +31,17 @@ Host: {{Hostname}}`, parseURL(t, "https://example.com:8080/test"), false, false) Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test"), false, false) require.Nil(t, err, "could not parse GET request") // url.values are sorted to avoid randomness of using maps - require.Equal(t, "https://example.com:8080/test?password=test&username=test", request.FullURL, "Could not parse request url correctly") + require.Equal(t, "https://example.com:8080/test?username=test&password=test", request.FullURL, "Could not parse request url correctly") request, err = Parse(`GET ?username=test&password=test HTTP/1.1 Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false) require.Nil(t, err, "could not parse GET request") - require.Equal(t, "https://example.com:8080/test/?password=test&username=test", request.FullURL, "Could not parse request url correctly") + require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly") request, err = Parse(`GET /?username=test&password=test HTTP/1.1 Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false) require.Nil(t, err, "could not parse GET request") - require.Equal(t, "https://example.com:8080/test/?password=test&username=test", request.FullURL, "Could not parse request url correctly") + require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly") }) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index fec270823..7e5f40040 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -134,7 +134,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV ctx := request.newContext(input) generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues) if err != nil { - if err == io.EOF { + if err == types.ErrNoMoreRequests { break } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) @@ -301,7 +301,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous Values: generated.dynamicValues, BaseRequest: generated.request, }) - if err == io.EOF { + if err == types.ErrNoMoreRequests { return nil } if err != nil { @@ -354,7 +354,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) if err != nil { - if err == io.EOF { + if err == types.ErrNoMoreRequests { return true, nil } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) @@ -535,7 +535,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes options.ForceReadAllBody = request.ForceReadAllBody options.SNI = request.options.Options.SNI - resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) + inputUrl := input.MetaInput.Input + if url, err := urlutil.ParseURL(inputUrl, false); err == nil { + inputUrl = fmt.Sprintf("%s://%s", url.Scheme, url.Host) + } + formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path) + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) } else { hostname = generatedRequest.request.URL.Host formedURL = generatedRequest.request.URL.String() @@ -636,7 +641,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil && !request.Race { bodyBytes, _ := generatedRequest.request.BodyBytes() resp.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes)) - command, err := http2curl.GetCurlCommand(resp.Request) + command, err := http2curl.GetCurlCommand(generatedRequest.request.Request) if err == nil && command != nil { curlCommand = command.String() } diff --git a/v2/pkg/protocols/http/request_annotations.go b/v2/pkg/protocols/http/request_annotations.go index 5f0e26777..6555957c8 100644 --- a/v2/pkg/protocols/http/request_annotations.go +++ b/v2/pkg/protocols/http/request_annotations.go @@ -19,9 +19,9 @@ var ( // @tls-sni:target overrides the input target with the annotated one // special values: // request.host: takes the value from the host header - // target: overiddes with the specific value + // target: overrides with the specific value reSniAnnotation = regexp.MustCompile(`(?m)^@tls-sni:\s*(.+)\s*$`) - // @timeout:duration overrides the input timout with a custom duration + // @timeout:duration overrides the input timeout with a custom duration reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`) // @once sets the request to be executed only once for a specific URL reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`) @@ -36,15 +36,15 @@ const ( // parseFlowAnnotations and override requests flow func parseFlowAnnotations(rawRequest string) (flowMark, bool) { var fm flowMark - // parse request for known ovverride annotations - var hasFlowOveride bool + // parse request for known override annotations + var hasFlowOverride bool // @once if reOnceAnnotation.MatchString(rawRequest) { fm = Once - hasFlowOveride = true + hasFlowOverride = true } - return fm, hasFlowOveride + return fm, hasFlowOverride } type annotationOverrides struct { @@ -55,7 +55,7 @@ type annotationOverrides struct { // parseAnnotations and override requests settings func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) { - // parse request for known ovverride annotations + // parse request for known override annotations // @Host:target if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 86292a16c..c344538f5 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 97003c9ab..744f4fd6d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -29,6 +29,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" errorutil "github.com/projectdiscovery/utils/errors" + mapsutil "github.com/projectdiscovery/utils/maps" ) var _ protocols.Request = &Request{} @@ -59,10 +60,19 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, variablesMap := request.options.Variables.Evaluate(variables) variables = generators.MergeMaps(variablesMap, variables, request.options.Constants) + visitedAddresses := make(mapsutil.Map[string, struct{}]) + for _, kv := range request.addresses { actualAddress := replacer.Replace(kv.address, variables) + if visitedAddresses.Has(actualAddress) && !request.options.Options.DisableClustering { + continue + } + visitedAddresses.Set(actualAddress, struct{}{}) + if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { + outputEvent := request.responseToDSLMap("", "", "", address, "") + callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err) continue } @@ -110,7 +120,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac conn net.Conn err error ) - if host, _, err := net.SplitHostPort(actualAddress); err == nil { hostname = host } @@ -126,7 +135,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac return errors.Wrap(err, "could not connect to server") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) + _ = conn.SetDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) var interactshURLs []string diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index ff8b70134..89c528724 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -87,7 +87,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }) require.Nil(t, err, "could not execute network request") }) - require.Nil(t, finalEvent, "could not get event output from request") + require.Nil(t, finalEvent.Results, "could not get event output from request") request.Inputs[0].Type = NetworkInputTypeHolder{NetworkInputType: hexType} request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index de1f16b75..b13c3030c 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -111,10 +111,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { case request.ScanMode == "openssl" && !openssl.IsAvailable(): // if openssl is not installed instead of failing "auto" scanmode is used request.ScanMode = "auto" - - case options.Options.ZTLS && request.ScanMode == "ctls": - // only override if scanmode in template is "ctls" since auto internally uses ztls as fallback - request.ScanMode = "ztls" } tlsxOptions := &clients.Options{ diff --git a/v2/pkg/protocols/http/utils/requtils.go b/v2/pkg/protocols/utils/http/requtils.go similarity index 89% rename from v2/pkg/protocols/http/utils/requtils.go rename to v2/pkg/protocols/utils/http/requtils.go index cfb75c154..4f2f860e7 100644 --- a/v2/pkg/protocols/http/utils/requtils.go +++ b/v2/pkg/protocols/utils/http/requtils.go @@ -1,4 +1,4 @@ -package utils +package httputil import ( "regexp" @@ -11,8 +11,9 @@ import ( var ( // TODO: adapt regex for cases where port is updated urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`) - // regex to detect traling slash in path (not applicable to raw requests) + // regex to detect trailing slash in path (not applicable to raw requests) trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`) + // ErrNoMoreRequests is internal error to ) // HasTrailingSlash returns true if path(that has default variables) has trailing slash diff --git a/v2/pkg/protocols/http/utils/requtils_test.go b/v2/pkg/protocols/utils/http/requtils_test.go similarity index 99% rename from v2/pkg/protocols/http/utils/requtils_test.go rename to v2/pkg/protocols/utils/http/requtils_test.go index 635d62855..056b13b81 100644 --- a/v2/pkg/protocols/http/utils/requtils_test.go +++ b/v2/pkg/protocols/utils/http/requtils_test.go @@ -1,4 +1,4 @@ -package utils +package httputil import ( "testing" diff --git a/v2/pkg/protocols/utils/variables.go b/v2/pkg/protocols/utils/variables.go index a94c99c99..92e562ce1 100644 --- a/v2/pkg/protocols/utils/variables.go +++ b/v2/pkg/protocols/utils/variables.go @@ -115,7 +115,7 @@ func GenerateVariables(input interface{}, removeTrailingSlash bool, additionalVa func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[string]interface{} { parsed := inputURL.Clone() - parsed.Params = make(urlutil.Params) + parsed.Params = urlutil.NewOrderedParams() port := parsed.Port() if port == "" { if parsed.Scheme == "https" { diff --git a/v2/pkg/protocols/utils/variables_test.go b/v2/pkg/protocols/utils/variables_test.go index a603137f3..5c4e25efa 100644 --- a/v2/pkg/protocols/utils/variables_test.go +++ b/v2/pkg/protocols/utils/variables_test.go @@ -12,7 +12,7 @@ import ( func TestHTTPVariables(t *testing.T) { baseURL := "http://localhost:9001/test/123" parsed, _ := urlutil.Parse(baseURL) - // trailingslash is only true when both target/inputURL and payload {{BaseURL}}/xyz both have slash + // trailing slash is only true when both target/inputURL and payload {{BaseURL}}/xyz both have slash values := GenerateVariables(parsed, false, nil) require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl") diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index e7438bbc9..a2d811eef 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -64,7 +64,7 @@ type Request struct { // Payloads support both key-values combinations where a list // of payloads is provided, or optionally a single file can also // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the websocket request,description=Payloads contains any payloads for the current request"` generator *generators.PayloadGenerator @@ -106,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go b/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go index 973ec13d4..e2e16b289 100644 --- a/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go +++ b/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go @@ -17,7 +17,8 @@ type Exporter struct { // Options contains the configuration options for JSON exporter client type Options struct { // File is the file to export found JSON result to - File string `yaml:"file"` + File string `yaml:"file"` + IncludeRawPayload bool `yaml:"include-raw-payload"` } // New creates a new JSON exporter integration client based on options. @@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { exporter.mutex.Lock() defer exporter.mutex.Unlock() + // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid + // writing them to the list of events. + // This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since + // the property is set to "omitempty" + if !exporter.options.IncludeRawPayload { + event.Request = "" + event.Response = "" + } + // Add the event to the rows exporter.rows = append(exporter.rows, *event) diff --git a/v2/pkg/reporting/exporters/jsonl/jsonl.go b/v2/pkg/reporting/exporters/jsonl/jsonl.go index 76750aaf0..07e5b5f8f 100644 --- a/v2/pkg/reporting/exporters/jsonl/jsonl.go +++ b/v2/pkg/reporting/exporters/jsonl/jsonl.go @@ -17,7 +17,8 @@ type Exporter struct { // Options contains the configuration options for JSONL exporter client type Options struct { // File is the file to export found JSONL result to - File string `yaml:"file"` + File string `yaml:"file"` + IncludeRawPayload bool `yaml:"include-raw-payload"` } // New creates a new JSONL exporter integration client based on options. @@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { exporter.mutex.Lock() defer exporter.mutex.Unlock() + // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid + // writing them to the list of events. + // This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since + // the property is set to "omitempty" + if !exporter.options.IncludeRawPayload { + event.Request = "" + event.Response = "" + } + // Add the event to the rows exporter.rows = append(exporter.rows, *event) diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 9924dcabb..e3458f767 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -6,9 +6,12 @@ import ( "path/filepath" "strings" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -23,7 +26,9 @@ type Exporter struct { // Options contains the configuration options for GitHub issue tracker client type Options struct { // Directory is the directory to export found results to - Directory string `yaml:"directory"` + Directory string `yaml:"directory"` + IncludeRawPayload bool `yaml:"include-raw-payload"` + SortMode string `yaml:"sort-mode"` } // New creates a new markdown exporter integration client based on options. @@ -51,6 +56,15 @@ func New(options *Options) (*Exporter, error) { // Export exports a passed result event to markdown func (exporter *Exporter) Export(event *output.ResultEvent) error { + // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid + // writing them to the list of events. + // This will reduce the amount of storage as well as the fields being excluded from the markdown report output since + // the property is set to "omitempty" + if !exporter.options.IncludeRawPayload { + event.Request = "" + event.Response = "" + } + // index file generation file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644) if err != nil { @@ -59,7 +73,35 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { defer file.Close() filename := createFileName(event) - host := util.CreateLink(event.Host, filename) + + // If the sort mode is set to severity, host, or template, then we need to get a safe version of the name for a + // subdirectory to store the file in. + // This will allow us to sort the files into subdirectories based on the sort mode. The subdirectory will need to + // be created if it does not exist. + fileUrl := filename + subdirectory := "" + switch exporter.options.SortMode { + case "severity": + subdirectory = event.Info.SeverityHolder.Severity.String() + case "host": + subdirectory = event.Host + case "template": + subdirectory = event.TemplateID + } + if subdirectory != "" { + // Sanitize the subdirectory name to remove any characters that are not allowed in a directory name + subdirectory = sanitizeFilename(subdirectory) + + // Prepend the subdirectory name to the filename for the fileUrl + fileUrl = filepath.Join(subdirectory, filename) + + // Create the subdirectory if it does not exist + if err = fileutil.CreateFolders(filepath.Join(exporter.directory, subdirectory)); err != nil { + gologger.Warning().Msgf("Could not create subdirectory for markdown report: %s", err) + } + } + + host := util.CreateLink(event.Host, fileUrl) finding := event.TemplateID + " " + event.MatcherName severity := event.Info.SeverityHolder.Severity.String() @@ -75,7 +117,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{})) data := dataBuilder.Bytes() - return os.WriteFile(filepath.Join(exporter.directory, filename), data, 0644) + return os.WriteFile(filepath.Join(exporter.directory, subdirectory, filename), data, 0644) } func createFileName(event *output.ResultEvent) string { diff --git a/v2/pkg/reporting/format/format_utils.go b/v2/pkg/reporting/format/format_utils.go index c02f00622..c9af7b119 100644 --- a/v2/pkg/reporting/format/format_utils.go +++ b/v2/pkg/reporting/format/format_utils.go @@ -113,7 +113,7 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte } reference := event.Info.Reference - if !reference.IsEmpty() { + if reference != nil && !reference.IsEmpty() { builder.WriteString("\nReferences: \n") referenceSlice := reference.ToSlice() diff --git a/v2/pkg/reporting/format/format_utils_test.go b/v2/pkg/reporting/format/format_utils_test.go index 2a950b50b..97f258aa9 100644 --- a/v2/pkg/reporting/format/format_utils_test.go +++ b/v2/pkg/reporting/format/format_utils_test.go @@ -1,10 +1,11 @@ package format import ( - "github.com/stretchr/testify/assert" "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" @@ -18,7 +19,7 @@ func TestToMarkdownTableString(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.NewRaw("reference1"), + Reference: stringslice.NewRawStringSlice("reference1"), Metadata: map[string]interface{}{ "customDynamicKey1": "customDynamicValue1", "customDynamicKey2": "customDynamicValue2", diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 301d66a04..d1c8caf10 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -29,7 +29,7 @@ import ( // to the first individual request is compared for equality. // The equality check is performed as described below - // -// Cases where clustering is not perfomed (request is considered different) +// Cases where clustering is not performed (request is considered different) // - If request contains payloads,raw,body,unsafe,req-condition,name attributes // - If request methods,max-redirects,cookie-reuse,redirects are not equal // - If request paths aren't identical. diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 5ceea4f5e..954c4960c 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -44,7 +44,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo var reader io.ReadCloser if utils.IsURL(filePath) { - //todo:instead of creating a new client each time, a default one should be reused (same as the standard library) // use retryablehttp (tls verification is enabled by default in the standard library) resp, err := retryablehttp.DefaultClient().Get(filePath) if err != nil { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 6d00f2a4a..b870306ed 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -121,7 +121,7 @@ type Template struct { Variables variables.Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request"` // description: | - // Constants contains any scalar costant for the current template + // Constants contains any scalar constant for the current template Constants map[string]interface{} `yaml:"constants,omitempty" json:"constants,omitempty" jsonschema:"title=constant for the template,description=constants contains any constant for the template"` // TotalRequests is the total number of requests for the template. diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index d3c653df8..c44b8af4c 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -141,8 +141,8 @@ func init() { TemplateDoc.Fields[16].Name = "constants" TemplateDoc.Fields[16].Type = "map[string]interface{}" TemplateDoc.Fields[16].Note = "" - TemplateDoc.Fields[16].Description = "Constants contains any scalar costant for the current template" - TemplateDoc.Fields[16].Comments[encoder.LineComment] = "Constants contains any scalar costant for the current template" + TemplateDoc.Fields[16].Description = "Constants contains any scalar constant for the current template" + TemplateDoc.Fields[16].Comments[encoder.LineComment] = "Constants contains any scalar constant for the current template" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -298,7 +298,7 @@ func init() { FieldName: "classification", }, } - MODELClassificationDoc.Fields = make([]encoder.Doc, 6) + MODELClassificationDoc.Fields = make([]encoder.Doc, 7) MODELClassificationDoc.Fields[0].Name = "cve-id" MODELClassificationDoc.Fields[0].Type = "stringslice.StringSlice" MODELClassificationDoc.Fields[0].Note = "" @@ -334,13 +334,20 @@ func init() { MODELClassificationDoc.Fields[4].Comments[encoder.LineComment] = "EPSS Score for the template." MODELClassificationDoc.Fields[4].AddExample("", "0.42509") - MODELClassificationDoc.Fields[5].Name = "cpe" - MODELClassificationDoc.Fields[5].Type = "string" + MODELClassificationDoc.Fields[5].Name = "epss-percentile" + MODELClassificationDoc.Fields[5].Type = "float64" MODELClassificationDoc.Fields[5].Note = "" - MODELClassificationDoc.Fields[5].Description = "CPE for the template." - MODELClassificationDoc.Fields[5].Comments[encoder.LineComment] = "CPE for the template." + MODELClassificationDoc.Fields[5].Description = "EPSS Percentile for the template." + MODELClassificationDoc.Fields[5].Comments[encoder.LineComment] = "EPSS Percentile for the template." - MODELClassificationDoc.Fields[5].AddExample("", "cpe:/a:vendor:product:version") + MODELClassificationDoc.Fields[5].AddExample("", "0.42509") + MODELClassificationDoc.Fields[6].Name = "cpe" + MODELClassificationDoc.Fields[6].Type = "string" + MODELClassificationDoc.Fields[6].Note = "" + MODELClassificationDoc.Fields[6].Description = "CPE for the template." + MODELClassificationDoc.Fields[6].Comments[encoder.LineComment] = "CPE for the template." + + MODELClassificationDoc.Fields[6].AddExample("", "cpe:/a:vendor:product:version") HTTPRequestDoc.Type = "http.Request" HTTPRequestDoc.Comments[encoder.LineComment] = " Request contains a http request to be made from a template" diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 496bc10b8..c2650d091 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -20,7 +20,7 @@ var ( Name: "Argument Injection in Ruby Dragonfly", Authors: stringslice.StringSlice{Value: "0xspara"}, SeverityHolder: severity.Holder{Severity: severity.High}, - Reference: stringslice.NewRaw("https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"), + Reference: stringslice.NewRawStringSlice("https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"), Tags: stringslice.StringSlice{Value: "cve,cve2021,rce,ruby"}, } exampleNormalHTTPRequest = &http.Request{ diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index b3a90c5d0..bd773a9c3 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -52,6 +52,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro cmd.Args = append(cmd.Args, "-duc") // disable auto updates cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") + cmd.Args = append(cmd.Args, "-allow-local-file-access") if debug { cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr @@ -63,7 +64,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro if debug { fmt.Println(string(data)) } - if err != nil { + if len(data) < 1 && err != nil { return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) } var parts []string @@ -76,6 +77,36 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro return parts, nil } +// RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) +func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei") + extra = append(extra, ExtraDebugArgs...) + cmd.Env = append(os.Environ(), env...) + cmd.Args = append(cmd.Args, extra...) + cmd.Args = append(cmd.Args, "-duc") // disable auto updates + cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") + cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") + cmd.Args = append(cmd.Args, "-allow-local-file-access") + cmd.Args = append(cmd.Args, "-nc") // disable color + data, err := cmd.CombinedOutput() + if debug { + fmt.Println(string(data)) + } + results := []string{} + for _, v := range strings.Split(string(data), "\n") { + line := strings.TrimSpace(v) + switch { + case strings.HasPrefix(line, "[ERR]"): + results = append(results, line) + case strings.HasPrefix(line, "[WRN]"): + results = append(results, line) + case strings.HasPrefix(line, "[FTL]"): + results = append(results, line) + } + } + return results, err +} + var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) // RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 04a6ebe75..616e0c5eb 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -37,7 +37,7 @@ var DefaultOptions = &types.Options{ NoColor: true, UpdateTemplates: false, JSONL: false, - JSONRequests: false, + OmitRawRequests: false, EnableProgressBar: false, TemplateList: false, Stdin: false, @@ -65,8 +65,8 @@ var DefaultOptions = &types.Options{ InteractionsEviction: 60, InteractionsCoolDownPeriod: 5, InteractionsPollDuration: 5, - GithubTemplateRepo: []string{}, - GithubToken: "", + GitHubTemplateRepo: []string{}, + GitHubToken: "", } // TemplateInfo contains info for a mock executed template. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 1c1cddf1a..9aef497cf 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -1,6 +1,8 @@ package types import ( + "io" + "strings" "time" "github.com/projectdiscovery/goflags" @@ -9,6 +11,11 @@ import ( fileutil "github.com/projectdiscovery/utils/file" ) +var ( + // ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate + ErrNoMoreRequests = io.EOF +) + // Options contains the configuration options for nuclei scanner. type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths @@ -87,13 +94,15 @@ type Options struct { ReportingConfig string // MarkdownExportDirectory is the directory to export reports in Markdown format MarkdownExportDirectory string + // MarkdownExportSortMode is the method to sort the markdown reports (options: severity, template, host, none) + MarkdownExportSortMode string // SarifExport is the file to export sarif output format to SarifExport string // CloudURL is the URL for the nuclei cloud endpoint CloudURL string // CloudAPIKey is the api-key for the nuclei cloud endpoint CloudAPIKey string - // Scanlist feature to get all the scan ids for a user + // ScanList feature to get all the scan ids for a user ScanList bool // ListDatasources enables listing of datasources for user ListDatasources bool @@ -189,6 +198,8 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool + // HeadlessOptionalArguments specifies optional arguments to pass to Chrome + HeadlessOptionalArguments goflags.StringSlice // NoTables disables pretty printing of cloud results in tables NoTables bool // DisableClustering disables clustering of templates @@ -231,7 +242,10 @@ type Options struct { // JSON writes json line output to files JSONL bool // JSONRequests writes requests/responses for matches in JSON output + // Deprecated: use OmitRawRequests instead as of now JSONRequests(include raw requests) is always true JSONRequests bool + // OmitRawRequests omits requests/responses for matches in JSON output + OmitRawRequests bool // JSONExport is the file to export JSON output format to JSONExport string // JSONLExport is the file to export JSONL output format to @@ -274,10 +288,12 @@ type Options struct { ClientKeyFile string // ClientCAFile client certificate authority file (PEM-encoded) used for authenticating against scanned hosts ClientCAFile string - // Use ZTLS library + // Deprecated: Use ZTLS library ZTLS bool - // Sandbox enables sandboxed nuclei template execution - Sandbox bool + // AllowLocalFileAccess allows local file access from templates payloads + AllowLocalFileAccess bool + // RestrictLocalNetworkAccess restricts local network access from templates requests + RestrictLocalNetworkAccess bool // ShowMatchLine enables display of match line number ShowMatchLine bool // EnablePprof enables exposing pprof runtime information with a webserver. @@ -310,7 +326,7 @@ type Options struct { IncludeConditions goflags.StringSlice // Custom Config Directory CustomConfigDir string - // Enable uncover egine + // Enable uncover engine Uncover bool // Uncover search query UncoverQuery goflags.StringSlice @@ -326,16 +342,22 @@ type Options struct { ScanAllIPs bool // IPVersion to scan (4,6) IPVersion goflags.StringSlice + // PublicTemplateDisableDownload disables downloading templates from the nuclei-templates public repository + PublicTemplateDisableDownload bool // GitHub token used to clone/pull from private repos for custom templates - GithubToken string - // GithubTemplateRepo is the list of custom public/private templates GitHub repos - GithubTemplateRepo []string + GitHubToken string + // GitHubTemplateRepo is the list of custom public/private templates GitHub repos + GitHubTemplateRepo []string + // GitHubTemplateDisableDownload disables downloading templates from custom GitHub repositories + GitHubTemplateDisableDownload bool // GitLabServerURL is the gitlab server to use for custom templates GitLabServerURL string // GitLabToken used to clone/pull from private repos for custom templates GitLabToken string // GitLabTemplateRepositoryIDs is the comma-separated list of custom gitlab repositories IDs GitLabTemplateRepositoryIDs []int + // GitLabTemplateDisableDownload disables downloading templates from custom GitLab repositories + GitLabTemplateDisableDownload bool // AWS access key for downloading templates from S3 bucket AwsAccessKey string // AWS secret key for downloading templates from S3 bucket @@ -344,6 +366,8 @@ type Options struct { AwsBucketName string // AWS Region name where AWS S3 bucket is located AwsRegion string + // AwsTemplateDisableDownload disables downloading templates from AWS S3 buckets + AwsTemplateDisableDownload bool // AzureContainerName for downloading templates from Azure Blob Storage. Example: templates AzureContainerName string // AzureTenantID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000 @@ -354,6 +378,8 @@ type Options struct { AzureClientSecret string // AzureServiceURL for downloading templates from Azure Blob Storage. Example: https://XXXXXXXXXX.blob.core.windows.net/ AzureServiceURL string + // AzureTemplateDisableDownload disables downloading templates from Azure Blob Storage + AzureTemplateDisableDownload bool // Scan Strategy (auto,hosts-spray,templates-spray) ScanStrategy string // Fuzzing Type overrides template level fuzzing-type configuration @@ -424,3 +450,17 @@ func (options *Options) HasCloudOptions() bool { func (options *Options) ShouldUseHostError() bool { return options.MaxHostError > 0 && !options.NoHostErrors } + +func (options *Options) ParseHeadlessOptionalArguments() map[string]string { + optionalArguments := make(map[string]string) + for _, v := range options.HeadlessOptionalArguments { + if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { + key := strings.TrimSpace(argParts[0]) + value := strings.TrimSpace(argParts[1]) + if key != "" && value != "" { + optionalArguments[key] = value + } + } + } + return optionalArguments +} diff --git a/v2/pkg/utils/http_probe.go b/v2/pkg/utils/http_probe.go new file mode 100644 index 000000000..f79567d75 --- /dev/null +++ b/v2/pkg/utils/http_probe.go @@ -0,0 +1,33 @@ +package utils + +import ( + "fmt" + "net/http" + + "github.com/corpix/uarand" + "github.com/projectdiscovery/httpx/common/httpx" +) + +var ( + HttpSchemes = []string{"https", "http"} +) + +// probeURL probes the scheme for a URL. first HTTPS is tried +// and if any errors occur http is tried. If none succeeds, probing +// is abandoned for such URLs. +func ProbeURL(input string, httpxclient *httpx.HTTPX) string { + for _, scheme := range HttpSchemes { + formedURL := fmt.Sprintf("%s://%s", scheme, input) + req, err := httpxclient.NewRequest(http.MethodHead, formedURL) + if err != nil { + continue + } + req.Header.Set("User-Agent", uarand.GetRandom()) + + if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { + continue + } + return formedURL + } + return "" +}