From 831114bdbd51a94228290e82af7a1c02c61e86aa Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 13:09:22 +0530 Subject: [PATCH 01/16] Added default fields for DNS requests --- v2/pkg/protocols/dns/dns.go | 14 ++++++++++++-- v2/pkg/protocols/dns/dns_test.go | 3 ++- v2/pkg/protocols/dns/operators_test.go | 12 ++++++++---- v2/pkg/protocols/dns/request_test.go | 3 ++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 7cde7fed4..177a497b7 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -83,7 +83,7 @@ type Request struct { // description: | // Recursion determines if resolver should recurse all records to get fresh results. - Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` + Recursion *bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` // Resolvers to use for the dns requests Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` } @@ -99,6 +99,12 @@ func (request *Request) GetID() string { // Compile compiles the protocol request for further execution. func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if request.Retries == 0 { + request.Retries = 3 + } + if request.Recursion == nil { + *request.Recursion = true + } dnsClientOptions := &dnsclientpool.Configuration{ Retries: request.Retries, } @@ -162,7 +168,7 @@ func (request *Request) Make(domain string) (*dns.Msg, error) { // Build a request on the specified URL req := new(dns.Msg) req.Id = dns.Id() - req.RecursionDesired = request.Recursion + req.RecursionDesired = *request.Recursion var q dns.Question @@ -207,6 +213,8 @@ func questionTypeToInt(questionType string) uint16 { question = dns.TypeDS case "AAAA": question = dns.TypeAAAA + default: + question = dns.TypeA } return question } @@ -229,6 +237,8 @@ func classToInt(class string) uint16 { result = dns.ClassNONE case "ANY": result = dns.ClassANY + default: + result = dns.ClassINET } return uint16(result) } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index f44ba2205..0aae3ed20 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -24,6 +24,7 @@ func TestGenerateDNSVariables(t *testing.T) { func TestDNSCompileMake(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) const templateID = "testing-dns" request := &Request{ @@ -31,7 +32,7 @@ func TestDNSCompileMake(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 5651ea422..963f6bd8d 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -20,6 +20,7 @@ import ( func TestResponseToDSLMap(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -27,7 +28,7 @@ func TestResponseToDSLMap(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -52,6 +53,7 @@ func TestResponseToDSLMap(t *testing.T) { func TestDNSOperatorMatch(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -59,7 +61,7 @@ func TestDNSOperatorMatch(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -163,6 +165,7 @@ func TestDNSOperatorMatch(t *testing.T) { func TestDNSOperatorExtract(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -170,7 +173,7 @@ func TestDNSOperatorExtract(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -220,6 +223,7 @@ func TestDNSOperatorExtract(t *testing.T) { func TestDNSMakeResult(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -227,7 +231,7 @@ func TestDNSMakeResult(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 7b6a3e6ab..e425939ee 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -17,6 +17,7 @@ import ( func TestDNSExecuteWithResults(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -24,7 +25,7 @@ func TestDNSExecuteWithResults(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ From 44e304179a4a4e82593798d7c97ca25b1168c689 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 13:15:24 +0530 Subject: [PATCH 02/16] Fixed a linter error --- v2/pkg/protocols/dns/dns.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 177a497b7..c7dbd3134 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -213,8 +213,6 @@ func questionTypeToInt(questionType string) uint16 { question = dns.TypeDS case "AAAA": question = dns.TypeAAAA - default: - question = dns.TypeA } return question } @@ -237,8 +235,6 @@ func classToInt(class string) uint16 { result = dns.ClassNONE case "ANY": result = dns.ClassANY - default: - result = dns.ClassINET } return uint16(result) } From a908a1515aa3a704f5dd614cd580a50614901c02 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 13:17:19 +0530 Subject: [PATCH 03/16] Fixed example docs --- v2/pkg/templates/templates_doc_examples.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 49e58eb00..aa8a47a1f 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -36,12 +36,13 @@ var ( } _ = exampleNormalHTTPRequest + recursion = false exampleNormalDNSRequest = &dns.Request{ Name: "{{FQDN}}", RequestType: dns.DNSRequestTypeHolder{DNSRequestType: dns.CNAME}, Class: "inet", Retries: 2, - Recursion: true, + Recursion: &recursion, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, From f60e93aae1b7d578c3d0a169a9cec72de7597901 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 29 Nov 2021 16:01:06 +0530 Subject: [PATCH 04/16] add template level stop at first match option --- v2/pkg/protocols/common/executer/executer.go | 5 +++++ v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/protocols/protocols.go | 2 ++ v2/pkg/templates/compile.go | 1 + v2/pkg/templates/templates.go | 3 +++ 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index fe650e945..e2f653856 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -81,6 +81,11 @@ func (e *Executer) Execute(input string) (bool, error) { } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } + // + if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { + gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) + break + } } return results, nil } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1868f85bc..cfe6daf56 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -270,7 +270,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou request.options.Progress.IncrementRequests() // If this was a match, and we want to stop at first match, skip all further requests. - if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { + if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { break } } @@ -310,7 +310,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) - } + } } var formedURL string var hostname string diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index f288b717f..fb7f04a1b 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -61,6 +61,8 @@ type ExecuterOptions struct { Interactsh *interactsh.Client // HostErrorsCache is an optional cache for handling host errors HostErrorsCache *hosterrorscache.Cache + // Stop execution once first match is found + StopAtFirstMatch bool Operators []*operators.Operators // only used by offlinehttp module diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 2899ffc6b..079c96f4a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -69,6 +69,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplateID = template.ID options.TemplateInfo = template.Info options.TemplatePath = filePath + options.StopAtFirstMatch = template.StopAtFirstMatch // If no requests, and it is also not a workflow, return error. if template.Requests() == 0 { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index a7f272732..0eb400afe 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -74,6 +74,9 @@ type Template struct { // description: | // Self Contained marks Requests for the template as self-contained SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + // description: | + // Stop execution once first match is found + StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` From a7ecadf1892881985567d68528fd8e845fb84ee1 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 29 Nov 2021 18:06:25 +0530 Subject: [PATCH 05/16] add template level stop-at-first-match to workflows --- v2/pkg/protocols/common/executer/executer.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index e2f653856..cfefe55a4 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -81,7 +81,7 @@ func (e *Executer) Execute(input string) (bool, error) { } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } - // + // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) break @@ -94,6 +94,7 @@ func (e *Executer) Execute(input string) (bool, error) { func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) previous := make(map[string]interface{}) + var results bool for _, req := range e.requests { req := req @@ -113,6 +114,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve if event.OperatorsResult == nil { return } + results = true callback(event) }) if err != nil { @@ -123,6 +125,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } + // If a match was found and stop at first match is set, break out of the loop and return + if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { + gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) + break + } } return nil } From de236379a1d8d1937bd94c1895c35c27559f8459 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 30 Nov 2021 17:22:39 +0530 Subject: [PATCH 06/16] remove stop-at-first-match info log #1188 --- v2/pkg/protocols/common/executer/executer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index cfefe55a4..16a3ede71 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -83,7 +83,6 @@ func (e *Executer) Execute(input string) (bool, error) { } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) break } } @@ -127,7 +126,6 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) break } } From 62679f37e211b95b0b5259c5b16705529481db07 Mon Sep 17 00:00:00 2001 From: Xc1Ym Date: Thu, 2 Dec 2021 15:18:52 +0800 Subject: [PATCH 07/16] Update README_CN.md and translated new README Last year I translated README. Now I find README is updated. I translated it again --- README_CN.md | 398 +++++++++++++++++++++++++++++---------------------- 1 file changed, 224 insertions(+), 174 deletions(-) diff --git a/README_CN.md b/README_CN.md index 04138acd2..b52263563 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,253 +1,303 @@

- nuclei
+ Nuclei

-[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT) -[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei) -[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues) -[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases) -[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei) -[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei) -[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH) +

基于YAML语法模板的定制化快速漏洞扫描器

+

- + + + + + + +

+ +

+ 工作流程 • + 安装 • + 对于安全工程师 • + 对于开发者 • + 文档 • + 致谢 • + 常见问题 • + 加入Discord

+

+ English • + 中文 +

-Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具，同时还提供了强大的可扩展性和易用性。 +--- -基于模板的nuclei被用来发送请求给目标，有着实现零误报的优点，并且可以对已知的路径进行有效的扫描。nuclei的主要用于在初期的探测阶段快速地对已知的且易于检测的漏洞或者CVE进行扫描。如果存在WAF的话，nuclei使用[retryablehttp-go库](https://github.com/projectdiscovery/retryablehttp-go)来处理各种错误，并且重新尝试攻击，这也是我们自定义功能的核心模块之一。 +Nuclei使用零误报的定制模板向目标发送请求，同时可以对大量主机进行快速扫描。Nuclei提供TCP、DNS、HTTP、FILE等各类协议的扫描，通过强大且灵活的模板，可以使用Nuclei模拟各种安全检查。 -我们也维护一个具有各个类型的模板的[开源库](https://github.com/projectdiscovery/nuclei-templates)，我们希望你也能贡献一些模板，贡献的这些模板最好是有效的，并且能允许每个人基于你的模板重新构建。查看[**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/)这个网站去学习制作模板的入门知识。 +我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过200**安全研究员和工程师提供的模板。 -## 目录 -- [目录](#目录) -- [功能](#功能) -- [安装](#安装) -- [Nuclei模板](#nuclei模板) -- [用法](#用法) -- [运行Nuclei](#运行nuclei) -- [排除模板](#排除模板) -- [致谢](#致谢) -## 功能 +## 工作流程 -

- nuclei -
-

-- 有着易于开发的、简单的、模块化的代码库 -- 使用了基于模板的引擎，运行速度极快，可以修改所以配置 -- 可以对特殊情况处理、重试、绕过等，可以绕过WAF -- 智能匹配，零误报 +

+ nuclei-flow +

-## 安装 -### 二进制文件安装 +# 安装Nuclei -二进制文件安装很简单，你可以从[Releases](https://github.com/projectdiscovery/nuclei/releases/)页面下载已经构建好的二进制文件压缩包，使用解压工具提取下载的压缩包，并将解压的文件夹移动到$PATH目录，就可以直接使用了。 +Nuclei需要**go1.17**才能安装成功。执行下列命令安装最新版本的Nuclei ```sh -Download latest binary from https://github.com/projectdiscovery/nuclei/releases - -▶ tar -xzvf nuclei-linux-amd64.tar.gz -▶ mv nuclei /usr/local/bin/ -▶ nuclei -version +go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest ``` -### 源码安装 +**更多的安装方式 [请点击此处](https://nuclei.projectdiscovery.io/nuclei/get-started/).** -nuclei需要**go1.14+**才能成功安装，运行以下命令获取repo + + + + +
-```sh -▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei -``` +### Nuclei模板 -### GitHub安装 +自从[v2.5.2]((https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2))起，Nuclei就内置了自动下载和更新模板的功能。[**Nuclei模板**](https://github.com/projectdiscovery/nuclei-templates)仓库随时更新社区中可用的模板列表。 -```sh -▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version -``` +您仍然可以随时使用`update-templates`命令更新模板，您可以根据[模板指南](https://nuclei.projectdiscovery.io/templating-guide/)编写您自己的模板。 -## Nuclei模板 +YAML的语法规范在[这里](SYNTAX-REFERENCE.md)。 -你可以使用`update-templates`来下载和更新nuclei模板，该命令会从unclei的[模板库]()中下载最新版本，这个由社区来维护的库是可以随时使用的。 +
-```sh -▶ nuclei -update-templates -``` - -此外您可以根据自己的工作情况或者需求编写模板，请参阅**nuclei[模板向导](https://nuclei.projectdiscovery.io/templating-guide/)去编写自定义模板** - -## 用法 +### 用法 ```sh nuclei -h ``` -这条命令会显示帮助，以下是nuclei支持的所有命令 +这将显示Nuclei的帮助，以下是所有支持的命令 -|命令|描述|例子| -|-----|-----|-----| -|bulk-size|每个模板最大并行的主机数(默认25)|nuclei -bulk-size 25| -|burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX| -|c|并行的最大模板数量(默认10)|nuclei -c 10| -|l|对URL列表进行测试|nuclei -l urls.txt| -|target|对目标进行测试|nuclei -target hxxps://example.com -target hxxps://example2.com| -|t|要检测的模板种类|nuclei -t git-core.yaml -t cves/| -|no-color|输出不显示颜色|nuclei -no-color| -|no-meta|不显示匹配的元数据|nuclei -no-meta| -|json|输出为json格式|nuclei -json| -|include-rr|json输出格式中包含请求和响应数据|nuclei -json -include-rr| -|o|输出为文件|nuclei -o output.txt| -|project|避免发送相同的请求|nuclei -project| -|stats|使用进度条|nuclei -stats| -|silent|只输出测试成功的结果|nuclei -silent| -|retries|失败后的重试次数|nuclei -retries 1| -|timeout|超时时间(默认为5秒)|nuclei -timeout 5| -|trace-log|输出日志到log文件|nuclei -trace-log logs| -|rate-limit|每秒最大请求数(默认150)|nuclei -rate-limit 150| -|severity|根据严重性选择模板|nuclei -severity critical,high| -|stop-at-first-match|第一次匹配不要处理HTTP请求|nuclei -stop-at-frst-match| -|exclude|排除的模板或文件夹|nuclei -exclude panels -exclude tokens| -|debug|调试请求或者响应|nuclei -debug| -|update-templates|下载或者升级模板|nuclei -update-templates| -|update-directory|选择储存模板的目录(可选)|nuclei -update-directory templates| -|tl|列出可用的模板|nuclei -tl| -|templates-version|显示已安装的模板版本|nuclei -templates-version| -|v|显示发送请求的详细信息|nuclei -v| -|version|显示nuclei的版本号|nuclei -version| -|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| -|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| -|random-agent|使用随机的UA|nuclei -random-agent| -|H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| -## 运行Nuclei +```yaml +Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板的快速漏洞扫描器。 -### 运行单个模板 +用法： + nuclei [命令] -这将对`urls.txt`中所有的主机运行`git-core.yaml`并返回结果到`results.txt` +命令： +目标： + -u, -target string[] 指定扫描的URL/主机 + -l, -list string 指定需要扫描的URL/主机文件（一行一个） + +模板： + -t, -templates string[] 指定需要扫描的模板或者模板的路径 + -nt, -new-templates 只扫描最新版本中添加的模板 + -w, -workflows string[] 指定扫描中的工作流或者工作流目录 + -validate 验证通过的模板 + -tl 列出所有可用的模板 + +过滤： + -tags string[] 执行有标记的模板子集 + -etags, -exclude-tags string[] 执行标记为排除的模板 + -itags, -include-tags string[] 不执行具有攻击性的模板 + -et, -exclude-templates string[] 要排除的模板或者模板目录 + -it, -include-templates string[] 执行默认或配置中排除的模板 + -s, -severity value[] 根据严重性运行模板，允许的值有：info,low,medium,high,critical + -es, -exclude-severity value[] 根据严重性排除模板，允许的值有：info,low,medium,high,critical + -a, -author string[] 执行指定作者的模板 + +输出： + -o, -output string 输出发现的问题到文件 + -silent 只显示结果 + -nc, -no-color 禁用输出内容着色（ANSI转义码） + -json 输出为jsonL（ines） + -irr, -include-rr 在JSONL中输出对应的请求和相应（仅结果） + -nm, -no-meta 不显示匹配的元数据 + -nts, -no-timestamp 不在输出中显示时间戳 + -rdb, -report-db string 本地的Nuclei结果数据库（始终使用该数据库保存结果） + -me, -markdown-export string 以markdown导出结果 + -se, -sarif-export string 以SARIF导出结果 + +配置： + -config string 指定Nuclei的配置文件 + -rc, -report-config string 指定Nuclei报告模板文件 + -H, -header string[] 指定报告中的标题：value格式 + -V, -var value 通过var=value指定var值 + -r, -resolvers string 指定Nuclei的解析文件 + -sr, -system-resolvers 当DNS错误时使用系统DNS + -passive 启用被动扫描处理HTTP响应 + -ev, env-vars 在模板中使用环境变量 + +交互： + -inserver, -ineractsh-server string 使用interactsh反连检测平台（默认为"https://interactsh.com"） + -itoken, -interactsh-token string 指定反连检测平台的身份凭证 + -interactions-cache-size int 指定保存在交互缓存中的请求数（默认：5000） + -interactions-eviction int 聪缓存中删除请求前等待的时间（默认为60秒） + -interactions-poll-duration int 每个轮询前等待时间（默认为5秒） + -interactions-cooldown-period int 退出轮询前的等待时间（默认为5秒） + -ni, -no-interactsh 禁用反连检测平台，同时排除基于反连检测的模板 + +限速： + -r1, -rate-limit int 每秒最大请求量（默认：150） + -rlm, -rate-limit-minute int 每分钟最大请求量 + -bs, -bulk-size int 每个模板最大并行检测数（默认：25） + -c, -concurrency int 并行执行的最大模板数量（默认：25） + + +优化： + -timeout int 超时时间（默认为5秒） + -retries int 重试次数（默认：1） + -mhe, -max-host-error int 某主机扫描失败次数，跳过该主机（默认：30） + -project 使用项目文件夹避免多次发送同一请求 + -project-path string 设置特定的项目文件夹 + -spm, -stop-at-first-path 得到一个结果后停止（或许会中断模板和工作流的逻辑） + -stream 流模式 - 在不整理输入的情况下详细描述 + +无界面浏览器： + -headless 启用需要无界面浏览器的模板 + -page-timeout int 在无界面下超时秒数（默认：20） + -sb, -show-brower 在无界面浏览器运行模板时，显示浏览器 + -sc, -system-chrome 不使用Nuclei自带的浏览器，使用本地浏览器 + +调试： + -debug 显示所有请求和响应 + -debug-req 显示所有请求 + -debug-resp 显示所有响应 + -proxy, -proxy-url string 使用HTTP代理 + -proxy-socks-url string 使用SOCK5代理 + -tlog, -trace-log string 写入请求日志到文件 + -version 显示版本信息 + -v, -verbose 显示详细信息 + -vv 显示额外的详细信息 + -tv, -templates-version 显示已安装的模板版本 + +升级： + -update 更新Nuclei到最新版本 + -ut, -update-templates 更新Nuclei模板到最新版 + -ud, -update-directory string 覆盖安装模板 + -duc, -disable-update-check 禁用更新 + +统计： + -stats 显示正在扫描的统计信息 + -sj, -stats-json 将统计信息以JSONL格式输出到文件 + -si, -stats-inerval int 显示统计信息更新的间隔秒数（默认：5） + -m, -metrics 显示Nuclei端口信息 + -mp, -metrics-port int 更改Nuclei默认端口（默认：9092） +``` + +### 运行Nuclei + +使用[社区提供的模板](https://github.com/projectdiscovery/nuclei-templates)扫描单个目标 ```sh -▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt +nuclei -u https://example.com ``` -你可以轻松的通过管道使用标准的输入(STDIN)传递URL列表。 +使用[社区提供的模板](https://github.com/projectdiscovery/nuclei-templates)扫描多个目标 ```sh -▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt +nuclei -list urls.txt ``` -💡 Nuclei可以接受如下列表的URL作为输入，例如以下URL： +Example of `urls.txt`: -``` -https://test.some-site.com -http://vuls-testing.com -https://test.com -``` -### 运行多个模板 - -这将会对`urls.txt`中所有的URL运行`cves`和`files`模板检查，并返回输出到`results.txt` - -```sh -▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt +```yaml +http://example.com +http://app.example.com +http://test.example.com +http://uat.example.com ``` -### 使用subfinder运行 +**更多关于Nuclei的详细实例可以在[这里](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei)找到** -```sh -▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt -``` +# 对于安全工程师 -### 在docker中运行 +Nuclei提供了大量有助于安全工程师在工作流定制相关的功能。通过各种扫描功能（如DNS、HTTP、TCP），安全工程师可以更轻松的使用Nuclei创建一套自定义的检查方式。 -你需要使用[nuclei的docker镜像](https://hub.docker.com/r/projectdiscovery/nuclei)来运行 +- 支持多种协议：TCP、DNS、HTTP、FILE等 +- 通过工作流和动态请求实现复杂的漏洞扫描 +- 易于集成到CI/CD，旨在可以轻松的集成到周期扫描中，以主动检测漏洞的修复和重新出现 -```sh -▶ docker pull projectdiscovery/nuclei -``` +

+ Learn More +

-下载并构建完成后，运行以下命令： + + + + +
-```sh -▶ docker run -it projectdiscovery/nuclei -``` +**对于赏金猎人：** -这将会对`urls.txt`中的URL通过docker中的nuclei进行检测，并将结果输出到本机的`results.txt`文件的： +Nuclei允许您定制自己的测试方法，可以轻松的运行您的程序。此外Nuclei可以更容易的集成到您的漏扫设备中。 -```sh -▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t /app/nuclei-templates/files/git-config.yaml > results.txt -``` -记住更改的模板路径到本机 +- 可以集成到其他工作流中 +- 可以在几分钟处理上千台主机 +- 使用YAML语法定制自动化测试 -### 速率限制 +欢迎查看我们其他的开源项目，可能有适合您的赏金猎人工作流：[github.com/projectdiscovery](http://github.com/projectdiscovery)，我们还使用[Chaos绘制了每日的DNS数据](http://chaos.projectdiscovery.io)。 -Nuclei有多种控制速率的方法，包括并行执行多个模板、并行检查多个主机，以及使nuclei限制全局的请求速率，下面就是示例。 +
-- `-c`参数 => 限制并行的模板数 -- `-bulk-size`参数 => 限制并行的主机数 -- `-rate-limit`参数 => 全局速率限制 + + + + +
-如果你想快速扫描或者控制扫描，请使用这些标志并输入限制数，`速率限制`只保证控制传出的请求，与其他参数无关。 +**对于渗透测试：** -### 排除模板 +Nuclei通过增加手动、自动的过程，极大地改变了安全评估的方式。一些公司已经在用Nuclei升级他们的手动测试步骤，可以使用Nulcei对数千台主机使用同样的流程自动化测试。 -[Nuclei模板](https://github.com/projectdiscovery/nuclei-templates)包含多种检查，其中有许多对攻击有用的检查，但并不是都有用的。如果您只希望扫描少数特定的模板或目录，则可以使用如下的参数筛选模板，或将某些模板排除。 +渗透测试员可以使用公共模板或者自定义模板来更快的完成渗透测试，特别是漏洞验证时，可以轻松的验证漏洞是否修复。 -#### 排除模板运行 +- 轻松根据您的要求创建标准清单（例如：OWASP TOP 10） +- 通过[FUZZ](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing)和[工作流](https://nuclei.projectdiscovery.io/templating-guide/#workflows)等功能，可以使用Nuclei完成复杂的手动步骤和重复性渗透测试 +- 只需要重新运行Nuclei即可验证漏洞修复情况 -我们不建议同时运行所有的nuclei模板，如果要排除模板，可以使用`exclude`参数来排除特定的目录或模板。 +
-```sh -nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml -``` +# 对于开发和组织 -注意：如上述示例中显示的那样，目录和特定模板都将不会扫描 +Nuclei构建很简单，通过数百名安全研究员的社区模板，Nuclei可以随时扫描来了解安全威胁。Nuclei通常用来用于复测，以确定漏洞是否被修复。 -#### 基于严重性运行模板 +- **CI/CD：**工程师已经支持了CI/CD，可以使用Nuclei来监控生产环境 +- **周期性扫描：**使用Nuclei创建新发现的漏洞模板，通过Nuclei可以周期性扫描消除漏洞 -您可以根据模板的严重性运行模板，扫描时可以选择单个严重性或多个严重性。 +我们有个[讨论组](https://github.com/projectdiscovery/nuclei-templates/discussions/693)，黑客提交自己的模板后可以获得赏金，这可以减少资产的漏洞，并且减少重复。如果你想实行该计划，可以[联系我](mailto:contact@projectdiscovery.io)。我们非常乐意提供帮助，或者在[讨论组](https://github.com/projectdiscovery/nuclei-templates/discussions/693)中发布相关信息。 -```sh -nuclei -l urls.txt -t cves/ -severity critical,medium -``` +

+ regression-cycle-with-nuclei +

-上面的例子将运行`cves`目录下所有`严重`和`中等`的模板。 +

+ Learn More +

-```sh -nuclei -l urls.txt -t panels/ -t technologies -severity info -``` +### 资源 +- [使用Nuclei扫描](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/) +- [Nuclei Unleashed - 快速编写复杂漏洞](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/) +- [Nuclei - FUZZ一切](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/) +- [Nuclei + Interactsh Integration，用于自动化OOB测试](https://blog.projectdiscovery.io/nuclei-interactsh-integration/) +- [武器化Nuclei](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) 作者：[@dwisiswant0](https://github.com/dwisiswant0) +- [如何使用Nuclei连续扫描？](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) 作者：[@dwisiswant0](https://github.com/dwisiswant0) +- [自动化攻击](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) 作者：[@DhiyaneshGeek](https://github.com/DhiyaneshGeek) -上面的例子将运行`panels`和`technologies`目录下严重性标记为`info`的模板 +### 致谢 -#### 使用`.nuclei-ignore`文件排除模板 +感谢所有[社区贡献者提供的PR](https://github.com/projectdiscovery/nuclei/graphs/contributors)，另外您可以其他类似的开源项目： -自从nuclei的[v2.1.1版本](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1)以来，我们添加了对`.nuclei-ignore`文件的支持，该文件与`update-templates`参数一起使用，在 **.nuclei-ignore** 文件中，您可以定义要从nuclei扫描中排除的所有模板目录或者模板路径，要开始使用此功能，请确保使用`nuclei-update-templates`参数安装nuclei模板，现在可以根据`.nuclei-ignore`的文件来添加、更新、删除模板文件。 +[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop) -``` -nano ~/nuclei-templates/.nuclei-ignore -``` +### 许可证 -默认的**nuclei忽略**列表可以访问[这里]((https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore)，如果不想排除任何内容，只需要删除`.nuclei-ignore`文件。 +Nuclei使用[MIT许可证](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md) -* * * - -### 📋 笔记 - -- 进度条是实验性功能，在某些情况下可能无法使用。 -- 进度条不适用于工作流，因为是条件执行，所以不准确。 - - -## 致谢 - -也要看看这些类似的好项目，或许它们也适合你： - -[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop) - --------- - -Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。 +

+ Join Discord Check Nuclei Documentation +

From 0a8122571b05857df523135610b30380ccabf719 Mon Sep 17 00:00:00 2001 From: Xc1Ym Date: Thu, 2 Dec 2021 15:19:50 +0800 Subject: [PATCH 08/16] Update README.md Add Chinese Jump --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index bfa68972e..7ebe5b01b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ Join Discord

+

+ English • + 中文 +

+ --- Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei. From c807438ecdba1816292c3820491eff98d2a62b34 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 2 Dec 2021 11:19:37 +0100 Subject: [PATCH 09/16] fixing memory allocation for boolean pointer --- v2/pkg/protocols/dns/dns.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index c7dbd3134..d2890f8a8 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -103,7 +103,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.Retries = 3 } if request.Recursion == nil { - *request.Recursion = true + recursion := true + request.Recursion = &recursion } dnsClientOptions := &dnsclientpool.Configuration{ Retries: request.Retries, From 46a2a4440dd79591b8b670983714449ffd01baf9 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 2 Dec 2021 12:32:20 +0000 Subject: [PATCH 10/16] Auto Generate Syntax Docs + JSONSchema [Thu Dec 2 12:32:20 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 13 +++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 0d39e7fdc..08d0774b2 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -271,6 +271,19 @@ Self Contained marks Requests for the template as self-contained
+
+ +stop-at-first-match bool + +
+
+ +Stop execution once first match is found + +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index a2beba2c7..156090895 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -1123,6 +1123,11 @@ "type": "boolean", "title": "mark requests as self-contained", "description": "Mark Requests for the template as self-contained" + }, + "stop-at-first-match": { + "type": "boolean", + "title": "stop at first match", + "description": "Stop at first match for the template" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 6f21ce82c..d101c63b6 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -35,7 +35,7 @@ func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 11) + TemplateDoc.Fields = make([]encoder.Doc, 12) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -103,6 +103,11 @@ func init() { TemplateDoc.Fields[10].Note = "" TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained" TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[11].Name = "stop-at-first-match" + TemplateDoc.Fields[11].Type = "bool" + TemplateDoc.Fields[11].Note = "" + TemplateDoc.Fields[11].Description = "Stop execution once first match is found" + TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Stop execution once first match is found" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" From 5eb17833e43b7362a96cc64c017615fcf1c836e9 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 2 Dec 2021 19:06:54 +0000 Subject: [PATCH 11/16] Auto Generate Syntax Docs + JSONSchema [Thu Dec 2 19:06:54 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 459 ++++++++++++++++++++++++++++- v2/pkg/templates/templates_doc.go | 474 +++++++++++++++++++++++++++++- 2 files changed, 925 insertions(+), 8 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 08d0774b2..aed620891 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -576,6 +576,38 @@ Appears in: +
+ +
+ + Severity + +
+
+ + + + +Enum Values: + + + - undefined + + - info + + - low + + - medium + + - high + + - critical +
+ +
+ + + ## model.Classification @@ -715,8 +747,26 @@ path: method: GET ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- request - HTTP request made from the client +- response - HTTP response recieved from server +- status_code - Status Code received from the Server +- body - HTTP response body received from server (default) +- content_length - HTTP Response content length +- header,all_headers - HTTP response headers +- duration - HTTP request time duration +- all - HTTP response body + headers +- cookies_from_response - HTTP response cookies in name:value format +- headers_from_response - HTTP response headers in name:value format +
@@ -892,7 +942,7 @@ Valid values:
-method HTTPMethodTypeHolder +method HTTPMethodTypeHolder
@@ -1278,7 +1328,7 @@ Appears in:
-type MatcherTypeHolder +type MatcherTypeHolder
@@ -1597,6 +1647,52 @@ Valid values: +## MatcherTypeHolder +MatcherTypeHolder is used to hold internal type of the matcher + +Appears in: + + +- matchers.Matcher.type + + + + + +
+ +
+ + MatcherType + +
+
+ + + + +Enum Values: + + + - word + + - regex + + - binary + + - status + + - size + + - dsl +
+ +
+ + + + + ## extractors.Extractor Extractor is used to extract part of response using a regex. @@ -1649,7 +1745,7 @@ name: cookie-extractor
-type ExtractorTypeHolder +type ExtractorTypeHolder
@@ -1891,6 +1987,48 @@ Valid values: +## ExtractorTypeHolder +ExtractorTypeHolder is used to hold internal type of the extractor + +Appears in: + + +- extractors.Extractor.type + + + + + +
+ +
+ + ExtractorType + +
+
+ + + + +Enum Values: + + + - regex + + - kval + + - xpath + + - json +
+ +
+ + + + + ## generators.AttackTypeHolder AttackTypeHolder is used to hold internal type of the protocol @@ -1907,6 +2045,86 @@ Appears in: +
+ +
+ + AttackType + +
+
+ + + + +Enum Values: + + + - batteringram + + - pitchfork + + - clusterbomb +
+ +
+ + + + + +## HTTPMethodTypeHolder +HTTPMethodTypeHolder is used to hold internal type of the HTTP Method + +Appears in: + + +- http.Request.method + + + + + +
+ +
+ + HTTPMethodType + +
+
+ + + + +Enum Values: + + + - GET + + - GET + + - POST + + - PUT + + - DELETE + + - CONNECT + + - OPTIONS + + - TRACE + + - PATCH + + - PURGE +
+ +
+ + + ## dns.Request @@ -1931,8 +2149,24 @@ retries: 2 recursion: true ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- request - Request contains the DNS request in text format +- type - Type is the type of request made +- rcode - Rcode field returned for the DNS request +- question - Question contains the DNS question field +- extra - Extra contains the DNS response extra field +- answer - Answer contains the DNS response answer field +- ns - NS contains the DNS response NS field +- raw,body,all - Raw contains the raw DNS response (default) +- trace - Trace contains trace data for DNS request if enabled +
@@ -2027,7 +2261,7 @@ name: '{{FQDN}}'
-type DNSRequestTypeHolder +type DNSRequestTypeHolder
@@ -2159,6 +2393,58 @@ Resolvers to use for the dns requests +## DNSRequestTypeHolder +DNSRequestTypeHolder is used to hold internal type of the DNS type + +Appears in: + + +- dns.Request.type + + + + + +
+ +
+ + DNSRequestType + +
+
+ + + + +Enum Values: + + + - A + + - NS + + - DS + + - CNAME + + - SOA + + - PTR + + - MX + + - TXT + + - AAAA +
+ +
+ + + + + ## file.Request Request contains a File matching mechanism for local disk operations. @@ -2177,8 +2463,17 @@ extensions: - all ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- matched - Matched is the input which was matched upon +- path - Path is the path of file on local filesystem +- type - Type is the type of request made +- raw,body,all,data - Raw contains the raw file contents +
@@ -2366,8 +2661,19 @@ matchers: - zookeeper.version ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- request - Network request made from the client +- body,all,data - Network response recieved from server (default) +- raw - Full Network protocol data +
@@ -2606,7 +2912,7 @@ data: hex_decode('50494e47')
-type NetworkInputTypeHolder +type NetworkInputTypeHolder
@@ -2682,6 +2988,44 @@ name: prefix +## NetworkInputTypeHolder +NetworkInputTypeHolder is used to hold internal type of the Network type + +Appears in: + + +- network.Input.type + + + + + +
+ +
+ + NetworkInputType + +
+
+ + + + +Enum Values: + + + - hex + + - text +
+ +
+ + + + + ## headless.Request Request contains a Headless protocol request to be made from a template @@ -2692,8 +3036,18 @@ Appears in: +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- req - Headless request made from the client +- resp,body,data - Headless response recieved from client (default) +
@@ -2843,7 +3197,7 @@ Description is the optional description of the headless action
-action ActionTypeHolder +action ActionTypeHolder
@@ -2858,6 +3212,84 @@ Action is the type of the action to perform. +## ActionTypeHolder +ActionTypeHolder is used to hold internal type of the action + +Appears in: + + +- engine.Action.action + + + + + +
+ +
+ + ActionType + +
+
+ + + + +Enum Values: + + + - navigate + + - script + + - click + + - rightclick + + - text + + - screenshot + + - time + + - select + + - files + + - waitload + + - getresource + + - extract + + - setmethod + + - addheader + + - setheader + + - deleteheader + + - setbody + + - waitevent + + - keyboard + + - debug + + - sleep + + - waitvisible +
+ +
+ + + + + ## ssl.Request Request is a request for the SSL protocol @@ -2868,8 +3300,15 @@ Appears in: +Part Definitions: +- type - Type is the type of request made +- response - JSON SSL protocol handshake details +- not_after - Timestamp after which the remote cert expires +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +
@@ -2951,8 +3390,16 @@ Appears in: +Part Definitions: +- type - Type is the type of request made +- success - Success specifies whether websocket connection was successful +- request - Websocket request made to the server +- response - Websocket response recieved from the server +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index d101c63b6..c5c77200b 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -16,14 +16,20 @@ var ( MODELClassificationDoc encoder.Doc HTTPRequestDoc encoder.Doc MATCHERSMatcherDoc encoder.Doc + MatcherTypeHolderDoc encoder.Doc EXTRACTORSExtractorDoc encoder.Doc + ExtractorTypeHolderDoc encoder.Doc GENERATORSAttackTypeHolderDoc encoder.Doc + HTTPMethodTypeHolderDoc encoder.Doc DNSRequestDoc encoder.Doc + DNSRequestTypeHolderDoc encoder.Doc FILERequestDoc encoder.Doc NETWORKRequestDoc encoder.Doc NETWORKInputDoc encoder.Doc + NetworkInputTypeHolderDoc encoder.Doc HEADLESSRequestDoc encoder.Doc ENGINEActionDoc encoder.Doc + ActionTypeHolderDoc encoder.Doc SSLRequestDoc encoder.Doc WEBSOCKETRequestDoc encoder.Doc WEBSOCKETInputDoc encoder.Doc @@ -235,7 +241,20 @@ func init() { FieldName: "severity", }, } - SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0) + SEVERITYHolderDoc.Fields = make([]encoder.Doc, 1) + SEVERITYHolderDoc.Fields[0].Name = "" + SEVERITYHolderDoc.Fields[0].Type = "Severity" + SEVERITYHolderDoc.Fields[0].Note = "" + SEVERITYHolderDoc.Fields[0].Description = "" + SEVERITYHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + SEVERITYHolderDoc.Fields[0].EnumFields = []string{ + "undefined", + "info", + "low", + "medium", + "high", + "critical", + } MODELClassificationDoc.Type = "model.Classification" MODELClassificationDoc.Comments[encoder.LineComment] = "" @@ -287,6 +306,72 @@ func init() { FieldName: "requests", }, } + HTTPRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "request", + Value: "HTTP request made from the client", + }, + { + Key: "response", + Value: "HTTP response recieved from server", + }, + { + Key: "status_code", + Value: "Status Code received from the Server", + }, + { + Key: "body", + Value: "HTTP response body received from server (default)", + }, + { + Key: "content_length", + Value: "HTTP Response content length", + }, + { + Key: "header,all_headers", + Value: "HTTP response headers", + }, + { + Key: "duration", + Value: "HTTP request time duration", + }, + { + Key: "all", + Value: "HTTP response body + headers", + }, + { + Key: "cookies_from_response", + Value: "HTTP response cookies in name:value format", + }, + { + Key: "headers_from_response", + Value: "HTTP response headers in name:value format", + }, + } HTTPRequestDoc.Fields = make([]encoder.Doc, 27) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -590,6 +675,30 @@ func init() { "true", } + MatcherTypeHolderDoc.Type = "MatcherTypeHolder" + MatcherTypeHolderDoc.Comments[encoder.LineComment] = " MatcherTypeHolder is used to hold internal type of the matcher" + MatcherTypeHolderDoc.Description = "MatcherTypeHolder is used to hold internal type of the matcher" + MatcherTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "matchers.Matcher", + FieldName: "type", + }, + } + MatcherTypeHolderDoc.Fields = make([]encoder.Doc, 1) + MatcherTypeHolderDoc.Fields[0].Name = "" + MatcherTypeHolderDoc.Fields[0].Type = "MatcherType" + MatcherTypeHolderDoc.Fields[0].Note = "" + MatcherTypeHolderDoc.Fields[0].Description = "" + MatcherTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + MatcherTypeHolderDoc.Fields[0].EnumFields = []string{ + "word", + "regex", + "binary", + "status", + "size", + "dsl", + } + EXTRACTORSExtractorDoc.Type = "extractors.Extractor" EXTRACTORSExtractorDoc.Comments[encoder.LineComment] = " Extractor is used to extract part of response using a regex." EXTRACTORSExtractorDoc.Description = "Extractor is used to extract part of response using a regex." @@ -704,6 +813,28 @@ func init() { "true", } + ExtractorTypeHolderDoc.Type = "ExtractorTypeHolder" + ExtractorTypeHolderDoc.Comments[encoder.LineComment] = " ExtractorTypeHolder is used to hold internal type of the extractor" + ExtractorTypeHolderDoc.Description = "ExtractorTypeHolder is used to hold internal type of the extractor" + ExtractorTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "extractors.Extractor", + FieldName: "type", + }, + } + ExtractorTypeHolderDoc.Fields = make([]encoder.Doc, 1) + ExtractorTypeHolderDoc.Fields[0].Name = "" + ExtractorTypeHolderDoc.Fields[0].Type = "ExtractorType" + ExtractorTypeHolderDoc.Fields[0].Note = "" + ExtractorTypeHolderDoc.Fields[0].Description = "" + ExtractorTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + ExtractorTypeHolderDoc.Fields[0].EnumFields = []string{ + "regex", + "kval", + "xpath", + "json", + } + GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" GENERATORSAttackTypeHolderDoc.Description = "AttackTypeHolder is used to hold internal type of the protocol" @@ -721,7 +852,45 @@ func init() { FieldName: "attack", }, } - GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 0) + GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 1) + GENERATORSAttackTypeHolderDoc.Fields[0].Name = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Type = "AttackType" + GENERATORSAttackTypeHolderDoc.Fields[0].Note = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Description = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + GENERATORSAttackTypeHolderDoc.Fields[0].EnumFields = []string{ + "batteringram", + "pitchfork", + "clusterbomb", + } + + HTTPMethodTypeHolderDoc.Type = "HTTPMethodTypeHolder" + HTTPMethodTypeHolderDoc.Comments[encoder.LineComment] = " HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" + HTTPMethodTypeHolderDoc.Description = "HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" + HTTPMethodTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "http.Request", + FieldName: "method", + }, + } + HTTPMethodTypeHolderDoc.Fields = make([]encoder.Doc, 1) + HTTPMethodTypeHolderDoc.Fields[0].Name = "" + HTTPMethodTypeHolderDoc.Fields[0].Type = "HTTPMethodType" + HTTPMethodTypeHolderDoc.Fields[0].Note = "" + HTTPMethodTypeHolderDoc.Fields[0].Description = "" + HTTPMethodTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + HTTPMethodTypeHolderDoc.Fields[0].EnumFields = []string{ + "GET", + "GET", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE", + } DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" @@ -734,6 +903,64 @@ func init() { FieldName: "dns", }, } + DNSRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "request", + Value: "Request contains the DNS request in text format", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "rcode", + Value: "Rcode field returned for the DNS request", + }, + { + Key: "question", + Value: "Question contains the DNS question field", + }, + { + Key: "extra", + Value: "Extra contains the DNS response extra field", + }, + { + Key: "answer", + Value: "Answer contains the DNS response answer field", + }, + { + Key: "ns", + Value: "NS contains the DNS response NS field", + }, + { + Key: "raw,body,all", + Value: "Raw contains the raw DNS response (default)", + }, + { + Key: "trace", + Value: "Trace contains trace data for DNS request if enabled", + }, + } DNSRequestDoc.Fields = make([]encoder.Doc, 12) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -814,6 +1041,33 @@ func init() { DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests" DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" + DNSRequestTypeHolderDoc.Type = "DNSRequestTypeHolder" + DNSRequestTypeHolderDoc.Comments[encoder.LineComment] = " DNSRequestTypeHolder is used to hold internal type of the DNS type" + DNSRequestTypeHolderDoc.Description = "DNSRequestTypeHolder is used to hold internal type of the DNS type" + DNSRequestTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "dns.Request", + FieldName: "type", + }, + } + DNSRequestTypeHolderDoc.Fields = make([]encoder.Doc, 1) + DNSRequestTypeHolderDoc.Fields[0].Name = "" + DNSRequestTypeHolderDoc.Fields[0].Type = "DNSRequestType" + DNSRequestTypeHolderDoc.Fields[0].Note = "" + DNSRequestTypeHolderDoc.Fields[0].Description = "" + DNSRequestTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + DNSRequestTypeHolderDoc.Fields[0].EnumFields = []string{ + "A", + "NS", + "DS", + "CNAME", + "SOA", + "PTR", + "MX", + "TXT", + "AAAA", + } + FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." FILERequestDoc.Description = "Request contains a File matching mechanism for local disk operations." @@ -825,6 +1079,36 @@ func init() { FieldName: "file", }, } + FILERequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "path", + Value: "Path is the path of file on local filesystem", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "raw,body,all,data", + Value: "Raw contains the raw file contents", + }, + } FILERequestDoc.Fields = make([]encoder.Doc, 8) FILERequestDoc.Fields[0].Name = "matchers" FILERequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -888,6 +1172,44 @@ func init() { FieldName: "network", }, } + NETWORKRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "request", + Value: "Network request made from the client", + }, + { + Key: "body,all,data", + Value: "Network response recieved from server (default)", + }, + { + Key: "raw", + Value: "Full Network protocol data", + }, + } NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" @@ -993,6 +1315,26 @@ func init() { NETWORKInputDoc.Fields[3].AddExample("", "prefix") + NetworkInputTypeHolderDoc.Type = "NetworkInputTypeHolder" + NetworkInputTypeHolderDoc.Comments[encoder.LineComment] = " NetworkInputTypeHolder is used to hold internal type of the Network type" + NetworkInputTypeHolderDoc.Description = "NetworkInputTypeHolder is used to hold internal type of the Network type" + NetworkInputTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "network.Input", + FieldName: "type", + }, + } + NetworkInputTypeHolderDoc.Fields = make([]encoder.Doc, 1) + NetworkInputTypeHolderDoc.Fields[0].Name = "" + NetworkInputTypeHolderDoc.Fields[0].Type = "NetworkInputType" + NetworkInputTypeHolderDoc.Fields[0].Note = "" + NetworkInputTypeHolderDoc.Fields[0].Description = "" + NetworkInputTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + NetworkInputTypeHolderDoc.Fields[0].EnumFields = []string{ + "hex", + "text", + } + HEADLESSRequestDoc.Type = "headless.Request" HEADLESSRequestDoc.Comments[encoder.LineComment] = " Request contains a Headless protocol request to be made from a template" HEADLESSRequestDoc.Description = "Request contains a Headless protocol request to be made from a template" @@ -1002,6 +1344,40 @@ func init() { FieldName: "headless", }, } + HEADLESSRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "req", + Value: "Headless request made from the client", + }, + { + Key: "resp,body,data", + Value: "Headless response recieved from client (default)", + }, + } HEADLESSRequestDoc.Fields = make([]encoder.Doc, 5) HEADLESSRequestDoc.Fields[0].Name = "id" HEADLESSRequestDoc.Fields[0].Type = "string" @@ -1064,6 +1440,46 @@ func init() { ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform." ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform." + ActionTypeHolderDoc.Type = "ActionTypeHolder" + ActionTypeHolderDoc.Comments[encoder.LineComment] = " ActionTypeHolder is used to hold internal type of the action" + ActionTypeHolderDoc.Description = "ActionTypeHolder is used to hold internal type of the action" + ActionTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "engine.Action", + FieldName: "action", + }, + } + ActionTypeHolderDoc.Fields = make([]encoder.Doc, 1) + ActionTypeHolderDoc.Fields[0].Name = "" + ActionTypeHolderDoc.Fields[0].Type = "ActionType" + ActionTypeHolderDoc.Fields[0].Note = "" + ActionTypeHolderDoc.Fields[0].Description = "" + ActionTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + ActionTypeHolderDoc.Fields[0].EnumFields = []string{ + "navigate", + "script", + "click", + "rightclick", + "text", + "screenshot", + "time", + "select", + "files", + "waitload", + "getresource", + "extract", + "setmethod", + "addheader", + "setheader", + "deleteheader", + "setbody", + "waitevent", + "keyboard", + "debug", + "sleep", + "waitvisible", + } + SSLRequestDoc.Type = "ssl.Request" SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol" SSLRequestDoc.Description = "Request is a request for the SSL protocol" @@ -1073,6 +1489,28 @@ func init() { FieldName: "ssl", }, } + SSLRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "response", + Value: "JSON SSL protocol handshake details", + }, + { + Key: "not_after", + Value: "Timestamp after which the remote cert expires", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + } SSLRequestDoc.Fields = make([]encoder.Doc, 4) SSLRequestDoc.Fields[0].Name = "matchers" SSLRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1108,6 +1546,32 @@ func init() { FieldName: "websocket", }, } + WEBSOCKETRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "success", + Value: "Success specifies whether websocket connection was successful", + }, + { + Key: "request", + Value: "Websocket request made to the server", + }, + { + Key: "response", + Value: "Websocket response recieved from the server", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + } WEBSOCKETRequestDoc.Fields = make([]encoder.Doc, 8) WEBSOCKETRequestDoc.Fields[0].Name = "matchers" WEBSOCKETRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1259,14 +1723,20 @@ func GetTemplateDoc() *encoder.FileDoc { &MODELClassificationDoc, &HTTPRequestDoc, &MATCHERSMatcherDoc, + &MatcherTypeHolderDoc, &EXTRACTORSExtractorDoc, + &ExtractorTypeHolderDoc, &GENERATORSAttackTypeHolderDoc, + &HTTPMethodTypeHolderDoc, &DNSRequestDoc, + &DNSRequestTypeHolderDoc, &FILERequestDoc, &NETWORKRequestDoc, &NETWORKInputDoc, + &NetworkInputTypeHolderDoc, &HEADLESSRequestDoc, &ENGINEActionDoc, + &ActionTypeHolderDoc, &SSLRequestDoc, &WEBSOCKETRequestDoc, &WEBSOCKETInputDoc, From 0ca39bb6d2b8e8aa275644a6431813f5f079b937 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 Dec 2021 03:46:08 +0000 Subject: [PATCH 12/16] Auto Generate Syntax Docs + JSONSchema [Fri Dec 3 03:46:08 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 6 +++--- v2/pkg/templates/templates_doc.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index aed620891..ab5ea9ae9 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -136,7 +136,7 @@ dns: type: CNAME class: inet retries: 2 - recursion: true + recursion: false ``` @@ -2146,7 +2146,7 @@ name: '{{FQDN}}' type: CNAME class: inet retries: 2 -recursion: true +recursion: false ``` Part Definitions: @@ -2365,7 +2365,7 @@ trace-max-recursion: 100
-recursion bool +recursion dns.bool
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index c5c77200b..e1b739e9f 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -1031,7 +1031,7 @@ func init() { DNSRequestDoc.Fields[9].AddExample("Use a retry of 100 to 150 generally", 100) DNSRequestDoc.Fields[10].Name = "recursion" - DNSRequestDoc.Fields[10].Type = "bool" + DNSRequestDoc.Fields[10].Type = "dns.bool" DNSRequestDoc.Fields[10].Note = "" DNSRequestDoc.Fields[10].Description = "Recursion determines if resolver should recurse all records to get fresh results." DNSRequestDoc.Fields[10].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." From 83364211790eefc5d205d9c894d4770510310848 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 4 Dec 2021 18:40:11 +0100 Subject: [PATCH 13/16] Improving headless test cases (#1313) * Adding headless test cases --- .../headless/headless-basic.yaml | 18 +++++ .../headless/headless-extract-values.yaml | 31 +++++++ .../headless/headless-header-action.yaml | 24 ++++++ v2/cmd/integration-test/headless.go | 81 +++++++++++++++++++ v2/cmd/integration-test/integration-test.go | 1 + 5 files changed, 155 insertions(+) create mode 100644 integration_tests/headless/headless-basic.yaml create mode 100644 integration_tests/headless/headless-extract-values.yaml create mode 100644 integration_tests/headless/headless-header-action.yaml create mode 100644 v2/cmd/integration-test/headless.go diff --git a/integration_tests/headless/headless-basic.yaml b/integration_tests/headless/headless-basic.yaml new file mode 100644 index 000000000..cfc7dcb3c --- /dev/null +++ b/integration_tests/headless/headless-basic.yaml @@ -0,0 +1,18 @@ +id: headless-basic +info: + name: Headless Basic + author: pdteam + severity: info + tags: headless + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}/" + + - action: waitload + matchers: + - type: word + words: + - "" \ No newline at end of file diff --git a/integration_tests/headless/headless-extract-values.yaml b/integration_tests/headless/headless-extract-values.yaml new file mode 100644 index 000000000..e780ac32c --- /dev/null +++ b/integration_tests/headless/headless-extract-values.yaml @@ -0,0 +1,31 @@ + +id: headless-extract-values +info: + name: Headless Extract Value + author: pdteam + severity: info + tags: headless + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + - action: waitload + # From headless/extract-urls.yaml + - action: script + name: extract + args: + code: | + '\n' + [...new Set(Array.from(document.querySelectorAll('[src], [href], [url], [action]')).map(i => i.src || i.href || i.url || i.action))].join('\r\n') + '\n' + + matchers: + - type: word + words: + - "test.html" + + extractors: + - type: kval + part: extract + kval: + - extract \ No newline at end of file diff --git a/integration_tests/headless/headless-header-action.yaml b/integration_tests/headless/headless-header-action.yaml new file mode 100644 index 000000000..ca3c329d8 --- /dev/null +++ b/integration_tests/headless/headless-header-action.yaml @@ -0,0 +1,24 @@ +id: headless-header-action +info: + name: Headless Header Action + author: pdteam + severity: info + tags: headless + +headless: + - steps: + - action: setheader + args: + part: request + key: Test + value: test value + + - action: navigate + args: + url: "{{BaseURL}}/" + + - action: waitload + matchers: + - type: word + words: + - "test value" \ No newline at end of file diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go new file mode 100644 index 000000000..6039fdb64 --- /dev/null +++ b/v2/cmd/integration-test/headless.go @@ -0,0 +1,81 @@ +package main + +import ( + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var headlessTestcases = map[string]testutils.TestCase{ + "headless/headless-basic.yaml": &headlessBasic{}, + "headless/headless-header-action.yaml": &headlessHeaderActions{}, + "headless/headless-extract-values.yaml": &headlessExtractValues{}, +} + +type headlessBasic struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessBasic) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type headlessHeaderActions struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessHeaderActions) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + testValue := r.Header.Get("test") + if r.Header.Get("test") != "" { + _, _ = w.Write([]byte("" + testValue + "")) + } + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type headlessExtractValues struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessExtractValues) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("test")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + if len(results) != 3 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index d55f74e0f..70bb193ac 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -29,6 +29,7 @@ func main() { "workflow": workflowTestcases, "loader": loaderTestcases, "websocket": websocketTestCases, + "headless": headlessTestcases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { From 3c88afac0ca24fd7df80de3af20a0c08f316b7e0 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 5 Dec 2021 15:11:14 +0100 Subject: [PATCH 14/16] Fixing payloads path during validation (#1320) * Fixing payloads path during validation * Added GH Action for public template parsing / validation * tracking payload errors as syntax warnings * improving path parsing + introducing hard failure for runtime errors on validation Co-authored-by: sandeep --- .github/workflows/template-validate.yml | 29 +++++++++++++++++++ v2/go.mod | 8 ++--- v2/go.sum | 23 +++------------ v2/internal/runner/runner.go | 3 +- v2/pkg/catalog/loader/loader.go | 2 ++ v2/pkg/parsers/parser.go | 6 ++-- .../protocols/common/generators/validate.go | 14 ++++++--- 7 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/template-validate.yml diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml new file mode 100644 index 000000000..0f78d4de6 --- /dev/null +++ b/.github/workflows/template-validate.yml @@ -0,0 +1,29 @@ +name: 🛠 Template Validate + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Cache Go + id: cache-go + uses: actions/cache@v2 + with: + path: /home/runner/go + key: ${{ runner.os }}-go + + - name: Installing Nuclei + if: steps.cache-go.outputs.cache-hit != 'true' + run: | + go install github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest + + - name: Template Validation + run: | + nuclei -validate + nuclei -validate -w ./workflows \ No newline at end of file diff --git a/v2/go.mod b/v2/go.mod index 709577509..490908476 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -65,6 +65,8 @@ require ( moul.io/http2curl v1.0.0 ) +require github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e + require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect @@ -77,7 +79,6 @@ require ( github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/c4milo/unpackit v0.1.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/dave/dst v0.26.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect @@ -92,7 +93,6 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.6 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect @@ -126,14 +126,10 @@ require ( github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/mod v0.4.2 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - golang.org/x/tools v0.1.3 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - mvdan.cc/gofumpt v0.1.1 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 731a08159..bd73d8ced 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -163,7 +163,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -596,6 +595,10 @@ github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/folderutil v0.0.0-20210804143510-68474319fd84 h1:+VqGxv8ywpIHwGGSCOcGn/q5kkuB6F1AZtY42I8VnXc= +github.com/projectdiscovery/folderutil v0.0.0-20210804143510-68474319fd84/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= +github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e h1:ozfSeEc5j1f7NCEZAiAskP/KYfBD/TzPmFTIfh+CEwE= +github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk= github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -640,22 +643,7 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125104740-9b1096de655d h1:sXbcjsLPDgOrlGXgCKbT6MMyH/hTY3OJhhwsyM2bNlI= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125104740-9b1096de655d/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125105154-082a0a3cc326 h1:/fGqkG8GlfdvlTCfvAoSA/WgEAJrmCnW5qtdd7QXnwA= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125105154-082a0a3cc326/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125110259-585ff5584784 h1:Be2eD7oXNvCbFufVhvkiO5a0SRVN+Ri2V9pXKL2uJp8= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125110259-585ff5584784/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125112758-99d87632e49a h1:xGeZvil8Fe5LpGJbTGZGafnnbedNGtVcvLv3nYtmhXQ= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125112758-99d87632e49a/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135034-67f4c31feb2b h1:oSBnxdyyDU/WpNKUAXrhydgx5+JtDT7KfJR+hOEaBXk= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135034-67f4c31feb2b/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0 h1:mOptvTJ32yUuqQjjSfiPkPCelTWzqnts92uNOZBXZZo= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e h1:0ZxOM0Q0/ESa24L/vq3fxs9YipxfHR4Y3jM/H2ReJ5E= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -907,7 +895,6 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1126,7 +1113,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1267,7 +1253,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= -mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index a5aea7e9e..0d33511b8 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -279,7 +279,7 @@ func (r *Runner) RunEnumeration() error { if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil { return err } - if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 { + if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 && stats.GetValue(parsers.RuntimeWarningsStats) == 0 { gologger.Info().Msgf("All templates validated successfully\n") } else { return errors.New("encountered errors while performing template validation") @@ -362,6 +362,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { // Display stats for any loaded templates' syntax warnings or errors stats.Display(parsers.SyntaxWarningStats) stats.Display(parsers.SyntaxErrorStats) + stats.Display(parsers.RuntimeWarningsStats) builder := &strings.Builder{} if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 12ddcfc39..23dfd980a 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) // Config contains the configuration options for the loader @@ -218,6 +219,7 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template if loaded { parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) if err != nil { + stats.Increment(parsers.RuntimeWarningsStats) gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err) } else if parsed != nil { loadedTemplates = append(loadedTemplates, parsed) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index e26e7488e..c63922466 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -108,8 +108,9 @@ var ( ) const ( - SyntaxWarningStats = "syntax-warnings" - SyntaxErrorStats = "syntax-errors" + SyntaxWarningStats = "syntax-warnings" + SyntaxErrorStats = "syntax-errors" + RuntimeWarningsStats = "runtime-warnings" ) func init() { @@ -118,6 +119,7 @@ func init() { stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") + stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") } // ParseTemplate parses a template and returns a *templates.Template structure diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index 305cae092..b04f8034c 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -4,9 +4,9 @@ import ( "errors" "fmt" "os" - "path/filepath" "strings" + "github.com/projectdiscovery/folderutil" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -26,10 +26,16 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat } changed := false - pathTokens := strings.Split(templatePath, string(os.PathSeparator)) - for i := range pathTokens { - payloadPath := filepath.Join(filepath.Join(pathTokens[:i]...), payloadType) + templatePathInfo, err := folderutil.NewPathInfo(templatePath) + if err != nil { + return err + } + payloadPathsToProbe, err := templatePathInfo.MeshWith(payloadType) + if err != nil { + return err + } + for _, payloadPath := range payloadPathsToProbe { if fileExists(payloadPath) { payloads[name] = payloadPath changed = true From fb3b4fc510f7406dd7722e5b715b3fa32f0102c7 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 5 Dec 2021 19:45:44 +0530 Subject: [PATCH 15/16] version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 87e6c672d..bb5cfb24f 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.4-dev` +const Version = `2.5.4` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From df55f7a2eb591a7949b5c66a9c54db54aa28f450 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 5 Dec 2021 20:14:16 +0530 Subject: [PATCH 16/16] Disabling no-sandbox in headless engine (#1135) * Disabling no-sandbox in headless engine * limiting disabling sandbox to bare minimum * adding warnings related to linux os and root user requirement Co-authored-by: mzack --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 3 +++ v2/pkg/protocols/headless/engine/engine.go | 13 ++++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 059a96167..bd015fede 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,7 +134,7 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "headless", "Headless", - flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), + flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support (root user on linux will disable sandbox)"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "Use local installed chrome browser instead of nuclei installed"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0d33511b8..b71587267 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -76,6 +76,9 @@ func New(options *types.Options) (*Runner, error) { gologger.Warning().Msgf("Could not update templates: %s\n", err) } if options.Headless { + if engine.MustDisableSandbox() { + gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n") + } browser, err := engine.New(options) if err != nil { return nil, err diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index d73e79044..6c1046b67 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net/http" "os" + "runtime" "strings" "github.com/corpix/uarand" @@ -44,12 +45,15 @@ func New(options *types.Options) (*Browser, error) { Set("disable-notifications", "true"). Set("hide-scrollbars", "true"). Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). - Set("no-sandbox", "true"). Set("mute-audio", "true"). Set("incognito", "true"). Delete("use-mock-keychain"). UserDataDir(dataStore) + if MustDisableSandbox() { + chromeLauncher = chromeLauncher.NoSandbox(true) + } + if options.UseInstalledChrome { if chromePath, hasChrome := launcher.LookPath(); hasChrome { chromeLauncher.Bin(chromePath) @@ -105,6 +109,13 @@ func New(options *types.Options) (*Browser, error) { return engine, nil } +// MustDisableSandbox determines if the current os and user needs sandbox mode disabled +func MustDisableSandbox() bool { + // linux with root user needs "--no-sandbox" option + // https://github.com/chromium/chromium/blob/c4d3c31083a2e1481253ff2d24298a1dfe19c754/chrome/test/chromedriver/client/chromedriver.py#L209 + return runtime.GOOS == "linux" && os.Geteuid() == 0 +} + // Close closes the browser engine func (b *Browser) Close() { b.engine.Close()