From 7448869f29efd9c4d5078edc247941c90c04c25c Mon Sep 17 00:00:00 2001 From: Geeknik Labs <466878+geeknik@users.noreply.github.com> Date: Tue, 27 Jul 2021 08:35:37 -0500 Subject: [PATCH 01/28] Update main.go typo fix --- v2/cmd/nuclei/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 85bc7ae87..4202aa371 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -153,7 +153,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "stats", "Statistics", flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), - flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to and output file in JSONL(ines) format"), + flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"), From 870f9be64b1a7f26469ff4fb33508cc87c6f2c8e Mon Sep 17 00:00:00 2001 From: Geeknik Labs <466878+geeknik@users.noreply.github.com> Date: Tue, 27 Jul 2021 08:37:04 -0500 Subject: [PATCH 02/28] Update README.md typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b475133c5..90a4cda46 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ UPDATE: STATISTICS: -stats display statistics about the running scan - -stats-json write statistics data to and output file in JSONL(ines) format + -stats-json write statistics data to an output file in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) -metrics expose nuclei metrics on a port -metrics-port int port to expose nuclei metrics on (default 9092) From 808efc3dac11e8632bb3594ebe5f1b05b87036ba Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 27 Jul 2021 20:46:14 +0530 Subject: [PATCH 03/28] dev update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 5480062f6..75fd62156 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -28,7 +28,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.4.1` +const Version = `2.4.2-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 119f7380fb2324431015a30db913efdede039048 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 28 Jul 2021 23:01:05 +0300 Subject: [PATCH 04/28] Removing normalization of case-sensitive parameters Requires integration of a new goflags --- v2/cmd/nuclei/main.go | 10 +++++----- v2/pkg/types/types.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 4202aa371..468a69289 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -57,15 +57,15 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "filters", "Filtering", - flagSet.StringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), - flagSet.StringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list - flagSet.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), + flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), + flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list + flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"), flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"), - flagSet.StringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"), - flagSet.StringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.NormalizedStringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"), + flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), ) createGroup(flagSet, "output", "Output", diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c80ed5b1f..31ee54e51 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -7,9 +7,9 @@ type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths // can be specified with -l flag and -tags can be used in combination with // the -l flag. - Tags goflags.StringSlice + Tags goflags.NormalizedStringSlice // ExcludeTags is the list of tags to exclude - ExcludeTags goflags.StringSlice + ExcludeTags goflags.NormalizedStringSlice // Workflows specifies any workflows to run by nuclei Workflows goflags.StringSlice // Templates specifies the template/templates to use @@ -19,11 +19,11 @@ type Options struct { // CustomHeaders is the list of custom global headers to send with each request. CustomHeaders goflags.StringSlice // Severity filters templates based on their severity and only run the matching ones. - Severity goflags.StringSlice + Severity goflags.NormalizedStringSlice // Author filters templates based on their author and only run the matching ones. - Author goflags.StringSlice + Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist - IncludeTags goflags.StringSlice + IncludeTags goflags.NormalizedStringSlice // IncludeTemplates includes specified templates to be run even while being in denylist IncludeTemplates goflags.StringSlice From 80d0def9638a59021dfb1c29fa5f676d1a50b243 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 29 Jul 2021 16:06:32 +0530 Subject: [PATCH 05/28] go mod update --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 483e96728..4920a1f82 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -29,7 +29,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.8 - github.com/projectdiscovery/goflags v0.0.5 + github.com/projectdiscovery/goflags v0.0.6 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.3 diff --git a/v2/go.sum b/v2/go.sum index c11c78831..97fb14f0d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -258,8 +258,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= -github.com/projectdiscovery/goflags v0.0.5 h1:jI6HD9Z7vkg4C4Cz16BfZKICnIf94W3KFU5M3DcUgUk= -github.com/projectdiscovery/goflags v0.0.5/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= +github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew= +github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From 53de8d77f29194e91948b99801f4538c7aaa8bfa Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 29 Jul 2021 13:36:44 +0300 Subject: [PATCH 06/28] Removing normalization of case-sensitive parameters Integrated new goflags version (v0.0.6) --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 483e96728..4920a1f82 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -29,7 +29,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.8 - github.com/projectdiscovery/goflags v0.0.5 + github.com/projectdiscovery/goflags v0.0.6 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.3 diff --git a/v2/go.sum b/v2/go.sum index c11c78831..97fb14f0d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -258,8 +258,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= -github.com/projectdiscovery/goflags v0.0.5 h1:jI6HD9Z7vkg4C4Cz16BfZKICnIf94W3KFU5M3DcUgUk= -github.com/projectdiscovery/goflags v0.0.5/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= +github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew= +github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From 774bb0145f6c4fb056c0a53062943174d8850fa6 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 29 Jul 2021 19:18:47 +0530 Subject: [PATCH 07/28] version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 75fd62156..4604d2b92 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -28,7 +28,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.4.2-dev` +const Version = `2.4.2` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 5b03d257432b058e09e709eb240859a9b436697a Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 29 Jul 2021 19:24:39 +0530 Subject: [PATCH 08/28] Do no check for template updates with flag --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/update.go | 3 +++ v2/pkg/types/types.go | 2 ++ 3 files changed, 6 insertions(+) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 468a69289..6d381126a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -148,6 +148,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "update", "Update", flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"), flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"), + flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"), flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"), ) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 7e62e5e4d..45591f04f 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -82,6 +82,9 @@ func (r *Runner) updateTemplates() error { r.templatesConfig = currentConfig } + if r.options.NoUpdateTemplates { + return nil + } // Check if last checked for nuclei-ignore is more than 1 hours. // and if true, run the check. // diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 31ee54e51..291ec872a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -139,4 +139,6 @@ type Options struct { NoInteractsh bool // UpdateNuclei checks for an update for the nuclei engine UpdateNuclei bool + // NoUpdateTemplates disables checking for nuclei templates updates + NoUpdateTemplates bool } From 571a4c3c330b19ff124ad5e1b129d7590236aaa9 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 29 Jul 2021 20:04:21 +0530 Subject: [PATCH 09/28] dev update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 4604d2b92..478ef3793 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -28,7 +28,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.4.2` +const Version = `2.4.3-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 8d0b5d62034a3699ee5d9e82001b7751f02d799d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 30 Jul 2021 15:29:12 +0530 Subject: [PATCH 10/28] RES-120 Added functional testing implementation --- .gitignore | 5 +- v2/cmd/functional-test/main.go | 79 +++++++++++++++++++ v2/cmd/functional-test/run.sh | 11 +++ v2/cmd/functional-test/testcases.txt | 51 ++++++++++++ v2/cmd/integration-test/integration-test.go | 7 +- v2/internal/testutils/integration.go | 19 +++++ .../deserialization/testdata/Deserialize.java | 2 +- 7 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 v2/cmd/functional-test/main.go create mode 100644 v2/cmd/functional-test/run.sh create mode 100644 v2/cmd/functional-test/testcases.txt diff --git a/.gitignore b/.gitignore index c3605e789..e812e00d2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ v2/cmd/integration-test/integration-test bin v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class -v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser \ No newline at end of file +v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser +v2/cmd/functional-test/nuclei_dev +v2/cmd/functional-test/nuclei_main +v2/cmd/functional-test/functional-test \ No newline at end of file diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go new file mode 100644 index 000000000..1e54cda7e --- /dev/null +++ b/v2/cmd/functional-test/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" +) + +var ( + success = aurora.Green("[✓]").String() + failed = aurora.Red("[✘]").String() + errored = false + + mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary") + devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary") + testcases = flag.String("testcases", "", "Test cases file for nuclei functional tests") +) + +func main() { + flag.Parse() + + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + file, err := os.Open(*testcases) + if err != nil { + return errors.Wrap(err, "could not open test cases") + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + if err := runIndividualTestCase(text); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, text) + } + } + return nil +} + +func runIndividualTestCase(testcase string) error { + parts := strings.Fields(testcase) + + var finalArgs []string + if len(parts) > 1 { + finalArgs = parts[1:] + } + mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run nuclei main test") + } + devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run nuclei dev test") + } + if mainOutput == devOutput { + return nil + } + return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) +} diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh new file mode 100644 index 000000000..c609b9b79 --- /dev/null +++ b/v2/cmd/functional-test/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +git checkout master +cd ../nuclei/ +go build +cp nuclei ../functional-test/nuclei_main +git checkout dev +go build +cp nuclei ../functional-test/nuclei_dev +cd ../functional-test +./functional-test -main ./nuclei_main -dev ./nuclei_dev -testcases testcases.txt diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt new file mode 100644 index 000000000..18bfd326a --- /dev/null +++ b/v2/cmd/functional-test/testcases.txt @@ -0,0 +1,51 @@ +{{binary}} +{{binary}} -tags cve +{{binary}} -tags cve,exposure +{{binary}} -tags cve,exposure -tags token +{{binary}} -tags cve,exposure -tags token,logs +{{binary}} -tags "cve","exposure" -tags "token","logs" +{{binary}} -tags 'cve','exposure' -tags 'token','logs' +{{binary}} -tags cve -severity high +{{binary}} -tags cve,exposure -severity high,critical +{{binary}} -tags cve,exposure -severity "high,critical,medium" +{{binary}} -tags cve -author geeknik +{{binary}} -tags cve -author geeknik,pdteam +{{binary}} -tags cve -author geeknik -severity high +{{binary}} -tags cve +{{binary}} -tags cve,exposure +{{binary}} -tags cve,exposure -tags token +{{binary}} -tags cve,exposure -tags token,logs +{{binary}} -tags "cve","exposure" -tags "token","logs" +{{binary}} -tags 'cve','exposure' -tags 'token','logs' +{{binary}} -tags cve -severity high +{{binary}} -tags cve,exposure -severity high,critical +{{binary}} -tags cve,exposure -severity "high,critical,medium" +{{binary}} -tags cve -author geeknik +{{binary}} -tags cve -author geeknik,pdteam +{{binary}} -tags cve -author geeknik -severity high +{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical +{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity "high,critical" +{{binary}} -tags cve -etags ssrf +{{binary}} -tags cve,exposure -etags ssrf,config +{{binary}} -tags cve,exposure -etags ssrf,config -severity high +{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik +{{binary}} -tags cve,dos,fuzz +{{binary}} -tags cve -include-tags dos,fuzz +{{binary}} -tags cve -exclude-tags cve2020 +{{binary}} -tags cve -exclude-templates cves/2020/ +{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml +{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/ +{{binary}} -t cves/ +{{binary}} -t cves/ -t exposures/ +{{binary}} -t cves/ -t exposures/ -tags config +{{binary}} -t cves/ -t exposures/ -tags config,ssrf +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/ +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml +{{binary}} -w workflows +{{binary}} -w workflows -author geeknik,pdteam +{{binary}} -w workflows -severity high,critical +{{binary}} -w workflows -author geeknik,pdteam -severity high,critical \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 6b9e161c9..ad416b5c4 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -13,6 +13,8 @@ var ( debug = os.Getenv("DEBUG") == "true" customTest = os.Getenv("TEST") protocol = os.Getenv("PROTO") + + errored = false ) func main() { @@ -36,13 +38,16 @@ func main() { err := test.Execute(file) if err != nil { fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err) - os.Exit(1) + errored = true } else { fmt.Printf("%s Test \"%s\" passed!\n", success, file) } } } } + if errored { + os.Exit(1) + } } func errIncorrectResultsCount(results []string) error { diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index b7af84452..2e457bfb5 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -1,9 +1,11 @@ package testutils import ( + "errors" "net" "os" "os/exec" + "regexp" "strings" ) @@ -30,6 +32,23 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( return parts, nil } +var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: ([0-9]+)`) + +// RunNucleiAndGetResults returns a list of results for a template +func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) { + cmd := exec.Command(nucleiBinary, args...) + + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + matches := templateLoaded.FindAllStringSubmatch(string(data), -1) + if len(matches) == 0 { + return "", errors.New("no matches found") + } + return matches[0][1], nil +} + // RunNucleiWorkflowAndGetResults returns a list of results for a workflow func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent") diff --git a/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java b/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java index 2b74c0175..b63528ddf 100644 --- a/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java +++ b/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java @@ -1,6 +1,6 @@ import java.io.*; -class Deserialize{ +class Deserialize { public static void main(String args[]) { FileInputStream fileIn = null; ObjectInputStream in = null; From a9ecc1fbca7415b51c32e8bdf75adab8fe058ff3 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 30 Jul 2021 15:30:44 +0530 Subject: [PATCH 11/28] Statisfying the linter --- v2/internal/testutils/integration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 2e457bfb5..1f59c3011 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -32,7 +32,7 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( return parts, nil } -var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: ([0-9]+)`) +var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: (\d+)`) // RunNucleiAndGetResults returns a list of results for a template func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) { From 0eea45fb60088ba4cc907c0bcb3fe4ceba3d89f9 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 30 Jul 2021 20:01:27 +0530 Subject: [PATCH 12/28] Add building step --- v2/cmd/functional-test/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index c609b9b79..35262957d 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -8,4 +8,5 @@ git checkout dev go build cp nuclei ../functional-test/nuclei_dev cd ../functional-test +go build ./functional-test -main ./nuclei_main -dev ./nuclei_dev -testcases testcases.txt From eb080223d87c612d5dc6c0827597bf441e0699d2 Mon Sep 17 00:00:00 2001 From: Souvik Hazra Date: Sat, 31 Jul 2021 15:46:21 +0530 Subject: [PATCH 13/28] Added support for rate limit per minute --- v2/cmd/nuclei/main.go | 1 + v2/go.mod | 2 +- v2/go.sum | 4 ++++ v2/internal/runner/runner.go | 4 +++- v2/pkg/types/types.go | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 6d381126a..a3268c840 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -109,6 +109,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "rate-limit", "Rate-Limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), + flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"), ) diff --git a/v2/go.mod b/v2/go.mod index 4920a1f82..2e1a97acf 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -52,7 +52,7 @@ require ( github.com/xanzy/go-gitlab v0.44.0 go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 - go.uber.org/ratelimit v0.1.0 + go.uber.org/ratelimit v0.2.0 golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect golang.org/x/net v0.0.0-20210521195947-fe42d452be8f golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 diff --git a/v2/go.sum b/v2/go.sum index 97fb14f0d..f8512f4c7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,6 +40,8 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -366,6 +368,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= +go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 017d3d869..5388f3432 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -236,7 +236,9 @@ func New(options *types.Options) (*Runner, error) { } } - if options.RateLimit > 0 { + if options.RateLimitMinute > 0 { + runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second)) + } else if options.RateLimit > 0 { runner.ratelimiter = ratelimit.New(options.RateLimit) } else { runner.ratelimiter = ratelimit.NewUnlimited() diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 291ec872a..872b107ee 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -70,6 +70,8 @@ type Options struct { Retries int // Rate-Limit is the maximum number of requests per specified target RateLimit int + // Rate-Limit is the maximum number of requests per minute for specified target + RateLimitMinute int // PageTimeout is the maximum time to wait for a page in seconds PageTimeout int // InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time. From 3529cfa1d63b45af3c00251001e1634da137a38c Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sat, 31 Jul 2021 22:49:23 +0200 Subject: [PATCH 14/28] json extractor support --- v2/go.mod | 6 ++++ v2/go.sum | 15 ++++++++++ v2/pkg/operators/extractors/compile.go | 13 +++++++++ v2/pkg/operators/extractors/extract.go | 34 +++++++++++++++++++++++ v2/pkg/operators/extractors/extractors.go | 9 ++++++ v2/pkg/protocols/http/operators.go | 2 ++ 6 files changed, 79 insertions(+) diff --git a/v2/go.mod b/v2/go.mod index 4920a1f82..3f7498d49 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -4,6 +4,7 @@ go 1.15 require ( github.com/Knetic/govaluate v3.0.0+incompatible + github.com/alecthomas/participle v0.3.0 // indirect github.com/andygrunwald/go-jira v1.13.0 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible @@ -17,15 +18,20 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/hokaccha/go-prettyjson v0.0.0-20180920040306-f579f869bbfe // indirect + github.com/itchyny/astgen-go v0.0.0-20190623123749-11f6ca3e2b1f // indirect + github.com/itchyny/gojq v0.12.4 // indirect github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 + github.com/lestrrat-go/strftime v0.0.0-20190725011945-5c849dd2c51d // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mattn/go-runewidth v0.0.10 // indirect github.com/miekg/dns v1.1.38 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.4 + github.com/pbnjay/strptime v0.0.0-20140226051138-5c05b0d668c9 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.8 diff --git a/v2/go.sum b/v2/go.sum index 97fb14f0d..8e3438892 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,6 +40,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/participle v0.3.0/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -138,6 +139,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -173,11 +175,18 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hokaccha/go-prettyjson v0.0.0-20180920040306-f579f869bbfe/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/itchyny/astgen-go v0.0.0-20190623123749-11f6ca3e2b1f/go.mod h1:9Gyr9nZoENI+woes+xm+BFhmvYmAp6bPtXD866pQH9g= +github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= +github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= +github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -212,6 +221,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/strftime v0.0.0-20190725011945-5c849dd2c51d/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -220,6 +231,7 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -248,6 +260,7 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= +github.com/pbnjay/strptime v0.0.0-20140226051138-5c05b0d668c9/go.mod h1:6Hr+C/olSdkdL3z68MlyXWzwhvwmwN7KuUFXGb3PoOk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -474,6 +487,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -493,6 +507,7 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index b189c3de1..5d1b134c3 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -2,6 +2,7 @@ package extractors import ( "fmt" + "github.com/itchyny/gojq" "regexp" "strings" ) @@ -28,6 +29,18 @@ func (e *Extractor) CompileExtractors() error { e.KVal[i] = strings.ToLower(kval) } + for _, query := range e.Json { + query, err := gojq.Parse(query) + if err != nil { + return fmt.Errorf("could not parse json: %s", query) + } + compiled, err := gojq.Compile(query) + if err != nil { + return fmt.Errorf("could not compile json: %s", query) + } + e.jsonCompiled = append(e.jsonCompiled, compiled) + } + // Setup the part of the request to match, if any. if e.Part == "" { e.Part = "body" diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 5fecdcf9f..c1c9fbd6e 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -1,6 +1,7 @@ package extractors import ( + "encoding/json" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -42,3 +43,36 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} } return results } + +// ExtractJson extracts key value pairs from a data map +func (e *Extractor) ExtractJson(corpus string) map[string]struct{} { + results := make(map[string]struct{}) + + var jsonObj interface{} + + err := json.Unmarshal([]byte(corpus), &jsonObj) + + if err != nil { + return results + } + + for _, k := range e.jsonCompiled { + iter := k.Run(jsonObj) + for { + v, ok := iter.Next() + if !ok { + break + } + if _, ok := v.(error); ok { + break + } + bytes, err := json.Marshal(v) + if err != nil { + break + } + results[string(bytes)] = struct{}{} + } + } + + return results +} diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index f593a1747..9e08c05f0 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -1,6 +1,7 @@ package extractors import "regexp" +import "github.com/itchyny/gojq" // Extractor is used to extract part of response using a regex. type Extractor struct { @@ -21,6 +22,11 @@ type Extractor struct { // KVal are the kval to be present in the response headers/cookies KVal []string `yaml:"kval,omitempty"` + // Regex are the regex pattern required to be present in the response + Json []string `yaml:"json"` + // jsonCompiled is the compiled variant + jsonCompiled []*gojq.Code + // Part is the part of the request to match // // By default, matching is performed in request body. @@ -37,12 +43,15 @@ const ( RegexExtractor ExtractorType = iota + 1 // KValExtractor extracts responses with key:value KValExtractor + // JSONExtractor extracts responses with json + JSONExtractor ) // ExtractorTypes is an table for conversion of extractor type from string. var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, + "json": JSONExtractor, } // GetType returns the type of the matcher diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 3f0a63ea9..57307c619 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -54,6 +54,8 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return extractor.ExtractRegex(item) case extractors.KValExtractor: return extractor.ExtractKval(data) + case extractors.JSONExtractor: + return extractor.ExtractJson(item) } return nil } From 166344d7939e8a7ea2bc343083b296e1ed747120 Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sat, 31 Jul 2021 23:48:14 +0200 Subject: [PATCH 15/28] add and fix tests for json --- v2/pkg/operators/extractors/extract.go | 8 +-- v2/pkg/protocols/http/operators_test.go | 74 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index c1c9fbd6e..81f995b1a 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -66,13 +66,11 @@ func (e *Extractor) ExtractJson(corpus string) map[string]struct{} { if _, ok := v.(error); ok { break } - bytes, err := json.Marshal(v) - if err != nil { - break + itemString := types.ToString(v) + if _, ok := results[itemString]; !ok { + results[itemString] = struct{}{} } - results[string(bytes)] = struct{}{} } } - return results } diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 20ff31d9b..d658edc78 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -1,6 +1,7 @@ package http import ( + "fmt" "net/http" "testing" "time" @@ -168,6 +169,20 @@ func TestHTTPOperatorExtract(t *testing.T) { require.Greater(t, len(data), 0, "could not extractor kval valid response") require.Equal(t, map[string]struct{}{"Test-Response": {}}, data, "could not extract correct kval data") }) + + t.Run("json", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + Json: []string{".batters | .batter | .[] | .id"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + event["body"] = exampleJSONResponseBody + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"1001": {}, "1002": {}, "1003": {}, "1004": {}}, data, "could not extract correct json data") + }) } func TestHTTPMakeResult(t *testing.T) { @@ -305,3 +320,62 @@ const exampleResponseBody = ` ` + +const exampleJSONResponseBody = ` +{ + "id": "0001", + "type": "donut", + "name": "Cake", + "ppu": 0.55, + "batters": { + "batter": [ + { + "id": "1001", + "type": "Regular" + }, + { + "id": "1002", + "type": "Chocolate" + }, + { + "id": "1003", + "type": "Blueberry" + }, + { + "id": "1004", + "type": "Devil's Food" + } + ] + }, + "topping": [ + { + "id": "5001", + "type": "None" + }, + { + "id": "5002", + "type": "Glazed" + }, + { + "id": "5005", + "type": "Sugar" + }, + { + "id": "5007", + "type": "Powdered Sugar" + }, + { + "id": "5006", + "type": "Chocolate with Sprinkles" + }, + { + "id": "5003", + "type": "Chocolate" + }, + { + "id": "5004", + "type": "Maple" + } + ] +} +` From da71945f97338e3fc332190d5ab6e991e9b6a990 Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sat, 31 Jul 2021 23:51:23 +0200 Subject: [PATCH 16/28] Update operators_test.go --- v2/pkg/protocols/http/operators_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index d658edc78..37e58f263 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -1,7 +1,6 @@ package http import ( - "fmt" "net/http" "testing" "time" From d61c7578c14dd2e237d783881793b4f781362021 Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sat, 31 Jul 2021 23:59:32 +0200 Subject: [PATCH 17/28] make tidy --- v2/go.mod | 7 +------ v2/go.sum | 12 +++--------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3f7498d49..79f166332 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -4,7 +4,6 @@ go 1.15 require ( github.com/Knetic/govaluate v3.0.0+incompatible - github.com/alecthomas/participle v0.3.0 // indirect github.com/andygrunwald/go-jira v1.13.0 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible @@ -18,20 +17,16 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect - github.com/hokaccha/go-prettyjson v0.0.0-20180920040306-f579f869bbfe // indirect - github.com/itchyny/astgen-go v0.0.0-20190623123749-11f6ca3e2b1f // indirect - github.com/itchyny/gojq v0.12.4 // indirect + github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 - github.com/lestrrat-go/strftime v0.0.0-20190725011945-5c849dd2c51d // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mattn/go-runewidth v0.0.10 // indirect github.com/miekg/dns v1.1.38 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.4 - github.com/pbnjay/strptime v0.0.0-20140226051138-5c05b0d668c9 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.8 diff --git a/v2/go.sum b/v2/go.sum index 8e3438892..789b0b6ac 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,7 +40,6 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/participle v0.3.0/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -137,8 +136,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -175,13 +174,11 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hokaccha/go-prettyjson v0.0.0-20180920040306-f579f869bbfe/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/itchyny/astgen-go v0.0.0-20190623123749-11f6ca3e2b1f/go.mod h1:9Gyr9nZoENI+woes+xm+BFhmvYmAp6bPtXD866pQH9g= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= @@ -221,16 +218,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/strftime v0.0.0-20190725011945-5c849dd2c51d/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= @@ -260,7 +255,6 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= -github.com/pbnjay/strptime v0.0.0-20140226051138-5c05b0d668c9/go.mod h1:6Hr+C/olSdkdL3z68MlyXWzwhvwmwN7KuUFXGb3PoOk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -505,8 +499,8 @@ golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 3df9be0fa7cd22b7821346e1d762558c9090b7ae Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sun, 1 Aug 2021 00:02:43 +0200 Subject: [PATCH 18/28] Update extractors.go --- v2/pkg/operators/extractors/extractors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 9e08c05f0..6093b5e4f 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -22,7 +22,7 @@ type Extractor struct { // KVal are the kval to be present in the response headers/cookies KVal []string `yaml:"kval,omitempty"` - // Regex are the regex pattern required to be present in the response + // JSON are the json pattern required to be present in the response Json []string `yaml:"json"` // jsonCompiled is the compiled variant jsonCompiled []*gojq.Code From 25235523dafc5725d55d1a9ddaf6809353d86026 Mon Sep 17 00:00:00 2001 From: Souvik Hazra Date: Sun, 1 Aug 2021 11:00:38 +0530 Subject: [PATCH 19/28] Support for environment variables with .env file --- v2/cmd/nuclei/main.go | 4 ++++ v2/go.mod | 1 + v2/go.sum | 2 ++ v2/internal/runner/options.go | 8 ++++++++ v2/pkg/protocols/http/build_request.go | 14 ++++++++++++++ v2/pkg/protocols/http/http.go | 2 ++ v2/pkg/types/types.go | 2 ++ 7 files changed, 33 insertions(+) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index a3268c840..ebcc37ebb 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -162,6 +162,10 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), ) + createGroup(flagSet, "env", "DotEnv", + flagSet.StringVar(&options.DotEnvPath, "env", ".env", "path of .env file to use"), + ) + _ = flagSet.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 2e1a97acf..8cb745c96 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -17,6 +17,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/joho/godotenv v1.3.0 // indirect github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible diff --git a/v2/go.sum b/v2/go.sum index f8512f4c7..a85d89b28 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -183,6 +183,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f60a3b6a6..16284a3f9 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/joho/godotenv" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -59,6 +60,13 @@ func ParseOptions(options *types.Options) { if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } + + if len(options.DotEnvPath) > 0 { + err := godotenv.Load(options.DotEnvPath) + if err != nil { + gologger.Warning().Msgf("Failed loading .env file: %s\n", err) + } + } } // hasStdin returns true if we have stdin input diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 805f5e0ba..78c3248b7 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/url" + "os" "regexp" "strings" "time" @@ -129,6 +130,19 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest // request values. finalValues := generators.MergeMaps(generatorValues, values) + // read env variables + envVars := strings.Split(strings.Trim(r.request.EnvVars, " "), ",") + if len(envVars) > 0 { + envVarValues := make(map[string]interface{}) + for _, envVar := range envVars { + envVarValue := os.Getenv(envVar) + if len(envVarValue) > 0 { + envVarValues[envVar] = envVarValue + } + } + finalValues = generators.MergeMaps(finalValues, envVarValues) + } + // Evaulate the expressions for raw request if any. var err error rawRequest, err = expressions.Evaluate(rawRequest, finalValues) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index ecd297f8d..2159aff46 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -73,6 +73,8 @@ type Request struct { // their history for being matched at the end. // Currently only works with sequential http requests. ReqCondition bool `yaml:"req-condition"` + // Dynamic ENV Variables + EnvVars string `yaml:"env-vars"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 872b107ee..827c99e87 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -143,4 +143,6 @@ type Options struct { UpdateNuclei bool // NoUpdateTemplates disables checking for nuclei templates updates NoUpdateTemplates bool + // .env file path + DotEnvPath string } From c76504f0d60d052ff90d91758ad75925452c7b60 Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sun, 1 Aug 2021 12:38:13 +0200 Subject: [PATCH 20/28] use import () in extractors.go --- v2/pkg/operators/extractors/extractors.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 6093b5e4f..179dc53ef 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -1,7 +1,9 @@ package extractors -import "regexp" -import "github.com/itchyny/gojq" +import ( + "github.com/itchyny/gojq" + "regexp" +) // Extractor is used to extract part of response using a regex. type Extractor struct { From cc12babf58b074aab59470aba22dd3e7cd4e9759 Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sun, 1 Aug 2021 12:38:35 +0200 Subject: [PATCH 21/28] Update extractors.go --- v2/pkg/operators/extractors/extractors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 179dc53ef..6008c08cb 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -53,7 +53,7 @@ const ( var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, - "json": JSONExtractor, + "json": JSONExtractor, } // GetType returns the type of the matcher From d2e7cca17bb9f075dd0596818910789f36d4899a Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Sun, 1 Aug 2021 14:42:04 +0200 Subject: [PATCH 22/28] fix golint --- v2/pkg/operators/extractors/compile.go | 5 +++-- v2/pkg/operators/extractors/extract.go | 5 +++-- v2/pkg/operators/extractors/extractors.go | 5 +++-- v2/pkg/protocols/http/operators.go | 2 +- v2/pkg/protocols/http/operators_test.go | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 5d1b134c3..341340a38 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -2,9 +2,10 @@ package extractors import ( "fmt" - "github.com/itchyny/gojq" "regexp" "strings" + + "github.com/itchyny/gojq" ) // CompileExtractors performs the initial setup operation on a extractor @@ -29,7 +30,7 @@ func (e *Extractor) CompileExtractors() error { e.KVal[i] = strings.ToLower(kval) } - for _, query := range e.Json { + for _, query := range e.JSON { query, err := gojq.Parse(query) if err != nil { return fmt.Errorf("could not parse json: %s", query) diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 81f995b1a..01321ca37 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -2,6 +2,7 @@ package extractors import ( "encoding/json" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -44,8 +45,8 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} return results } -// ExtractJson extracts key value pairs from a data map -func (e *Extractor) ExtractJson(corpus string) map[string]struct{} { +// ExtractJSON extracts key value pairs from a data map +func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { results := make(map[string]struct{}) var jsonObj interface{} diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 6008c08cb..2542bb0d4 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -1,8 +1,9 @@ package extractors import ( - "github.com/itchyny/gojq" "regexp" + + "github.com/itchyny/gojq" ) // Extractor is used to extract part of response using a regex. @@ -25,7 +26,7 @@ type Extractor struct { KVal []string `yaml:"kval,omitempty"` // JSON are the json pattern required to be present in the response - Json []string `yaml:"json"` + JSON []string `yaml:"json"` // jsonCompiled is the compiled variant jsonCompiled []*gojq.Code diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 57307c619..ba6f9a6cd 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -55,7 +55,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.JSONExtractor: - return extractor.ExtractJson(item) + return extractor.ExtractJSON(item) } return nil } diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 37e58f263..851a36203 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -172,7 +172,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("json", func(t *testing.T) { extractor := &extractors.Extractor{ Type: "json", - Json: []string{".batters | .batter | .[] | .id"}, + JSON: []string{".batters | .batter | .[] | .id"}, } err = extractor.CompileExtractors() require.Nil(t, err, "could not compile json extractor") From 250b87aa01b804383e4b4f39e68fc9a9080918c3 Mon Sep 17 00:00:00 2001 From: Souvik Hazra Date: Mon, 2 Aug 2021 11:37:37 +0530 Subject: [PATCH 23/28] Revert "Support for environment variables with .env file" This reverts commit 25235523dafc5725d55d1a9ddaf6809353d86026. --- v2/cmd/nuclei/main.go | 4 ---- v2/go.mod | 1 - v2/go.sum | 2 -- v2/internal/runner/options.go | 8 -------- v2/pkg/protocols/http/build_request.go | 14 -------------- v2/pkg/protocols/http/http.go | 2 -- v2/pkg/types/types.go | 2 -- 7 files changed, 33 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index ebcc37ebb..a3268c840 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -162,10 +162,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), ) - createGroup(flagSet, "env", "DotEnv", - flagSet.StringVar(&options.DotEnvPath, "env", ".env", "path of .env file to use"), - ) - _ = flagSet.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 8cb745c96..2e1a97acf 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -17,7 +17,6 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect - github.com/joho/godotenv v1.3.0 // indirect github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible diff --git a/v2/go.sum b/v2/go.sum index a85d89b28..f8512f4c7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -183,8 +183,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 16284a3f9..f60a3b6a6 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -7,7 +7,6 @@ import ( "os" "strings" - "github.com/joho/godotenv" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -60,13 +59,6 @@ func ParseOptions(options *types.Options) { if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } - - if len(options.DotEnvPath) > 0 { - err := godotenv.Load(options.DotEnvPath) - if err != nil { - gologger.Warning().Msgf("Failed loading .env file: %s\n", err) - } - } } // hasStdin returns true if we have stdin input diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 78c3248b7..805f5e0ba 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "net/url" - "os" "regexp" "strings" "time" @@ -130,19 +129,6 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest // request values. finalValues := generators.MergeMaps(generatorValues, values) - // read env variables - envVars := strings.Split(strings.Trim(r.request.EnvVars, " "), ",") - if len(envVars) > 0 { - envVarValues := make(map[string]interface{}) - for _, envVar := range envVars { - envVarValue := os.Getenv(envVar) - if len(envVarValue) > 0 { - envVarValues[envVar] = envVarValue - } - } - finalValues = generators.MergeMaps(finalValues, envVarValues) - } - // Evaulate the expressions for raw request if any. var err error rawRequest, err = expressions.Evaluate(rawRequest, finalValues) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 2159aff46..ecd297f8d 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -73,8 +73,6 @@ type Request struct { // their history for being matched at the end. // Currently only works with sequential http requests. ReqCondition bool `yaml:"req-condition"` - // Dynamic ENV Variables - EnvVars string `yaml:"env-vars"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 827c99e87..872b107ee 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -143,6 +143,4 @@ type Options struct { UpdateNuclei bool // NoUpdateTemplates disables checking for nuclei templates updates NoUpdateTemplates bool - // .env file path - DotEnvPath string } From 2c8730690b8e0dd90cc7cd87d39564a11f75b2c8 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 2 Aug 2021 11:40:44 +0530 Subject: [PATCH 24/28] WIP updates --- v2/cmd/functional-test/run.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 35262957d..cf5855e1b 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -1,12 +1,15 @@ #!/bin/bash -git checkout master -cd ../nuclei/ +currentBranch=$(git branch --show-current) + +echo 'Building functional-test binary' go build -cp nuclei ../functional-test/nuclei_main -git checkout dev -go build -cp nuclei ../functional-test/nuclei_dev -cd ../functional-test -go build -./functional-test -main ./nuclei_main -dev ./nuclei_dev -testcases testcases.txt + +echo 'Building Nuclei binary from' $currentBranch 'branch' +go build -o nuclei_$currentBranch ../nuclei + +echo 'Installing latest release of nuclei' +GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei + +echo 'Starting Nuclei functional test' +./functional-test -main $GOPATH/bin/nuclei -dev ./nuclei_$currentBranch -testcases testcases.txt \ No newline at end of file From ebd165dd22ecebf6b833d47d489ac7eb0ae041d3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 2 Aug 2021 11:45:15 +0530 Subject: [PATCH 25/28] Adding test workflow --- .github/workflows/functional-test.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/functional-test.yml diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 000000000..f55cadb84 --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,22 @@ +name: 🧪 Functional Test +on: + push: + pull_request: + workflow_dispatch: + + +jobs: + build: + name: Functional Test + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Functional Tests + run: | + cd "${GITHUB_WORKSPACE}/v2/cmd/functional-test/" + chmod +x run.sh + bash run.sh \ No newline at end of file From 637713cb68663207ea01f9b8721b6f734902f6c3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 2 Aug 2021 12:02:40 +0530 Subject: [PATCH 26/28] few updates --- .github/workflows/functional-test.yml | 7 +++++-- v2/cmd/functional-test/run.sh | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index f55cadb84..de2a43c93 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -15,8 +15,11 @@ jobs: with: go-version: 1.15 + - name: Check out code + uses: actions/checkout@v2 + - name: Functional Tests run: | - cd "${GITHUB_WORKSPACE}/v2/cmd/functional-test/" chmod +x run.sh - bash run.sh \ No newline at end of file + bash run.sh + working-directory: v2/cmd/functional-test \ No newline at end of file diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index cf5855e1b..43eaf7f68 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -12,4 +12,4 @@ echo 'Installing latest release of nuclei' GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo 'Starting Nuclei functional test' -./functional-test -main $GOPATH/bin/nuclei -dev ./nuclei_$currentBranch -testcases testcases.txt \ No newline at end of file +./functional-test -main nuclei -dev ./nuclei_$currentBranch -testcases testcases.txt \ No newline at end of file From d0251192be0811d5508fe436443ab47a93b2bc7a Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 2 Aug 2021 12:12:22 +0530 Subject: [PATCH 27/28] minor update --- v2/cmd/functional-test/run.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 43eaf7f68..030b25443 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -1,15 +1,13 @@ #!/bin/bash -currentBranch=$(git branch --show-current) - echo 'Building functional-test binary' go build -echo 'Building Nuclei binary from' $currentBranch 'branch' -go build -o nuclei_$currentBranch ../nuclei +echo 'Building Nuclei binary from current branch' +go build -o nuclei_dev ../nuclei echo 'Installing latest release of nuclei' GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo 'Starting Nuclei functional test' -./functional-test -main nuclei -dev ./nuclei_$currentBranch -testcases testcases.txt \ No newline at end of file +./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file From aae26f37a6e8722c9b1a9e26e1a92960004fe276 Mon Sep 17 00:00:00 2001 From: seb <45340011+KaulSe@users.noreply.github.com> Date: Mon, 2 Aug 2021 13:19:39 +0200 Subject: [PATCH 28/28] add JSONScalarToString and more tests for JSON extractor --- v2/pkg/operators/extractors/extract.go | 15 +++++--- v2/pkg/protocols/http/operators_test.go | 48 +++++++++++++++++++------ v2/pkg/types/interfaces.go | 17 +++++++++ 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 01321ca37..58845a833 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -45,7 +45,7 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} return results } -// ExtractJSON extracts key value pairs from a data map +// ExtractJSON extracts text from a corpus using JQ queries and returns it func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { results := make(map[string]struct{}) @@ -67,9 +67,16 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { if _, ok := v.(error); ok { break } - itemString := types.ToString(v) - if _, ok := results[itemString]; !ok { - results[itemString] = struct{}{} + var result string + if res, err := types.JSONScalarToString(v); err == nil { + result = res + } else if res, err := json.Marshal(v); err == nil { + result = string(res) + } else { + result = types.ToString(v) + } + if _, ok := results[result]; !ok { + results[result] = struct{}{} } } } diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 851a36203..7328fce86 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -170,17 +170,44 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("json", func(t *testing.T) { - extractor := &extractors.Extractor{ - Type: "json", - JSON: []string{".batters | .batter | .[] | .id"}, - } - err = extractor.CompileExtractors() - require.Nil(t, err, "could not compile json extractor") - event["body"] = exampleJSONResponseBody - data := request.Extract(event, extractor) - require.Greater(t, len(data), 0, "could not extractor json valid response") - require.Equal(t, map[string]struct{}{"1001": {}, "1002": {}, "1003": {}, "1004": {}}, data, "could not extract correct json data") + + t.Run("jq-simple", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + JSON: []string{".batters | .batter | .[] | .id"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"1001": {}, "1002": {}, "1003": {}, "1004": {}}, data, "could not extract correct json data") + }) + t.Run("jq-array", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + JSON: []string{".array"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"[\"hello\",\"world\"]": {}}, data, "could not extract correct json data") + }) + t.Run("jq-object", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + JSON: []string{".batters"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]}": {}}, data, "could not extract correct json data") + }) }) } @@ -326,6 +353,7 @@ const exampleJSONResponseBody = ` "type": "donut", "name": "Cake", "ppu": 0.55, + "array": ["hello", "world"], "batters": { "batter": [ { diff --git a/v2/pkg/types/interfaces.go b/v2/pkg/types/interfaces.go index 6b0b40f63..4461ca6ea 100644 --- a/v2/pkg/types/interfaces.go +++ b/v2/pkg/types/interfaces.go @@ -8,6 +8,23 @@ import ( "strings" ) +// JSONScalarToString converts an interface coming from json to string +// Inspired from: https://github.com/cli/cli/blob/09b09810dd812e3ede54b59ad9d6912b946ac6c5/pkg/export/template.go#L72 +func JSONScalarToString(input interface{}) (string, error) { + switch tt := input.(type) { + case string: + return ToString(tt), nil + case float64: + return ToString(tt), nil + case nil: + return ToString(tt), nil + case bool: + return ToString(tt), nil + default: + return "", fmt.Errorf("cannot convert type to string: %v", tt) + } +} + // ToString converts an interface to string in a quick way func ToString(data interface{}) string { switch s := data.(type) {