mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-18 18:05:26 +00:00
Merged from dev
This commit is contained in:
Ice3man543
commit
c743d5d299
25
.github/workflows/functional-test.yml
vendored
Normal file
25
.github/workflows/functional-test.yml
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
name: 🧪 Functional Test
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Functional Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.15
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Functional Tests
|
||||
run: |
|
||||
chmod +x run.sh
|
||||
bash run.sh
|
||||
working-directory: v2/cmd/functional-test
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -7,4 +7,7 @@ v2/cmd/integration-test/integration-test
|
||||
bin
|
||||
v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
|
||||
v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
|
||||
v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
|
||||
v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
|
||||
v2/cmd/functional-test/nuclei_dev
|
||||
v2/cmd/functional-test/nuclei_main
|
||||
v2/cmd/functional-test/functional-test
|
||||
@ -164,7 +164,7 @@ UPDATE:
|
||||
|
||||
STATISTICS:
|
||||
-stats display statistics about the running scan
|
||||
-stats-json write statistics data to and output file in JSONL(ines) format
|
||||
-stats-json write statistics data to an output file in JSONL(ines) format
|
||||
-si, -stats-interval int number of seconds to wait between showing a statistics update (default 5)
|
||||
-metrics expose nuclei metrics on a port
|
||||
-metrics-port int port to expose nuclei metrics on (default 9092)
|
||||
|
||||
79
v2/cmd/functional-test/main.go
Normal file
79
v2/cmd/functional-test/main.go
Normal file
@ -0,0 +1,79 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
|
||||
)
|
||||
|
||||
var (
|
||||
success = aurora.Green("[✓]").String()
|
||||
failed = aurora.Red("[✘]").String()
|
||||
errored = false
|
||||
|
||||
mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
|
||||
devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary")
|
||||
testcases = flag.String("testcases", "", "Test cases file for nuclei functional tests")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if err := runFunctionalTests(); err != nil {
|
||||
log.Fatalf("Could not run functional tests: %s\n", err)
|
||||
}
|
||||
if errored {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runFunctionalTests() error {
|
||||
file, err := os.Open(*testcases)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not open test cases")
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
text := strings.TrimSpace(scanner.Text())
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
if err := runIndividualTestCase(text); err != nil {
|
||||
errored = true
|
||||
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
|
||||
} else {
|
||||
fmt.Printf("%s Test \"%s\" passed!\n", success, text)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runIndividualTestCase(testcase string) error {
|
||||
parts := strings.Fields(testcase)
|
||||
|
||||
var finalArgs []string
|
||||
if len(parts) > 1 {
|
||||
finalArgs = parts[1:]
|
||||
}
|
||||
mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not run nuclei main test")
|
||||
}
|
||||
devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not run nuclei dev test")
|
||||
}
|
||||
if mainOutput == devOutput {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
|
||||
}
|
||||
13
v2/cmd/functional-test/run.sh
Normal file
13
v2/cmd/functional-test/run.sh
Normal file
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo 'Building functional-test binary'
|
||||
go build
|
||||
|
||||
echo 'Building Nuclei binary from current branch'
|
||||
go build -o nuclei_dev ../nuclei
|
||||
|
||||
echo 'Installing latest release of nuclei'
|
||||
GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
|
||||
|
||||
echo 'Starting Nuclei functional test'
|
||||
./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt
|
||||
51
v2/cmd/functional-test/testcases.txt
Normal file
51
v2/cmd/functional-test/testcases.txt
Normal file
@ -0,0 +1,51 @@
|
||||
{{binary}}
|
||||
{{binary}} -tags cve
|
||||
{{binary}} -tags cve,exposure
|
||||
{{binary}} -tags cve,exposure -tags token
|
||||
{{binary}} -tags cve,exposure -tags token,logs
|
||||
{{binary}} -tags "cve","exposure" -tags "token","logs"
|
||||
{{binary}} -tags 'cve','exposure' -tags 'token','logs'
|
||||
{{binary}} -tags cve -severity high
|
||||
{{binary}} -tags cve,exposure -severity high,critical
|
||||
{{binary}} -tags cve,exposure -severity "high,critical,medium"
|
||||
{{binary}} -tags cve -author geeknik
|
||||
{{binary}} -tags cve -author geeknik,pdteam
|
||||
{{binary}} -tags cve -author geeknik -severity high
|
||||
{{binary}} -tags cve
|
||||
{{binary}} -tags cve,exposure
|
||||
{{binary}} -tags cve,exposure -tags token
|
||||
{{binary}} -tags cve,exposure -tags token,logs
|
||||
{{binary}} -tags "cve","exposure" -tags "token","logs"
|
||||
{{binary}} -tags 'cve','exposure' -tags 'token','logs'
|
||||
{{binary}} -tags cve -severity high
|
||||
{{binary}} -tags cve,exposure -severity high,critical
|
||||
{{binary}} -tags cve,exposure -severity "high,critical,medium"
|
||||
{{binary}} -tags cve -author geeknik
|
||||
{{binary}} -tags cve -author geeknik,pdteam
|
||||
{{binary}} -tags cve -author geeknik -severity high
|
||||
{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
|
||||
{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity "high,critical"
|
||||
{{binary}} -tags cve -etags ssrf
|
||||
{{binary}} -tags cve,exposure -etags ssrf,config
|
||||
{{binary}} -tags cve,exposure -etags ssrf,config -severity high
|
||||
{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
|
||||
{{binary}} -tags cve,dos,fuzz
|
||||
{{binary}} -tags cve -include-tags dos,fuzz
|
||||
{{binary}} -tags cve -exclude-tags cve2020
|
||||
{{binary}} -tags cve -exclude-templates cves/2020/
|
||||
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
|
||||
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
|
||||
{{binary}} -t cves/
|
||||
{{binary}} -t cves/ -t exposures/
|
||||
{{binary}} -t cves/ -t exposures/ -tags config
|
||||
{{binary}} -t cves/ -t exposures/ -tags config,ssrf
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
|
||||
{{binary}} -w workflows
|
||||
{{binary}} -w workflows -author geeknik,pdteam
|
||||
{{binary}} -w workflows -severity high,critical
|
||||
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical
|
||||
@ -13,6 +13,8 @@ var (
|
||||
debug = os.Getenv("DEBUG") == "true"
|
||||
customTest = os.Getenv("TEST")
|
||||
protocol = os.Getenv("PROTO")
|
||||
|
||||
errored = false
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -36,13 +38,16 @@ func main() {
|
||||
err := test.Execute(file)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
|
||||
os.Exit(1)
|
||||
errored = true
|
||||
} else {
|
||||
fmt.Printf("%s Test \"%s\" passed!\n", success, file)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if errored {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func errIncorrectResultsCount(results []string) error {
|
||||
|
||||
@ -57,15 +57,15 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
)
|
||||
|
||||
createGroup(flagSet, "filters", "Filtering",
|
||||
flagSet.StringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
|
||||
flagSet.StringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
|
||||
flagSet.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
|
||||
flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
|
||||
flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
|
||||
flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
|
||||
|
||||
flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
|
||||
flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
|
||||
|
||||
flagSet.StringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"),
|
||||
flagSet.StringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
|
||||
flagSet.NormalizedStringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"),
|
||||
flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "output", "Output",
|
||||
@ -109,6 +109,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
|
||||
createGroup(flagSet, "rate-limit", "Rate-Limit",
|
||||
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
|
||||
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
|
||||
flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
|
||||
flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
|
||||
)
|
||||
@ -148,12 +149,13 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
createGroup(flagSet, "update", "Update",
|
||||
flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
|
||||
flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
|
||||
flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"),
|
||||
flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "stats", "Statistics",
|
||||
flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
|
||||
flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to and output file in JSONL(ines) format"),
|
||||
flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
|
||||
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
|
||||
|
||||
flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
|
||||
|
||||
@ -17,6 +17,7 @@ require (
|
||||
github.com/gosuri/uiprogress v0.0.1 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
|
||||
github.com/itchyny/gojq v0.12.4
|
||||
github.com/json-iterator/go v1.1.10
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/karlseguin/ccache v2.0.3+incompatible
|
||||
@ -29,7 +30,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/clistats v0.0.8
|
||||
github.com/projectdiscovery/fastdialer v0.0.8
|
||||
github.com/projectdiscovery/goflags v0.0.5
|
||||
github.com/projectdiscovery/goflags v0.0.6
|
||||
github.com/projectdiscovery/gologger v1.1.4
|
||||
github.com/projectdiscovery/hmap v0.0.1
|
||||
github.com/projectdiscovery/interactsh v0.0.3
|
||||
@ -52,7 +53,7 @@ require (
|
||||
github.com/xanzy/go-gitlab v0.44.0
|
||||
go.uber.org/atomic v1.7.0
|
||||
go.uber.org/multierr v1.6.0
|
||||
go.uber.org/ratelimit v0.1.0
|
||||
go.uber.org/ratelimit v0.2.0
|
||||
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
|
||||
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
|
||||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
|
||||
|
||||
23
v2/go.sum
23
v2/go.sum
@ -40,6 +40,8 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
|
||||
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
|
||||
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
|
||||
github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
|
||||
github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
|
||||
github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
|
||||
@ -136,8 +138,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
@ -178,6 +181,11 @@ github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKgh
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
|
||||
github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
|
||||
github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
|
||||
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
|
||||
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
|
||||
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
|
||||
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
@ -218,8 +226,9 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
|
||||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
|
||||
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
@ -258,8 +267,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS
|
||||
github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
|
||||
github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
|
||||
github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
|
||||
github.com/projectdiscovery/goflags v0.0.5 h1:jI6HD9Z7vkg4C4Cz16BfZKICnIf94W3KFU5M3DcUgUk=
|
||||
github.com/projectdiscovery/goflags v0.0.5/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
|
||||
github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
|
||||
github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
|
||||
github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
|
||||
github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
|
||||
github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
|
||||
@ -366,6 +375,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
|
||||
go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
|
||||
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
|
||||
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@ -474,6 +485,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -491,8 +503,9 @@ golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk=
|
||||
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
||||
@ -236,7 +236,9 @@ func New(options *types.Options) (*Runner, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if options.RateLimit > 0 {
|
||||
if options.RateLimitMinute > 0 {
|
||||
runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second))
|
||||
} else if options.RateLimit > 0 {
|
||||
runner.ratelimiter = ratelimit.New(options.RateLimit)
|
||||
} else {
|
||||
runner.ratelimiter = ratelimit.NewUnlimited()
|
||||
|
||||
@ -82,6 +82,9 @@ func (r *Runner) updateTemplates() error {
|
||||
r.templatesConfig = currentConfig
|
||||
}
|
||||
|
||||
if r.options.NoUpdateTemplates {
|
||||
return nil
|
||||
}
|
||||
// Check if last checked for nuclei-ignore is more than 1 hours.
|
||||
// and if true, run the check.
|
||||
//
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
package testutils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@ -30,6 +32,23 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: (\d+)`)
|
||||
|
||||
// RunNucleiAndGetResults returns a list of results for a template
|
||||
func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) {
|
||||
cmd := exec.Command(nucleiBinary, args...)
|
||||
|
||||
data, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
matches := templateLoaded.FindAllStringSubmatch(string(data), -1)
|
||||
if len(matches) == 0 {
|
||||
return "", errors.New("no matches found")
|
||||
}
|
||||
return matches[0][1], nil
|
||||
}
|
||||
|
||||
// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
|
||||
func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
|
||||
cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
|
||||
|
||||
@ -28,7 +28,7 @@ type Config struct {
|
||||
const nucleiConfigFilename = ".templates-config.json"
|
||||
|
||||
// Version is the current version of nuclei
|
||||
const Version = `2.4.1`
|
||||
const Version = `2.4.3-dev`
|
||||
|
||||
func getConfigDetails() (string, error) {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
|
||||
@ -4,6 +4,8 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/itchyny/gojq"
|
||||
)
|
||||
|
||||
// CompileExtractors performs the initial setup operation on a extractor
|
||||
@ -28,6 +30,18 @@ func (e *Extractor) CompileExtractors() error {
|
||||
e.KVal[i] = strings.ToLower(kval)
|
||||
}
|
||||
|
||||
for _, query := range e.JSON {
|
||||
query, err := gojq.Parse(query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse json: %s", query)
|
||||
}
|
||||
compiled, err := gojq.Compile(query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not compile json: %s", query)
|
||||
}
|
||||
e.jsonCompiled = append(e.jsonCompiled, compiled)
|
||||
}
|
||||
|
||||
// Setup the part of the request to match, if any.
|
||||
if e.Part == "" {
|
||||
e.Part = "body"
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
package extractors
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
)
|
||||
|
||||
@ -42,3 +44,41 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// ExtractJSON extracts text from a corpus using JQ queries and returns it
|
||||
func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
|
||||
results := make(map[string]struct{})
|
||||
|
||||
var jsonObj interface{}
|
||||
|
||||
err := json.Unmarshal([]byte(corpus), &jsonObj)
|
||||
|
||||
if err != nil {
|
||||
return results
|
||||
}
|
||||
|
||||
for _, k := range e.jsonCompiled {
|
||||
iter := k.Run(jsonObj)
|
||||
for {
|
||||
v, ok := iter.Next()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
if _, ok := v.(error); ok {
|
||||
break
|
||||
}
|
||||
var result string
|
||||
if res, err := types.JSONScalarToString(v); err == nil {
|
||||
result = res
|
||||
} else if res, err := json.Marshal(v); err == nil {
|
||||
result = string(res)
|
||||
} else {
|
||||
result = types.ToString(v)
|
||||
}
|
||||
if _, ok := results[result]; !ok {
|
||||
results[result] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
package extractors
|
||||
|
||||
import "regexp"
|
||||
import (
|
||||
"regexp"
|
||||
|
||||
"github.com/itchyny/gojq"
|
||||
)
|
||||
|
||||
// Extractor is used to extract part of response using a regex.
|
||||
type Extractor struct {
|
||||
@ -65,6 +69,17 @@ type Extractor struct {
|
||||
// - value: "\"body\""
|
||||
// - value: "\"raw\""
|
||||
Part string `yaml:"part,omitempty"`
|
||||
|
||||
// description: |
|
||||
// JSON allows using jq-style syntax to extract items from json response
|
||||
//
|
||||
// examples:
|
||||
// - value: "\".[] | .id\""
|
||||
// - value: "\".batters | .batter | .[] | .id\""
|
||||
JSON []string `yaml:"json"`
|
||||
// jsonCompiled is the compiled variant
|
||||
jsonCompiled []*gojq.Code
|
||||
|
||||
// description: |
|
||||
// Internal, when set to true will allow using the value extracted
|
||||
// in the next request for some protocols (like HTTP).
|
||||
@ -79,12 +94,15 @@ const (
|
||||
RegexExtractor ExtractorType = iota + 1
|
||||
// KValExtractor extracts responses with key:value
|
||||
KValExtractor
|
||||
// JSONExtractor extracts responses with json
|
||||
JSONExtractor
|
||||
)
|
||||
|
||||
// ExtractorTypes is an table for conversion of extractor type from string.
|
||||
var ExtractorTypes = map[string]ExtractorType{
|
||||
"regex": RegexExtractor,
|
||||
"kval": KValExtractor,
|
||||
"json": JSONExtractor,
|
||||
}
|
||||
|
||||
// GetType returns the type of the matcher
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import java.io.*;
|
||||
|
||||
class Deserialize{
|
||||
class Deserialize {
|
||||
public static void main(String args[]) {
|
||||
FileInputStream fileIn = null;
|
||||
ObjectInputStream in = null;
|
||||
|
||||
@ -54,6 +54,8 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
|
||||
return extractor.ExtractRegex(item)
|
||||
case extractors.KValExtractor:
|
||||
return extractor.ExtractKval(data)
|
||||
case extractors.JSONExtractor:
|
||||
return extractor.ExtractJSON(item)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -168,6 +168,47 @@ func TestHTTPOperatorExtract(t *testing.T) {
|
||||
require.Greater(t, len(data), 0, "could not extractor kval valid response")
|
||||
require.Equal(t, map[string]struct{}{"Test-Response": {}}, data, "could not extract correct kval data")
|
||||
})
|
||||
|
||||
t.Run("json", func(t *testing.T) {
|
||||
event["body"] = exampleJSONResponseBody
|
||||
|
||||
t.Run("jq-simple", func(t *testing.T) {
|
||||
extractor := &extractors.Extractor{
|
||||
Type: "json",
|
||||
JSON: []string{".batters | .batter | .[] | .id"},
|
||||
}
|
||||
err = extractor.CompileExtractors()
|
||||
require.Nil(t, err, "could not compile json extractor")
|
||||
|
||||
data := request.Extract(event, extractor)
|
||||
require.Greater(t, len(data), 0, "could not extractor json valid response")
|
||||
require.Equal(t, map[string]struct{}{"1001": {}, "1002": {}, "1003": {}, "1004": {}}, data, "could not extract correct json data")
|
||||
})
|
||||
t.Run("jq-array", func(t *testing.T) {
|
||||
extractor := &extractors.Extractor{
|
||||
Type: "json",
|
||||
JSON: []string{".array"},
|
||||
}
|
||||
err = extractor.CompileExtractors()
|
||||
require.Nil(t, err, "could not compile json extractor")
|
||||
|
||||
data := request.Extract(event, extractor)
|
||||
require.Greater(t, len(data), 0, "could not extractor json valid response")
|
||||
require.Equal(t, map[string]struct{}{"[\"hello\",\"world\"]": {}}, data, "could not extract correct json data")
|
||||
})
|
||||
t.Run("jq-object", func(t *testing.T) {
|
||||
extractor := &extractors.Extractor{
|
||||
Type: "json",
|
||||
JSON: []string{".batters"},
|
||||
}
|
||||
err = extractor.CompileExtractors()
|
||||
require.Nil(t, err, "could not compile json extractor")
|
||||
|
||||
data := request.Extract(event, extractor)
|
||||
require.Greater(t, len(data), 0, "could not extractor json valid response")
|
||||
require.Equal(t, map[string]struct{}{"{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]}": {}}, data, "could not extract correct json data")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestHTTPMakeResult(t *testing.T) {
|
||||
@ -305,3 +346,63 @@ const exampleResponseBody = `
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
|
||||
const exampleJSONResponseBody = `
|
||||
{
|
||||
"id": "0001",
|
||||
"type": "donut",
|
||||
"name": "Cake",
|
||||
"ppu": 0.55,
|
||||
"array": ["hello", "world"],
|
||||
"batters": {
|
||||
"batter": [
|
||||
{
|
||||
"id": "1001",
|
||||
"type": "Regular"
|
||||
},
|
||||
{
|
||||
"id": "1002",
|
||||
"type": "Chocolate"
|
||||
},
|
||||
{
|
||||
"id": "1003",
|
||||
"type": "Blueberry"
|
||||
},
|
||||
{
|
||||
"id": "1004",
|
||||
"type": "Devil's Food"
|
||||
}
|
||||
]
|
||||
},
|
||||
"topping": [
|
||||
{
|
||||
"id": "5001",
|
||||
"type": "None"
|
||||
},
|
||||
{
|
||||
"id": "5002",
|
||||
"type": "Glazed"
|
||||
},
|
||||
{
|
||||
"id": "5005",
|
||||
"type": "Sugar"
|
||||
},
|
||||
{
|
||||
"id": "5007",
|
||||
"type": "Powdered Sugar"
|
||||
},
|
||||
{
|
||||
"id": "5006",
|
||||
"type": "Chocolate with Sprinkles"
|
||||
},
|
||||
{
|
||||
"id": "5003",
|
||||
"type": "Chocolate"
|
||||
},
|
||||
{
|
||||
"id": "5004",
|
||||
"type": "Maple"
|
||||
}
|
||||
]
|
||||
}
|
||||
`
|
||||
|
||||
@ -8,6 +8,23 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// JSONScalarToString converts an interface coming from json to string
|
||||
// Inspired from: https://github.com/cli/cli/blob/09b09810dd812e3ede54b59ad9d6912b946ac6c5/pkg/export/template.go#L72
|
||||
func JSONScalarToString(input interface{}) (string, error) {
|
||||
switch tt := input.(type) {
|
||||
case string:
|
||||
return ToString(tt), nil
|
||||
case float64:
|
||||
return ToString(tt), nil
|
||||
case nil:
|
||||
return ToString(tt), nil
|
||||
case bool:
|
||||
return ToString(tt), nil
|
||||
default:
|
||||
return "", fmt.Errorf("cannot convert type to string: %v", tt)
|
||||
}
|
||||
}
|
||||
|
||||
// ToString converts an interface to string in a quick way
|
||||
func ToString(data interface{}) string {
|
||||
switch s := data.(type) {
|
||||
|
||||
@ -7,9 +7,9 @@ type Options struct {
|
||||
// Tags contains a list of tags to execute templates for. Multiple paths
|
||||
// can be specified with -l flag and -tags can be used in combination with
|
||||
// the -l flag.
|
||||
Tags goflags.StringSlice
|
||||
Tags goflags.NormalizedStringSlice
|
||||
// ExcludeTags is the list of tags to exclude
|
||||
ExcludeTags goflags.StringSlice
|
||||
ExcludeTags goflags.NormalizedStringSlice
|
||||
// Workflows specifies any workflows to run by nuclei
|
||||
Workflows goflags.StringSlice
|
||||
// Templates specifies the template/templates to use
|
||||
@ -19,11 +19,11 @@ type Options struct {
|
||||
// CustomHeaders is the list of custom global headers to send with each request.
|
||||
CustomHeaders goflags.StringSlice
|
||||
// Severity filters templates based on their severity and only run the matching ones.
|
||||
Severity goflags.StringSlice
|
||||
Severity goflags.NormalizedStringSlice
|
||||
// Author filters templates based on their author and only run the matching ones.
|
||||
Author goflags.StringSlice
|
||||
Author goflags.NormalizedStringSlice
|
||||
// IncludeTags includes specified tags to be run even while being in denylist
|
||||
IncludeTags goflags.StringSlice
|
||||
IncludeTags goflags.NormalizedStringSlice
|
||||
// IncludeTemplates includes specified templates to be run even while being in denylist
|
||||
IncludeTemplates goflags.StringSlice
|
||||
|
||||
@ -70,6 +70,8 @@ type Options struct {
|
||||
Retries int
|
||||
// Rate-Limit is the maximum number of requests per specified target
|
||||
RateLimit int
|
||||
// Rate-Limit is the maximum number of requests per minute for specified target
|
||||
RateLimitMinute int
|
||||
// PageTimeout is the maximum time to wait for a page in seconds
|
||||
PageTimeout int
|
||||
// InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time.
|
||||
@ -139,4 +141,6 @@ type Options struct {
|
||||
NoInteractsh bool
|
||||
// UpdateNuclei checks for an update for the nuclei engine
|
||||
UpdateNuclei bool
|
||||
// NoUpdateTemplates disables checking for nuclei templates updates
|
||||
NoUpdateTemplates bool
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user