From b6002a5e89576a3b128e62b30b7b3e5b4fe05181 Mon Sep 17 00:00:00 2001 From: Thomas Hendrickson Date: Mon, 19 Jun 2023 09:53:24 -0400 Subject: [PATCH 01/62] set read and write deadline --- v2/pkg/protocols/network/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index bdc6e474c..d0f144a04 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -124,7 +124,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac return errors.Wrap(err, "could not connect to server") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) + _ = conn.SetDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) var interactshURLs []string From 238ed0e9aabee2c51c695d26c9ce6cfa5db758a8 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 27 Jun 2023 01:04:33 +0530 Subject: [PATCH 02/62] added release template --- .github/release.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/release.yml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..01b4c9d33 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,17 @@ +changelog: + exclude: + authors: + - dependabot + categories: + - title: 🎉 Features + labels: + - "Type: Enhancement" + - title: 🐞 Bugs + labels: + - "Type: Bug" + - title: 🔨 Maintenance + labels: + - "Type: Maintenance" + - title: Other Changes + labels: + - "*" \ No newline at end of file From 6707bc777a92bfa6c2a5937a3cbc5a9b07ba7b74 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Fri, 30 Jun 2023 23:32:00 +0530 Subject: [PATCH 03/62] fix showing multiple failure matches per template on -ms set (#3770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix showing multiple failure matchers per template add integration test * exclude AS134029 from unit test * Add flag for match status per request * chore(deps): bump golangci/golangci-lint-action from 3.4.0 to 3.5.0 (#3777) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/xanzy/go-gitlab in /v2 (#3778) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.83.0 to 0.84.0. - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.83.0...v0.84.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/spf13/cast from 1.5.0 to 1.5.1 in /v2 (#3780) Bumps [github.com/spf13/cast](https://github.com/spf13/cast) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/spf13/cast/releases) - [Commits](https://github.com/spf13/cast/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: github.com/spf13/cast dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * enable no-httpx when passive scan is launched (#3789) * chore(deps): bump github.com/projectdiscovery/fastdialer from 0.0.26 to 0.0.28 in /v2 (#3779) * chore(deps): bump github.com/projectdiscovery/fastdialer in /v2 Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.26 to 0.0.28. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.26...v0.0.28) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump retryabledns to 0.28 * Update the retryabledns --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: shubhamrasal * deprecatedProtocolNameTemplates concurrent map writes (#3785) * deprecatedProtocolNameTemplates * use syncLock * fix lint error * change version in deprecated warning msg * comment asnmap expand unit test --------- Co-authored-by: Tarun Koyalwar Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> * Issue 3339 headless fuzz (#3790) * Basic headless fuzzing * Remove debug statements * Add integration tests * Update template * Fix recognize payload value in matcher * Update tempalte * use req.SetURL() --------- Co-authored-by: Tarun Koyalwar * Auto Generate Syntax Docs + JSONSchema [Fri Jun 9 00:23:32 UTC 2023] :robot: * Add headless header and status matchers (#3794) * add headless header and status matchers * rename headers as header * add integration test for header+status * fix typo * chore(deps): bump golang from 1.20.4-alpine to 1.20.5-alpine (#3809) Bumps golang from 1.20.4-alpine to 1.20.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/go-playground/validator/v10 in /v2 (#3810) Bumps [github.com/go-playground/validator/v10](https://github.com/go-playground/validator) from 10.11.2 to 10.14.1. - [Release notes](https://github.com/go-playground/validator/releases) - [Commits](https://github.com/go-playground/validator/compare/v10.11.2...v10.14.1) --- updated-dependencies: - dependency-name: github.com/go-playground/validator/v10 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/rawhttp in /v2 (#3811) Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.11 to 0.1.13. - [Release notes](https://github.com/projectdiscovery/rawhttp/releases) - [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.11...v0.1.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/rawhttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/go-git/go-git/v5 from 5.6.1 to 5.7.0 in /v2 (#3812) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.6.1 to 5.7.0. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.6.1...v5.7.0) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/hmap in /v2 (#3781) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.11 to 0.0.13. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.11...v0.0.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Using safe dereferencing * adding comment * fixing and condition * fixing test id * adding integration test * update goflags dependency * update goflags dependency * bump goflags v0.1.9 => v0.1.10 * handle failure matcher flags logic at executor itself * add integration test to matcher status per request * Adding random tls impersonate (#3844) * adding random tls impersonate * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> * Use templateman enhance api to populate CVE info (#3788) * use templateman enhance api to populate cve info * rename cve-annotate => tmc add additional flags to format, lint and enhance template using templateman apis * minior changes * remove duplicate code * misc update * Add validate and error log option * print if updated * print format and enhance only if updated * make max-request optional * fix reference unmarshal error * fix removing self-contained tag --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Tarun Koyalwar Co-authored-by: Sandeep Singh * fix matcher status with network protocol * fix test * remove -msr flag --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Co-authored-by: shubhamrasal Co-authored-by: 三米前有蕉皮 Co-authored-by: Tarun Koyalwar Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Co-authored-by: Shubham Rasal Co-authored-by: GitHub Action Co-authored-by: Mzack9999 Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Sandeep Singh --- integration_tests/http/matcher-status.yaml | 40 +++++++++++++++ v2/cmd/integration-test/http.go | 19 +++++++ v2/pkg/output/output.go | 53 ++++++++++---------- v2/pkg/protocols/common/executer/executer.go | 22 +++++--- v2/pkg/protocols/network/request.go | 2 + v2/pkg/protocols/network/request_test.go | 2 +- v2/pkg/testutils/integration.go | 2 +- 7 files changed, 106 insertions(+), 34 deletions(-) create mode 100644 integration_tests/http/matcher-status.yaml diff --git a/integration_tests/http/matcher-status.yaml b/integration_tests/http/matcher-status.yaml new file mode 100644 index 000000000..5704c2a3d --- /dev/null +++ b/integration_tests/http/matcher-status.yaml @@ -0,0 +1,40 @@ +id: matchet-status + +info: + name: Test Matcher Status + author: pdteam + severity: critical + +variables: + username: test + password: admin + date: 2023-05-31 + +http: + - method: GET + path: + - "{{RootURL}}/login?username={{username}}&password={{password}}" + - "{{BaseURL}}/admin-pannel" + + - method: GET + path: + - "{{BaseURL}}/dashboard?date={{date}}" + - "{{BaseURL}}/signup" + + - method: POST + path: + - "{{BaseURL}}/filemanager/upload.php" + body: "fldr=&url=file:///etc/passwd" + + + stop-at-first-match: true + matchers-condition: and + matchers: + - type: word + part: body + words: + - "matcher status" + + - type: status + status: + - 200 diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index fa57602ca..9a3046b48 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -78,6 +78,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, "http/save-extractor-values-to-file.yaml": &httpSaveExtractorValuesToFile{}, "http/cli-with-constants.yaml": &ConstantWithCliVar{}, + "http/matcher-status.yaml": &matcherStatusTest{}, "http/disable-path-automerge.yaml": &httpDisablePathAutomerge{}, } @@ -1425,6 +1426,24 @@ func (h *ConstantWithCliVar) Execute(filePath string) error { return expectResultsCount(got, 1) } +type matcherStatusTest struct{} + +// Execute executes a test case and returns an error if occurred +func (h *matcherStatusTest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-ms") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + // disable path automerge in raw request type httpDisablePathAutomerge struct{} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 9d2895bc7..a23d6f381 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -45,19 +45,19 @@ type Writer interface { // StandardWriter is a writer writing output to file and screen for results. type StandardWriter struct { - json bool - jsonReqResp bool - timestamp bool - noMetadata bool - matcherStatus bool - mutex *sync.Mutex - aurora aurora.Aurora - outputFile io.WriteCloser - traceFile io.WriteCloser - errorFile io.WriteCloser - severityColors func(severity.Severity) string - storeResponse bool - storeResponseDir string + json bool + jsonReqResp bool + timestamp bool + noMetadata bool + matcherStatus bool + mutex *sync.Mutex + aurora aurora.Aurora + outputFile io.WriteCloser + traceFile io.WriteCloser + errorFile io.WriteCloser + severityColors func(severity.Severity) string + storeResponse bool + storeResponseDir string } var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) @@ -187,20 +187,21 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err) } } + writer := &StandardWriter{ - json: options.JSONL, - jsonReqResp: options.JSONRequests, - noMetadata: options.NoMeta, - matcherStatus: options.MatcherStatus, - timestamp: options.Timestamp, - aurora: auroraColorizer, - mutex: &sync.Mutex{}, - outputFile: outputFile, - traceFile: traceOutput, - errorFile: errorOutput, - severityColors: colorizer.New(auroraColorizer), - storeResponse: options.StoreResponse, - storeResponseDir: options.StoreResponseDir, + json: options.JSONL, + jsonReqResp: options.JSONRequests, + noMetadata: options.NoMeta, + matcherStatus: options.MatcherStatus, + timestamp: options.Timestamp, + aurora: auroraColorizer, + mutex: &sync.Mutex{}, + outputFile: outputFile, + traceFile: traceOutput, + errorFile: errorOutput, + severityColors: colorizer.New(auroraColorizer), + storeResponse: options.StoreResponse, + storeResponseDir: options.StoreResponseDir, } return writer, nil } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index f9bae9597..8bfa1be0f 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -70,6 +70,17 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { }) } previous := make(map[string]interface{}) + + var lastMatcherEvent *output.InternalWrappedEvent + writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { + if !results.Load() && matcherStatus { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + results.CompareAndSwap(false, true) + } + } + for _, req := range e.requests { inputItem := input.Clone() if e.options.InputHelper != nil && input.MetaInput.Input != "" { @@ -94,16 +105,12 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. if !event.HasOperatorResult() && !event.UsesInteractsh { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + lastMatcherEvent = event } else { if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { results.CompareAndSwap(false, true) } else { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + lastMatcherEvent = event } } }) @@ -118,6 +125,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { break } } + if lastMatcherEvent != nil { + writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) + } return results.Load(), nil } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index d0f144a04..5acb518d7 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -61,6 +61,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, actualAddress := replacer.Replace(kv.address, variables) if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { + outputEvent := request.responseToDSLMap("", "", "", address, "") + callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err) continue } diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index ff8b70134..89c528724 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -87,7 +87,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }) require.Nil(t, err, "could not execute network request") }) - require.Nil(t, finalEvent, "could not get event output from request") + require.Nil(t, finalEvent.Results, "could not get event output from request") request.Inputs[0].Type = NetworkInputTypeHolder{NetworkInputType: hexType} request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index b3a90c5d0..42777fd8f 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -63,7 +63,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro if debug { fmt.Println(string(data)) } - if err != nil { + if len(data) < 1 && err != nil { return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) } var parts []string From 38129bac18e38bd9bad86b88f91290d528ab145d Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Mon, 3 Jul 2023 12:43:24 +0530 Subject: [PATCH 04/62] preserve order of query parameters (#3887) * preserve order of parameters * rawhttp version bump --------- Co-authored-by: Mzack9999 --- v2/go.mod | 14 +++---- v2/go.sum | 46 ++++++++++++---------- v2/pkg/protocols/common/fuzz/execute.go | 2 +- v2/pkg/protocols/common/fuzz/parts.go | 32 ++++++++------- v2/pkg/protocols/common/fuzz/parts_test.go | 8 ++-- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/http/raw/raw.go | 6 +-- v2/pkg/protocols/http/raw/raw_test.go | 6 +-- v2/pkg/protocols/utils/variables.go | 2 +- 9 files changed, 64 insertions(+), 54 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index ad870cc0f..3e9181d81 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,10 +21,10 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.18 - github.com/projectdiscovery/fastdialer v0.0.31 + github.com/projectdiscovery/fastdialer v0.0.32 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 - github.com/projectdiscovery/rawhttp v0.1.13 + github.com/projectdiscovery/rawhttp v0.1.16-0.20230703065054-806b4fcfab11 github.com/projectdiscovery/retryabledns v1.0.30 github.com/projectdiscovery/retryablehttp-go v1.0.18 github.com/projectdiscovery/yamldoc-go v1.0.4 @@ -77,7 +77,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.0 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.39 + github.com/projectdiscovery/utils v0.0.40 github.com/projectdiscovery/wappalyzergo v0.0.102 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -91,7 +91,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect @@ -211,16 +211,16 @@ require ( github.com/ysmood/leakless v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect + github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect golang.org/x/crypto v0.10.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/mod v0.11.0 // indirect golang.org/x/sys v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect + golang.org/x/tools v0.10.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.29.1 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/v2/go.sum b/v2/go.sum index b37dd5f47..ecf73bea7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -51,8 +51,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= @@ -370,6 +370,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= @@ -411,8 +412,8 @@ github.com/projectdiscovery/clistats v0.0.18 h1:WLQNqLXsKvjoieDwXJO/1jlnxR0x9vdF github.com/projectdiscovery/clistats v0.0.18/go.mod h1:YUnUrMHFw+FHwUTIKr1KDUwz81x+SFjPU3xfLqXfzf0= github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd h1:16DMjd4HeACrC9CkWJkkLeSh+LYPDorwNx11BlTbonU= github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd/go.mod h1:S72Cq/lfxzkldf64Sul1G2KFbGKNgpRFFCF/FazpznM= -github.com/projectdiscovery/fastdialer v0.0.31 h1:eu0wTBCWjT8dXChmBtnQaAxoFpkLdvq0VroRxZoe/M8= -github.com/projectdiscovery/fastdialer v0.0.31/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= +github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= +github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -433,8 +434,8 @@ github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzV github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= -github.com/projectdiscovery/rawhttp v0.1.13 h1:Xn3NY3SYIk0151K5Qfuvx3tayl2UOoxMuVyYvGT95BA= -github.com/projectdiscovery/rawhttp v0.1.13/go.mod h1:AjZUYdPCx4xqeWYPqFPLGCxQsVFeUrobxidnU6Nta8M= +github.com/projectdiscovery/rawhttp v0.1.16-0.20230703065054-806b4fcfab11 h1:kj5HPwajW4IbSYLrZIg03yXGp4id0tYMulSOhU5t5Lo= +github.com/projectdiscovery/rawhttp v0.1.16-0.20230703065054-806b4fcfab11/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0= @@ -448,8 +449,8 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.39 h1:iyi5qPilENRmFyt16qtd58pb65fUu0wAU2C0Lq5t6zo= -github.com/projectdiscovery/utils v0.0.39/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= +github.com/projectdiscovery/utils v0.0.40 h1:vChjk6PnSb6+nUaP3/GS8LjbGGlrUpFiZBbeoBa9Sww= +github.com/projectdiscovery/utils v0.0.40/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= github.com/projectdiscovery/wappalyzergo v0.0.102 h1:ABjZghof2U2yzGNL+q5ouWHEardLd2o53Ukgrf8CZzE= github.com/projectdiscovery/wappalyzergo v0.0.102/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -483,6 +484,7 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -552,11 +554,9 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= +github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/xanzy/go-gitlab v0.84.0 h1:PdpCaskQSgcVDsx21c6ikf8Rfyo7SNtFAJwP9PrbCFE= github.com/xanzy/go-gitlab v0.84.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= @@ -594,8 +594,12 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= -github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= +github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= +github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= +github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -620,6 +624,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -629,8 +634,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -646,9 +651,9 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -673,7 +678,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -688,6 +693,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -727,6 +733,7 @@ golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -746,8 +753,8 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -788,7 +795,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v2/pkg/protocols/common/fuzz/execute.go b/v2/pkg/protocols/common/fuzz/execute.go index a16c7107e..eca6d8d90 100644 --- a/v2/pkg/protocols/common/fuzz/execute.go +++ b/v2/pkg/protocols/common/fuzz/execute.go @@ -75,7 +75,7 @@ func (rule *Rule) isExecutable(input *contextargs.Context) bool { if err != nil { return false } - if len(parsed.Query()) > 0 && rule.partType == queryPartType { + if !parsed.Query().IsEmpty() && rule.partType == queryPartType { return true } return false diff --git a/v2/pkg/protocols/common/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go index 0576f302f..43e0f4201 100644 --- a/v2/pkg/protocols/common/fuzz/parts.go +++ b/v2/pkg/protocols/common/fuzz/parts.go @@ -7,9 +7,11 @@ import ( "strings" "github.com/corpix/uarand" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/retryablehttp-go" + sliceutil "github.com/projectdiscovery/utils/slice" urlutil "github.com/projectdiscovery/utils/url" ) @@ -29,32 +31,33 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) return err } origRequestURL := requestURL.Clone() - temp := urlutil.Params{} - for k, v := range origRequestURL.Query() { - // this has to be a deep copy - x := []string{} - x = append(x, v...) - temp[k] = x - } + // clone the params to avoid modifying the original + temp := origRequestURL.Params.Clone() - for key, values := range origRequestURL.Query() { + origRequestURL.Query().Iterate(func(key string, values []string) bool { + cloned := sliceutil.Clone(values) for i, value := range values { if !rule.matchKeyOrValue(key, value) { continue } var evaluated string evaluated, input.InteractURLs = rule.executeEvaluate(input, key, value, payload, input.InteractURLs) - temp[key][i] = evaluated + cloned[i] = evaluated if rule.modeType == singleModeType { + temp.Update(key, cloned) requestURL.Params = temp - if err := rule.buildQueryInput(input, requestURL, input.InteractURLs); err != nil { - return err + if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil { + err = qerr + gologger.Error().Msgf("Could not build request for query part rule %v: %s\n", rule, err) + return false } - temp[key][i] = value // change back to previous value for temp + cloned[i] = value // change back to previous value for temp } } - } + temp.Update(key, cloned) + return true + }) if rule.modeType == multipleModeType { requestURL.Params = temp @@ -62,7 +65,8 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) return err } } - return nil + + return err } // buildQueryInput returns created request for a Query Input diff --git a/v2/pkg/protocols/common/fuzz/parts_test.go b/v2/pkg/protocols/common/fuzz/parts_test.go index e1855a3ca..a8f6e141b 100644 --- a/v2/pkg/protocols/common/fuzz/parts_test.go +++ b/v2/pkg/protocols/common/fuzz/parts_test.go @@ -32,9 +32,9 @@ func TestExecuteQueryPartRule(t *testing.T) { }, "1337'") require.NoError(t, err, "could not execute part rule") require.ElementsMatch(t, []string{ - "http://localhost:8080/?file=passwdfile&mode=multiple&url=localhost1337'", - "http://localhost:8080/?file=passwdfile&mode=multiple1337'&url=localhost", - "http://localhost:8080/?file=passwdfile1337'&mode=multiple&url=localhost", + "http://localhost:8080/?url=localhost1337'&mode=multiple&file=passwdfile", + "http://localhost:8080/?url=localhost&mode=multiple1337'&file=passwdfile", + "http://localhost:8080/?url=localhost&mode=multiple&file=passwdfile1337'", }, generatedURL, "could not get generated url") }) t.Run("multiple", func(t *testing.T) { @@ -54,7 +54,7 @@ func TestExecuteQueryPartRule(t *testing.T) { }, }, "1337'") require.NoError(t, err, "could not execute part rule") - require.Equal(t, "http://localhost:8080/?file=passwdfile1337'&mode=multiple1337'&url=localhost1337'", generatedURL, "could not get generated url") + require.Equal(t, "http://localhost:8080/?url=localhost1337'&mode=multiple1337'&file=passwdfile1337'", generatedURL, "could not get generated url") }) } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ac492b9f6..1b7684026 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -149,7 +149,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, } // while merging parameters first preference is given to target params finalparams := parsed.Params - finalparams.Merge(reqURL.Params) + finalparams.Merge(reqURL.Params.Encode()) reqURL.Params = finalparams return r.generateHttpRequest(ctx, reqURL, finalVars, payloads) } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index b2892ceac..9acbeb2f8 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -61,13 +61,13 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b if (cloned.Path == "" || cloned.Path == "/") && !strings.HasPrefix(prevPath, "/") { // Edgecase if raw unsafe request is // GET 1337?with=param HTTP/1.1 - if tmpurl, err := urlutil.ParseRelativePath(prevPath, true); err == nil && len(tmpurl.Params) > 0 { + if tmpurl, err := urlutil.ParseRelativePath(prevPath, true); err == nil && !tmpurl.Params.IsEmpty() { // if raw request contains parameters - cloned.Params.Merge(tmpurl.Params) + cloned.Params.Merge(tmpurl.Params.Encode()) unsafeRelativePath = strings.TrimPrefix(tmpurl.Path, "/") + "?" + cloned.Params.Encode() } else { // if raw request does not contain param - if len(cloned.Params) > 0 { + if !cloned.Params.IsEmpty() { unsafeRelativePath = prevPath + "?" + cloned.Params.Encode() } else { unsafeRelativePath = prevPath diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index b004ff8f3..a44664d48 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -31,17 +31,17 @@ Host: {{Hostname}}`, parseURL(t, "https://example.com:8080/test"), false, false) Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test"), false, false) require.Nil(t, err, "could not parse GET request") // url.values are sorted to avoid randomness of using maps - require.Equal(t, "https://example.com:8080/test?password=test&username=test", request.FullURL, "Could not parse request url correctly") + require.Equal(t, "https://example.com:8080/test?username=test&password=test", request.FullURL, "Could not parse request url correctly") request, err = Parse(`GET ?username=test&password=test HTTP/1.1 Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false) require.Nil(t, err, "could not parse GET request") - require.Equal(t, "https://example.com:8080/test/?password=test&username=test", request.FullURL, "Could not parse request url correctly") + require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly") request, err = Parse(`GET /?username=test&password=test HTTP/1.1 Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false) require.Nil(t, err, "could not parse GET request") - require.Equal(t, "https://example.com:8080/test/?password=test&username=test", request.FullURL, "Could not parse request url correctly") + require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly") }) } diff --git a/v2/pkg/protocols/utils/variables.go b/v2/pkg/protocols/utils/variables.go index a94c99c99..92e562ce1 100644 --- a/v2/pkg/protocols/utils/variables.go +++ b/v2/pkg/protocols/utils/variables.go @@ -115,7 +115,7 @@ func GenerateVariables(input interface{}, removeTrailingSlash bool, additionalVa func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[string]interface{} { parsed := inputURL.Clone() - parsed.Params = make(urlutil.Params) + parsed.Params = urlutil.NewOrderedParams() port := parsed.Port() if port == "" { if parsed.Scheme == "https" { From 2b18c60378d7425411cb2aa519a96b663f42c30a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 12:45:32 +0530 Subject: [PATCH 05/62] chore(deps): bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /v2 (#3897) Bumps [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) from 1.11.67 to 1.11.71. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.11.67...feature/s3/manager/v1.11.71) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 4 ++-- v2/go.sum | 23 ++++------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3e9181d81..3af17af6b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -57,8 +57,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.18.1 github.com/aws/aws-sdk-go-v2/config v1.18.27 github.com/aws/aws-sdk-go-v2/credentials v1.13.26 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 - github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 + github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 github.com/docker/go-units v0.5.0 github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 diff --git a/v2/go.sum b/v2/go.sum index ecf73bea7..9e49467c3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -69,55 +69,40 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 h1:fI9/5BDEaAv/pv1VO1X1n3jfP9it+IGqWsCuuBQI8wM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67/go.mod h1:zQClPRIwQZfJlZq6WZve+s4Tb4JW+3V6eS+4+KrYeP8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o= -github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= From 4d1951221835add1979678e2ab25f69bb2d38181 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 13:00:31 +0530 Subject: [PATCH 06/62] chore(deps): bump github.com/projectdiscovery/clistats in /v2 (#3900) Bumps [github.com/projectdiscovery/clistats](https://github.com/projectdiscovery/clistats) from 0.0.18 to 0.0.19. - [Release notes](https://github.com/projectdiscovery/clistats/releases) - [Commits](https://github.com/projectdiscovery/clistats/compare/v0.0.18...v0.0.19) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/clistats dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3af17af6b..0ba6fc9a7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -20,7 +20,7 @@ require ( github.com/miekg/dns v1.1.55 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/clistats v0.0.18 + github.com/projectdiscovery/clistats v0.0.19 github.com/projectdiscovery/fastdialer v0.0.32 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 diff --git a/v2/go.sum b/v2/go.sum index 9e49467c3..00c71876c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -393,8 +393,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6uGfPofVpC4c= github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= -github.com/projectdiscovery/clistats v0.0.18 h1:WLQNqLXsKvjoieDwXJO/1jlnxR0x9vdFaRUAR3gXfKQ= -github.com/projectdiscovery/clistats v0.0.18/go.mod h1:YUnUrMHFw+FHwUTIKr1KDUwz81x+SFjPU3xfLqXfzf0= +github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= +github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd h1:16DMjd4HeACrC9CkWJkkLeSh+LYPDorwNx11BlTbonU= github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd/go.mod h1:S72Cq/lfxzkldf64Sul1G2KFbGKNgpRFFCF/FazpznM= github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= From 71ef8cb0ed11db6ca03f8485b627beb555591567 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 13:19:33 +0530 Subject: [PATCH 07/62] chore(deps): bump github.com/projectdiscovery/dsl in /v2 (#3901) Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.11-0.20230621170216-97e70ffb7efd to 0.0.12. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/commits/v0.0.12) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sandeep Singh --- v2/go.mod | 4 ++-- v2/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0ba6fc9a7..b32dfa33d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -66,7 +66,7 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd + github.com/projectdiscovery/dsl v0.0.12 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.10 @@ -128,7 +128,7 @@ require ( github.com/projectdiscovery/cdncheck v1.0.6 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/refraction-networking/utls v1.3.2 // indirect - github.com/sashabaranov/go-openai v1.11.2 // indirect + github.com/sashabaranov/go-openai v1.12.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 00c71876c..0cb3adfa1 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -395,8 +395,8 @@ github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6u github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd h1:16DMjd4HeACrC9CkWJkkLeSh+LYPDorwNx11BlTbonU= -github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd/go.mod h1:S72Cq/lfxzkldf64Sul1G2KFbGKNgpRFFCF/FazpznM= +github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= +github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw= github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -453,8 +453,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.11.2 h1:HuMf+18eldSKbqVblyeCQbtcqSpGVfqTshvi8Bn6zes= -github.com/sashabaranov/go-openai v1.11.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.12.0 h1:aRNHH0gtVfrpIaEolD0sWrLLRnYQNK4cH/bIAHwL8Rk= +github.com/sashabaranov/go-openai v1.12.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= From 5247b889bda8613e4f6a95423ce01226f3627b5a Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:19:08 +0530 Subject: [PATCH 08/62] goreleaser config update (fix) replacements is now deprecated - https://goreleaser.com/deprecations#archivesreplacements --- v2/.goreleaser.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 83b95e801..df77d595c 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -37,13 +37,7 @@ archives: - format: zip id: nuclei builds: [nuclei-cli] - replacements: - darwin: macOS - -- format: zip - id: annotate - builds: [annotate] - name_template: "{{ .Binary }}" + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: algorithm: sha256 From daecd169c543f03d2924048e0062b1e0f0cd17b6 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:24:01 +0530 Subject: [PATCH 09/62] version update --- v2/pkg/catalog/config/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 8ce1fbf95..46166eaa0 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConifgFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v2.9.7` + Version = `v2.9.8` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGithubTemplatesDirName = "github" From bb98eced070f4ae137b8cd2a7f887611bc1b9c93 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:42:25 +0530 Subject: [PATCH 10/62] dep update --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index b32dfa33d..c526a72ac 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -66,7 +66,7 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.12 + github.com/projectdiscovery/dsl v0.0.13-0.20230703115450-39d83b5a7026 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.10 diff --git a/v2/go.sum b/v2/go.sum index 0cb3adfa1..9045a5331 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -395,8 +395,8 @@ github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6u github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= -github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw= +github.com/projectdiscovery/dsl v0.0.13-0.20230703115450-39d83b5a7026 h1:wsV4dRVDJZlWu+aEGxs3Husnh3x7wCAZOifDpOEbXmY= +github.com/projectdiscovery/dsl v0.0.13-0.20230703115450-39d83b5a7026/go.mod h1:Zwn4CPdOxpQogu1rU2Z4kxbxUJdbLCQ+34ZK7AZH0i0= github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= From 1eb4c7c80c179bd64100a8d5b44017e6513bc1e9 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Tue, 4 Jul 2023 19:04:13 +0530 Subject: [PATCH 11/62] support disable-path-automerge in unsafe mode (#3888) * support disable-path-automerge in unsafe mode * fix dump request url * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- v2/pkg/protocols/http/raw/raw.go | 1 + v2/pkg/protocols/http/request.go | 7 ++++++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index b32dfa33d..ffceed94c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/fastdialer v0.0.32 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 - github.com/projectdiscovery/rawhttp v0.1.16-0.20230703065054-806b4fcfab11 + github.com/projectdiscovery/rawhttp v0.1.16 github.com/projectdiscovery/retryabledns v1.0.30 github.com/projectdiscovery/retryablehttp-go v1.0.18 github.com/projectdiscovery/yamldoc-go v1.0.4 diff --git a/v2/go.sum b/v2/go.sum index 0cb3adfa1..c63f90667 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -419,8 +419,8 @@ github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzV github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= -github.com/projectdiscovery/rawhttp v0.1.16-0.20230703065054-806b4fcfab11 h1:kj5HPwajW4IbSYLrZIg03yXGp4id0tYMulSOhU5t5Lo= -github.com/projectdiscovery/rawhttp v0.1.16-0.20230703065054-806b4fcfab11/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY= +github.com/projectdiscovery/rawhttp v0.1.16 h1:2wggRuVUty2Sypy1MKWM/NNmfbC97WkG6YtmoxzCOVc= +github.com/projectdiscovery/rawhttp v0.1.16/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0= diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 9acbeb2f8..46a6b562a 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -83,6 +83,7 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b } unsafeRelativePath = cloned.GetRelativePath() } + rawrequest.Path = cloned.GetRelativePath() rawrequest.UnsafeRawBytes = bytes.Replace(rawrequest.UnsafeRawBytes, []byte(prevPath), []byte(unsafeRelativePath), 1) default: diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 304eead2b..127b7a0ee 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -535,7 +535,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes options.ForceReadAllBody = request.ForceReadAllBody options.SNI = request.options.Options.SNI - resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) + inputUrl := input.MetaInput.Input + if url, err := urlutil.ParseURL(inputUrl, false); err == nil { + inputUrl = fmt.Sprintf("%s://%s", url.Scheme, url.Host) + } + formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path) + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) } else { hostname = generatedRequest.request.URL.Host formedURL = generatedRequest.request.URL.String() From b3ccb9a6e5796a2ec61df2fdec62a455964e32cc Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Tue, 4 Jul 2023 16:37:56 -0400 Subject: [PATCH 12/62] Exclude Raw Request Payloads (#3710) * Add command docs and CLI hook * Add configurable exclusion from reports * Register the CLI argument with exporter configuration * Switch to inverted logic with JSONRequest flag * Switch variable name for the -include-rr/-irr flag * Remove flags from README * Update call for -irr and -or * convert -irr to no-op --------- Co-authored-by: Tarun Koyalwar --- README.md | 3 +- README_CN.md | 1 + README_ID.md | 3 +- README_KR.md | 1 + v2/cmd/nuclei/main.go | 3 +- v2/internal/runner/runner.go | 30 ++++++++--- v2/pkg/output/output.go | 52 +++++++++---------- .../exporters/jsonexporter/jsonexporter.go | 12 ++++- v2/pkg/reporting/exporters/jsonl/jsonl.go | 12 ++++- .../reporting/exporters/markdown/markdown.go | 12 ++++- v2/pkg/testutils/testutils.go | 2 +- v2/pkg/types/types.go | 3 ++ 12 files changed, 95 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 12457a704..f421edbf6 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,8 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSONL output (for findings only) + -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED] + -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) -nm, -no-meta disable printing result metadata in cli output -ts, -timestamp enables printing timestamp in cli output -rdb, -report-db string nuclei reporting database (always use this to persist report data) diff --git a/README_CN.md b/README_CN.md index f2fd13997..c8c8ffa4b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -134,6 +134,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板 -nc, -no-color 禁用输出内容着色（ANSI转义码） -j, -jsonl 输出为jsonL（ines） -irr, -include-rr 在JSONL中输出对应的请求和相应（仅结果） + -or, -omit-raw -nm, -no-meta 不显示匹配的元数据 -nts, -no-timestamp 不在输出中显示时间戳 -rdb, -report-db string 本地的Nuclei结果数据库（始终使用该数据库保存结果） diff --git a/README_ID.md b/README_ID.md index ac24b0d77..df82bfd38 100644 --- a/README_ID.md +++ b/README_ID.md @@ -133,7 +133,8 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSONL output (for findings only) + -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED] + -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) -nm, -no-meta disable printing result metadata in cli output -nts, -no-timestamp disable printing timestamp in cli output -rdb, -report-db string nuclei reporting database (always use this to persist report data) diff --git a/README_KR.md b/README_KR.md index 8ab6cfd75..9608083d6 100644 --- a/README_KR.md +++ b/README_KR.md @@ -130,6 +130,7 @@ OUTPUT: -nc, -no-color 출력 내용 색상 비활성화 (ANSI escape codes) -j, -jsonl JSONL(ines) 형식으로 출력 -irr, -include-rr JSONL 출력에 요청/응답 쌍 포함(결과만) + -or, -omit-raw -nm, -no-meta cli 출력에서 결과 메타데이터 출력 비활성화 -nts, -no-timestamp cli 출력에서 결과 타임스탬프 출력 비활성화 -rdb, -report-db string nuclei 보고 데이터베이스(보고서 데이터를 유지하려면 항상 이것을 사용) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 28d1d534f..3f93911e3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -170,7 +170,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"), - flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), + flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"), + flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"), flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"), flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8b396a783..3a73fdc9e 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -340,10 +340,16 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } if options.MarkdownExportDirectory != "" { if reportingOptions != nil { - reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{ + Directory: options.MarkdownExportDirectory, + IncludeRawPayload: !options.OmitRawRequests, + } } else { reportingOptions = &reporting.Options{} - reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{ + Directory: options.MarkdownExportDirectory, + IncludeRawPayload: !options.OmitRawRequests, + } } } if options.SarifExport != "" { @@ -356,18 +362,30 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } if options.JSONExport != "" { if reportingOptions != nil { - reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport} + reportingOptions.JSONExporter = &jsonexporter.Options{ + File: options.JSONExport, + IncludeRawPayload: !options.OmitRawRequests, + } } else { reportingOptions = &reporting.Options{} - reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport} + reportingOptions.JSONExporter = &jsonexporter.Options{ + File: options.JSONExport, + IncludeRawPayload: !options.OmitRawRequests, + } } } if options.JSONLExport != "" { if reportingOptions != nil { - reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport} + reportingOptions.JSONLExporter = &jsonl.Options{ + File: options.JSONLExport, + IncludeRawPayload: !options.OmitRawRequests, + } } else { reportingOptions = &reporting.Options{} - reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport} + reportingOptions.JSONLExporter = &jsonl.Options{ + File: options.JSONLExport, + IncludeRawPayload: !options.OmitRawRequests, + } } } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index a23d6f381..a752eee18 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -45,19 +45,19 @@ type Writer interface { // StandardWriter is a writer writing output to file and screen for results. type StandardWriter struct { - json bool - jsonReqResp bool - timestamp bool - noMetadata bool - matcherStatus bool - mutex *sync.Mutex - aurora aurora.Aurora - outputFile io.WriteCloser - traceFile io.WriteCloser - errorFile io.WriteCloser - severityColors func(severity.Severity) string - storeResponse bool - storeResponseDir string + json bool + jsonReqResp bool + timestamp bool + noMetadata bool + matcherStatus bool + mutex *sync.Mutex + aurora aurora.Aurora + outputFile io.WriteCloser + traceFile io.WriteCloser + errorFile io.WriteCloser + severityColors func(severity.Severity) string + storeResponse bool + storeResponseDir string } var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) @@ -189,19 +189,19 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { } writer := &StandardWriter{ - json: options.JSONL, - jsonReqResp: options.JSONRequests, - noMetadata: options.NoMeta, - matcherStatus: options.MatcherStatus, - timestamp: options.Timestamp, - aurora: auroraColorizer, - mutex: &sync.Mutex{}, - outputFile: outputFile, - traceFile: traceOutput, - errorFile: errorOutput, - severityColors: colorizer.New(auroraColorizer), - storeResponse: options.StoreResponse, - storeResponseDir: options.StoreResponseDir, + json: options.JSONL, + jsonReqResp: !options.OmitRawRequests, + noMetadata: options.NoMeta, + matcherStatus: options.MatcherStatus, + timestamp: options.Timestamp, + aurora: auroraColorizer, + mutex: &sync.Mutex{}, + outputFile: outputFile, + traceFile: traceOutput, + errorFile: errorOutput, + severityColors: colorizer.New(auroraColorizer), + storeResponse: options.StoreResponse, + storeResponseDir: options.StoreResponseDir, } return writer, nil } diff --git a/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go b/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go index 973ec13d4..e2e16b289 100644 --- a/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go +++ b/v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go @@ -17,7 +17,8 @@ type Exporter struct { // Options contains the configuration options for JSON exporter client type Options struct { // File is the file to export found JSON result to - File string `yaml:"file"` + File string `yaml:"file"` + IncludeRawPayload bool `yaml:"include-raw-payload"` } // New creates a new JSON exporter integration client based on options. @@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { exporter.mutex.Lock() defer exporter.mutex.Unlock() + // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid + // writing them to the list of events. + // This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since + // the property is set to "omitempty" + if !exporter.options.IncludeRawPayload { + event.Request = "" + event.Response = "" + } + // Add the event to the rows exporter.rows = append(exporter.rows, *event) diff --git a/v2/pkg/reporting/exporters/jsonl/jsonl.go b/v2/pkg/reporting/exporters/jsonl/jsonl.go index 76750aaf0..07e5b5f8f 100644 --- a/v2/pkg/reporting/exporters/jsonl/jsonl.go +++ b/v2/pkg/reporting/exporters/jsonl/jsonl.go @@ -17,7 +17,8 @@ type Exporter struct { // Options contains the configuration options for JSONL exporter client type Options struct { // File is the file to export found JSONL result to - File string `yaml:"file"` + File string `yaml:"file"` + IncludeRawPayload bool `yaml:"include-raw-payload"` } // New creates a new JSONL exporter integration client based on options. @@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { exporter.mutex.Lock() defer exporter.mutex.Unlock() + // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid + // writing them to the list of events. + // This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since + // the property is set to "omitempty" + if !exporter.options.IncludeRawPayload { + event.Request = "" + event.Response = "" + } + // Add the event to the rows exporter.rows = append(exporter.rows, *event) diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 9924dcabb..362c65718 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -23,7 +23,8 @@ type Exporter struct { // Options contains the configuration options for GitHub issue tracker client type Options struct { // Directory is the directory to export found results to - Directory string `yaml:"directory"` + Directory string `yaml:"directory"` + IncludeRawPayload bool `yaml:"include-raw-payload"` } // New creates a new markdown exporter integration client based on options. @@ -51,6 +52,15 @@ func New(options *Options) (*Exporter, error) { // Export exports a passed result event to markdown func (exporter *Exporter) Export(event *output.ResultEvent) error { + // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid + // writing them to the list of events. + // This will reduce the amount of storage as well as the fields being excluded from the markdown report output since + // the property is set to "omitempty" + if !exporter.options.IncludeRawPayload { + event.Request = "" + event.Response = "" + } + // index file generation file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644) if err != nil { diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 93a2e4c7b..68e9e78fb 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -36,7 +36,7 @@ var DefaultOptions = &types.Options{ NoColor: true, UpdateTemplates: false, JSONL: false, - JSONRequests: false, + OmitRawRequests: false, EnableProgressBar: false, TemplateList: false, Stdin: false, diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 101e7c1bd..823ed858b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -231,7 +231,10 @@ type Options struct { // JSON writes json line output to files JSONL bool // JSONRequests writes requests/responses for matches in JSON output + // Deprecated: use OmitRawRequests instead as of now JSONRequests(include raw requests) is always true JSONRequests bool + // OmitRawRequests omits requests/responses for matches in JSON output + OmitRawRequests bool // JSONExport is the file to export JSON output format to JSONExport string // JSONLExport is the file to export JSONL output format to From b64d422b678e7693d7b37b9af4be91c0e33fe436 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:01:35 +0530 Subject: [PATCH 13/62] added disclaimer --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f421edbf6..0e089e3d6 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,10 @@ Nuclei is used to send requests across targets based on a template, leading to z We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 300** security researchers and engineers. - +| :exclamation: **Disclaimer** | +|---------------------------------| +| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. | +| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. | ## How it works From 305ac6a1439f02508cce8fe65546454d8dab33a9 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 6 Jul 2023 14:08:24 +0200 Subject: [PATCH 14/62] Bugfix 3871 workflow concurrency (#3903) * recursion waitgroup fix * cleanup --- v2/cmd/nuclei/test.yaml | 18 ------------------ v2/pkg/core/workflow_execute.go | 16 +++++++++++++--- v2/pkg/templates/compile.go | 1 - 3 files changed, 13 insertions(+), 22 deletions(-) delete mode 100644 v2/cmd/nuclei/test.yaml diff --git a/v2/cmd/nuclei/test.yaml b/v2/cmd/nuclei/test.yaml deleted file mode 100644 index ecb62d2ec..000000000 --- a/v2/cmd/nuclei/test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: basic-example - -info: - name: Test HTTP Template - author: pdteam - severity: info - -http: - - raw: - - |+ - GET / HTTP/1.1 - Host: {{Hostname}} - - unsafe: true - matchers: - - type: dsl - dsl: - - true \ No newline at end of file diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index b873ed6c1..d1734a580 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -25,14 +25,23 @@ func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Work ctxArgs.MetaInput = input ctxArgs.CookieJar = workflowCookieJar - swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) + // we can know the nesting level only at runtime, so the best we can do here is increase template threads by one unit in case it's equal to 1 to allow + // at least one subtemplate to go through, which it's idempotent to one in-flight template as the parent one is in an idle state + templateThreads := w.Options.Options.TemplateThreads + if templateThreads == 1 { + templateThreads++ + } + swg := sizedwaitgroup.New(templateThreads) + for _, template := range w.Workflows { swg.Add() + func(template *workflows.WorkflowTemplate) { + defer swg.Done() + if err := e.runWorkflowStep(template, ctxArgs, results, &swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } - swg.Done() }(template) } swg.Wait() @@ -126,10 +135,11 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co swg.Add() go func(subtemplate *workflows.WorkflowTemplate) { + defer swg.Done() + if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) } - swg.Done() }(subtemplate) } } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index f984e80b6..df51d8ee9 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -42,7 +42,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo var reader io.ReadCloser if utils.IsURL(filePath) { - //todo:instead of creating a new client each time, a default one should be reused (same as the standard library) // use retryablehttp (tls verification is enabled by default in the standard library) resp, err := retryablehttp.DefaultClient().Get(filePath) if err != nil { From 3dca03163ce2d5ff1f5d440ffadcd2e76d564b16 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 6 Jul 2023 18:03:52 +0200 Subject: [PATCH 15/62] Automatic target merge in network templates (#3904) * skip visited actual addressess * removed test * adding disable clustering support --- integration_tests/network/same-address.yaml | 29 +++++++++++++++++++++ v2/cmd/integration-test/network.go | 1 + v2/pkg/protocols/network/request.go | 9 ++++++- 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 integration_tests/network/same-address.yaml diff --git a/integration_tests/network/same-address.yaml b/integration_tests/network/same-address.yaml new file mode 100644 index 000000000..5cad9d21f --- /dev/null +++ b/integration_tests/network/same-address.yaml @@ -0,0 +1,29 @@ +id: same-target + +info: + name: same-target + author: pdteam + severity: info + description: Riak is a distributed NoSQL key-value data store that offers high availability, fault tolerance, operational simplicity, and scalability. + +network: + - host: + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + - "{{Hostname}}" + inputs: + - data: "PING\r\n" + read-size: 4 + matchers: + - type: word + part: data + words: + - "PONG" diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 37ce8c390..ff4d72a8c 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -12,6 +12,7 @@ var networkTestcases = map[string]testutils.TestCase{ "network/multi-step.yaml": &networkMultiStep{}, "network/self-contained.yaml": &networkRequestSelContained{}, "network/variables.yaml": &networkVariables{}, + "network/same-address.yaml": &networkBasic{}, } const defaultStaticPort = 5431 diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 5acb518d7..aecff19ea 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -29,6 +29,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" errorutil "github.com/projectdiscovery/utils/errors" + mapsutil "github.com/projectdiscovery/utils/maps" ) var _ protocols.Request = &Request{} @@ -57,9 +58,16 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, variablesMap := request.options.Variables.Evaluate(variables) variables = generators.MergeMaps(variablesMap, variables, request.options.Constants) + visitedAddressess := make(mapsutil.Map[string, struct{}]) + for _, kv := range request.addresses { actualAddress := replacer.Replace(kv.address, variables) + if visitedAddressess.Has(actualAddress) && !request.options.Options.DisableClustering { + continue + } + visitedAddressess.Set(actualAddress, struct{}{}) + if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { outputEvent := request.responseToDSLMap("", "", "", address, "") callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) @@ -110,7 +118,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac conn net.Conn err error ) - if host, _, err := net.SplitHostPort(actualAddress); err == nil { hostname = host } From 5e284879d3f54d6c29b2d9e1b7053c589f882347 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Fri, 7 Jul 2023 05:39:14 +0530 Subject: [PATCH 16/62] adds default ztls fallback support for tls errors (#3909) * bump fastdialer * make -ztls flag as no-op and deprecate * bump fastdialer version from main branch * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- v2/cmd/nuclei/main.go | 2 +- v2/go.mod | 2 +- v2/go.sum | 4 ++-- v2/pkg/protocols/common/protocolstate/state.go | 2 +- v2/pkg/protocols/ssl/ssl.go | 4 ---- v2/pkg/types/types.go | 2 +- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 3f93911e3..981db8499 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -201,7 +201,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"), - flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), + flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"), flagSet.BoolVar(&options.Sandbox, "sandbox", false, "sandbox nuclei for safe templates execution"), flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"), diff --git a/v2/go.mod b/v2/go.mod index ffceed94c..394f51795 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,7 +21,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.32 + github.com/projectdiscovery/fastdialer v0.0.33 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.16 diff --git a/v2/go.sum b/v2/go.sum index c63f90667..b718e6418 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -397,8 +397,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw= -github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= -github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= +github.com/projectdiscovery/fastdialer v0.0.33 h1:FKXZjJme5nhgnnRL0Y4KjZ5YXKg03dX5ZJYFJV3LOwU= +github.com/projectdiscovery/fastdialer v0.0.33/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index b4662aeda..4e9df494a 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -95,8 +95,8 @@ func Init(options *types.Options) error { opts.Deny = append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...) } opts.WithDialerHistory = true - opts.WithZTLS = options.ZTLS opts.SNIName = options.SNI + // fastdialer now by default fallbacks to ztls when there are tls related errors dialer, err := fastdialer.NewDialer(opts) if err != nil { return errors.Wrap(err, "could not create dialer") diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index fe84fc147..56972c01e 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -112,10 +112,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { case request.ScanMode == "openssl" && !openssl.IsAvailable(): // if openssl is not installed instead of failing "auto" scanmode is used request.ScanMode = "auto" - - case options.Options.ZTLS && request.ScanMode == "ctls": - // only override if scanmode in template is "ctls" since auto internally uses ztls as fallback - request.ScanMode = "ztls" } tlsxOptions := &clients.Options{ diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 823ed858b..b3880c05f 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -277,7 +277,7 @@ type Options struct { ClientKeyFile string // ClientCAFile client certificate authority file (PEM-encoded) used for authenticating against scanned hosts ClientCAFile string - // Use ZTLS library + // Deprecated: Use ZTLS library ZTLS bool // Sandbox enables sandboxed nuclei template execution Sandbox bool From 28f6c3797415290edc04b9fdec1ad37c5bc0559c Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 7 Jul 2023 23:35:32 +0530 Subject: [PATCH 17/62] misc fix --- v2/cmd/tmc/main.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go index 97f8d6122..ad75d08d6 100644 --- a/v2/cmd/tmc/main.go +++ b/v2/cmd/tmc/main.go @@ -206,11 +206,6 @@ func process(opts options) error { } if opts.enhance { - // currently enhance api only supports cve-id's - matches := idRegex.FindAllStringSubmatch(dataString, 1) - if len(matches) == 0 { - continue - } enhancedTemplateData, isEnhanced, err := enhanceTemplate(dataString) if err != nil { gologger.Info().Label("enhance").Msg(logErrMsg(path, err, opts.debug, errFile)) From 9a44deb9e1d5375e0fe2f1c1faaf0e1b6f82972b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 9 Jul 2023 01:19:56 +0530 Subject: [PATCH 18/62] added epss-percentile field under classification (#3911) * added epss-percentile field under classification * lint fix --- v2/cmd/tmc/main.go | 2 -- v2/pkg/catalog/loader/filter/tag_filter.go | 1 + v2/pkg/model/model.go | 5 +++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go index ad75d08d6..c334372da 100644 --- a/v2/cmd/tmc/main.go +++ b/v2/cmd/tmc/main.go @@ -76,8 +76,6 @@ func init() { } } -var idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)") - type options struct { input string errorLogFile string diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index 571c212af..1da6090e3 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -207,6 +207,7 @@ func tryCollectConditionsMatchinfo(template *templates.Template) map[string]inte parameters["cwe_id"] = template.Info.Classification.CWEID.ToSlice() parameters["cpe"] = template.Info.Classification.CPE parameters["epss_score"] = template.Info.Classification.EPSSScore + parameters["epss_percentile"] = template.Info.Classification.EPSSPercentile } if template.Type() == types.HTTPProtocol { diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index 93e92f966..63c0e1ee2 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -101,6 +101,11 @@ type Classification struct { // - value: "\"0.42509\"" EPSSScore float64 `json:"epss-score,omitempty" yaml:"epss-score,omitempty" jsonschema:"title=epss score for the template,description=EPSS Score for the template,example=0.42509"` // description: | + // EPSS Percentile for the template. + // examples: + // - value: "\"0.42509\"" + EPSSPercentile float64 `json:"epss-percentile,omitempty" yaml:"epss-percentile,omitempty" jsonschema:"title=epss percentile for the template,description=EPSS Percentile for the template,example=0.42509"` + // description: | // CPE for the template. // examples: // - value: "\"cpe:/a:vendor:product:version\"" From 0993ac4cbbf7eed88444b90f487c0d0f12354f9b Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sat, 8 Jul 2023 19:53:01 +0000 Subject: [PATCH 19/62] Auto Generate Syntax Docs + JSONSchema [Sat Jul 8 19:53:01 UTC 2023] :robot: --- SYNTAX-REFERENCE.md | 23 +++++++++++++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 19 +++++++++++++------ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index def307bae..a0406daa6 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -825,6 +825,29 @@ epss-score: "0.42509" ``` + + +
+ +
+ +epss-percentile float64 + +
+
+ +EPSS Percentile for the template. + + + +Examples: + + +```yaml +epss-percentile: "0.42509" +``` + +
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 452d4fd85..bb6506ad8 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -32,6 +32,11 @@ "title": "epss score for the template", "description": "EPSS Score for the template" }, + "epss-percentile": { + "type": "number", + "title": "epss percentile for the template", + "description": "EPSS Percentile for the template" + }, "cpe": { "type": "string", "title": "cpe for the template", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 25b094b01..992e5bed6 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -298,7 +298,7 @@ func init() { FieldName: "classification", }, } - MODELClassificationDoc.Fields = make([]encoder.Doc, 6) + MODELClassificationDoc.Fields = make([]encoder.Doc, 7) MODELClassificationDoc.Fields[0].Name = "cve-id" MODELClassificationDoc.Fields[0].Type = "stringslice.StringSlice" MODELClassificationDoc.Fields[0].Note = "" @@ -334,13 +334,20 @@ func init() { MODELClassificationDoc.Fields[4].Comments[encoder.LineComment] = "EPSS Score for the template." MODELClassificationDoc.Fields[4].AddExample("", "0.42509") - MODELClassificationDoc.Fields[5].Name = "cpe" - MODELClassificationDoc.Fields[5].Type = "string" + MODELClassificationDoc.Fields[5].Name = "epss-percentile" + MODELClassificationDoc.Fields[5].Type = "float64" MODELClassificationDoc.Fields[5].Note = "" - MODELClassificationDoc.Fields[5].Description = "CPE for the template." - MODELClassificationDoc.Fields[5].Comments[encoder.LineComment] = "CPE for the template." + MODELClassificationDoc.Fields[5].Description = "EPSS Percentile for the template." + MODELClassificationDoc.Fields[5].Comments[encoder.LineComment] = "EPSS Percentile for the template." - MODELClassificationDoc.Fields[5].AddExample("", "cpe:/a:vendor:product:version") + MODELClassificationDoc.Fields[5].AddExample("", "0.42509") + MODELClassificationDoc.Fields[6].Name = "cpe" + MODELClassificationDoc.Fields[6].Type = "string" + MODELClassificationDoc.Fields[6].Note = "" + MODELClassificationDoc.Fields[6].Description = "CPE for the template." + MODELClassificationDoc.Fields[6].Comments[encoder.LineComment] = "CPE for the template." + + MODELClassificationDoc.Fields[6].AddExample("", "cpe:/a:vendor:product:version") HTTPRequestDoc.Type = "http.Request" HTTPRequestDoc.Comments[encoder.LineComment] = " Request contains a http request to be made from a template" From 2a9929878bdccd3c07b7b616b3375f6423df7abe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:15:01 +0530 Subject: [PATCH 20/62] chore(deps): bump golang.org/x/oauth2 from 0.9.0 to 0.10.0 in /v2 (#3916) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.9.0 to 0.10.0. - [Commits](https://github.com/golang/oauth2/compare/v0.9.0...v0.10.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 12 ++++++------ v2/go.sum | 26 +++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 394f51795..897d0f8ae 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -39,9 +39,9 @@ require ( github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.11.0 - golang.org/x/oauth2 v0.9.0 - golang.org/x/text v0.10.0 + golang.org/x/net v0.12.0 + golang.org/x/oauth2 v0.10.0 + golang.org/x/text v0.11.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) @@ -215,14 +215,14 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.10.0 + golang.org/x/crypto v0.11.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/mod v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.10.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.29.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index b718e6418..a69243559 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -617,8 +617,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -651,12 +651,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= -golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -704,8 +704,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -714,7 +714,7 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -725,8 +725,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -751,8 +751,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 5283d332142c05d337466328bdae9c3822597f41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:15:17 +0530 Subject: [PATCH 21/62] chore(deps): bump github.com/go-rod/rod from 0.113.0 to 0.113.4 in /v2 (#3915) Bumps [github.com/go-rod/rod](https://github.com/go-rod/rod) from 0.113.0 to 0.113.4. - [Release notes](https://github.com/go-rod/rod/releases) - [Commits](https://github.com/go-rod/rod/compare/v0.113.0...v0.113.4) --- updated-dependencies: - dependency-name: github.com/go-rod/rod dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 897d0f8ae..dc5ea64c9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/go-playground/validator/v10 v10.14.1 - github.com/go-rod/rod v0.113.0 + github.com/go-rod/rod v0.113.4 github.com/gobwas/ws v1.2.1 github.com/google/go-github v17.0.0+incompatible github.com/itchyny/gojq v0.12.13 diff --git a/v2/go.sum b/v2/go.sum index a69243559..aeb9cb5c9 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -187,8 +187,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-rod/rod v0.113.0 h1:E7+GLjYVZnScewIB2u8+66joQLaDGbOLzSOT4orNHms= -github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-rod/rod v0.113.4 h1:O5a/VTl1h6a08ecXkOF9oVJBi8fjBnF0FtyMSgd3KaY= +github.com/go-rod/rod v0.113.4/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= From 7eadcf761561d9c7a6f83b7b5b361b195461fe1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:15:32 +0530 Subject: [PATCH 22/62] chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 (#3914) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.102 to 0.0.104. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.102...v0.0.104) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index dc5ea64c9..6dcacccd0 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.0 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 github.com/projectdiscovery/utils v0.0.40 - github.com/projectdiscovery/wappalyzergo v0.0.102 + github.com/projectdiscovery/wappalyzergo v0.0.104 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/v2/go.sum b/v2/go.sum index aeb9cb5c9..d2dacbe47 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -436,8 +436,8 @@ github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6L github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= github.com/projectdiscovery/utils v0.0.40 h1:vChjk6PnSb6+nUaP3/GS8LjbGGlrUpFiZBbeoBa9Sww= github.com/projectdiscovery/utils v0.0.40/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= -github.com/projectdiscovery/wappalyzergo v0.0.102 h1:ABjZghof2U2yzGNL+q5ouWHEardLd2o53Ukgrf8CZzE= -github.com/projectdiscovery/wappalyzergo v0.0.102/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/projectdiscovery/wappalyzergo v0.0.104 h1:hdda6WxAzXVpLBbJW1sLqrwOXHn0prP9IYFY7dfCMjE= +github.com/projectdiscovery/wappalyzergo v0.0.104/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= From 880adc4405fa4b5cc5c259d67254b3b2f39a5bfb Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Wed, 12 Jul 2023 02:54:55 +0300 Subject: [PATCH 23/62] Automate merging of dependabot PRs to dep branch (#3857) * add auto sync and merge workflows * misc update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .github/dependabot.yml | 6 +++--- .github/workflows/auto-merge.yml | 25 +++++++++++++++++++++++++ .github/workflows/sync-dep.yml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/auto-merge.yml create mode 100644 .github/workflows/sync-dep.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fddde924e..24bdcc299 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,7 +11,7 @@ updates: directory: "/" schedule: interval: "weekly" - target-branch: "dev" + target-branch: "dep" commit-message: prefix: "chore" include: "scope" @@ -23,7 +23,7 @@ updates: directory: "v2/" schedule: interval: "weekly" - target-branch: "dev" + target-branch: "dep" commit-message: prefix: "chore" include: "scope" @@ -35,7 +35,7 @@ updates: directory: "/" schedule: interval: "weekly" - target-branch: "dev" + target-branch: "dep" commit-message: prefix: "chore" include: "scope" diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml new file mode 100644 index 000000000..0e56f0a03 --- /dev/null +++ b/.github/workflows/auto-merge.yml @@ -0,0 +1,25 @@ +name: 🤖 dep auto merge + +on: + pull_request: + branches: + - dep + workflow_dispatch: + +permissions: + pull-requests: write + issues: write + repository-projects: write + +jobs: + automerge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.DEPENDABOT_PAT }} + + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + github-token: ${{ secrets.DEPENDABOT_PAT }} + target: all \ No newline at end of file diff --git a/.github/workflows/sync-dep.yml b/.github/workflows/sync-dep.yml new file mode 100644 index 000000000..dfe9af9af --- /dev/null +++ b/.github/workflows/sync-dep.yml @@ -0,0 +1,30 @@ +name: 🤖 sync dep + +on: + push: + branches: + - dev + +permissions: + pull-requests: write + issues: write + repository-projects: write + +jobs: + sync-dep-with-dev: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Node + uses: actions/setup-node@v3 + with: + node-version: 12 + + - name: Opening pull request + id: pull + uses: tretuna/sync-branches@1.4.0 + with: + GITHUB_TOKEN: ${{ secrets.DEPENDABOT_PAT }} + FROM_BRANCH: "${{ github.ref_name }}" + TO_BRANCH: "dep" \ No newline at end of file From eaa6209a77af89ebd4e31f6af00196bed617173d Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:27:57 +0530 Subject: [PATCH 24/62] misc update --- .github/workflows/sync-dep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-dep.yml b/.github/workflows/sync-dep.yml index dfe9af9af..73798b78d 100644 --- a/.github/workflows/sync-dep.yml +++ b/.github/workflows/sync-dep.yml @@ -25,6 +25,6 @@ jobs: id: pull uses: tretuna/sync-branches@1.4.0 with: - GITHUB_TOKEN: ${{ secrets.DEPENDABOT_PAT }} + GITHUB_TOKEN: "${{ secrets.DEPENDABOT_PAT }}" FROM_BRANCH: "${{ github.ref_name }}" TO_BRANCH: "dep" \ No newline at end of file From 4d1c9fe8b1d560f7614919bab03fb625cfc99761 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:30:20 +0530 Subject: [PATCH 25/62] chore(deps): bump github.com/projectdiscovery/ratelimit in /v2 (#3913) Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.8 to 0.0.9. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.8...v0.0.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 6dcacccd0..0c669bdc5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -72,7 +72,7 @@ require ( github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/httpx v1.3.0 github.com/projectdiscovery/mapcidr v1.1.2 - github.com/projectdiscovery/ratelimit v0.0.8 + github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.0 diff --git a/v2/go.sum b/v2/go.sum index d2dacbe47..8593fb82e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -417,8 +417,8 @@ github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4 github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= -github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= +github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8= +github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I= github.com/projectdiscovery/rawhttp v0.1.16 h1:2wggRuVUty2Sypy1MKWM/NNmfbC97WkG6YtmoxzCOVc= github.com/projectdiscovery/rawhttp v0.1.16/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= From d51e058791db22ac3f471544a5fbab4f3be15ccb Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:51:06 +0530 Subject: [PATCH 26/62] add ErrNoMoreRequests for generator (#3918) * add ErrNoMoreRequests for generator * fix gh repo name convention * fix dirname in unit test --- v2/pkg/external/customtemplates/github.go | 16 ++-------------- v2/pkg/external/customtemplates/github_test.go | 2 +- v2/pkg/protocols/common/fuzz/parts.go | 6 ++---- v2/pkg/protocols/common/generators/load.go | 4 ++-- v2/pkg/protocols/headless/request.go | 4 ++-- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/http/request.go | 6 +++--- v2/pkg/protocols/http/utils/requtils.go | 1 + v2/pkg/types/types.go | 6 ++++++ 9 files changed, 20 insertions(+), 27 deletions(-) diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index e0cd75d9c..d9317ca3a 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -150,21 +150,9 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e return nil } -// getLocalRepoClonePath returns the clone path. -// if same name repo directory exists from another owner then it appends the owner then and returns the path -// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep +// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string { - if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) { - return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) - } - return filepath.Join(downloadPath, ctr.reponame) -} - -// isRepoDirExists take the path and checks if the same repo or not -func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool { - r, _ := git.PlainOpen(repoPath) - local, _ := r.Config() - return local.User.Name == ctr.owner // repo already cloned no need to rename and clone + return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) } // returns the auth object with username and github token as password diff --git a/v2/pkg/external/customtemplates/github_test.go b/v2/pkg/external/customtemplates/github_test.go index 60d469565..e21a62a90 100644 --- a/v2/pkg/external/customtemplates/github_test.go +++ b/v2/pkg/external/customtemplates/github_test.go @@ -30,6 +30,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { ctm.Download(context.Background()) - require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists") + require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists") require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists") } diff --git a/v2/pkg/protocols/common/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go index 43e0f4201..0e09aaf80 100644 --- a/v2/pkg/protocols/common/fuzz/parts.go +++ b/v2/pkg/protocols/common/fuzz/parts.go @@ -2,14 +2,13 @@ package fuzz import ( "context" - "io" "net/http" "strings" "github.com/corpix/uarand" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/retryablehttp-go" sliceutil "github.com/projectdiscovery/utils/slice" urlutil "github.com/projectdiscovery/utils/url" @@ -49,7 +48,6 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) requestURL.Params = temp if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil { err = qerr - gologger.Error().Msgf("Could not build request for query part rule %v: %s\n", rule, err) return false } cloned[i] = value // change back to previous value for temp @@ -89,7 +87,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL, DynamicValues: input.Values, } if !input.Callback(request) { - return io.EOF + return types.ErrNoMoreRequests } return nil } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index de419488e..612f4e33e 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,11 +2,11 @@ package generators import ( "bufio" - "io" "path/filepath" "strings" "github.com/pkg/errors" + pkgTypes "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/spf13/cast" ) @@ -60,7 +60,7 @@ func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]stri } lines = append(lines, text) } - if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { + if err := scanner.Err(); err != nil && !errors.Is(err, pkgTypes.ErrNoMoreRequests) { return lines, scanner.Err() } return lines, nil diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 9365d142c..b7f40eb41 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -1,7 +1,6 @@ package headless import ( - "io" "net/url" "strings" "time" @@ -22,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v2/pkg/types" urlutil "github.com/projectdiscovery/utils/url" ) @@ -206,7 +206,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads Values: payloads, BaseRequest: nil, }) - if err == io.EOF { + if err == types.ErrNoMoreRequests { return nil } if err != nil { diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 1b7684026..eecd9eadf 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -65,7 +65,7 @@ func (r *requestGenerator) Total() int { } // Make creates a http request for the provided input. -// It returns io.EOF as error when all the requests have been exhausted. +// It returns ErrNoMoreRequests as error when all the requests have been exhausted. func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { // value of `reqData` depends on the type of request specified in template // 1. If request is raw request = reqData contains raw request (i.e http request dump) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 127b7a0ee..04469edc1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -134,7 +134,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV ctx := request.newContext(input) generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues) if err != nil { - if err == io.EOF { + if err == types.ErrNoMoreRequests { break } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) @@ -301,7 +301,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous Values: generated.dynamicValues, BaseRequest: generated.request, }) - if err == io.EOF { + if err == types.ErrNoMoreRequests { return nil } if err != nil { @@ -354,7 +354,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) if err != nil { - if err == io.EOF { + if err == types.ErrNoMoreRequests { return true, nil } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) diff --git a/v2/pkg/protocols/http/utils/requtils.go b/v2/pkg/protocols/http/utils/requtils.go index cfb75c154..7be1f2259 100644 --- a/v2/pkg/protocols/http/utils/requtils.go +++ b/v2/pkg/protocols/http/utils/requtils.go @@ -13,6 +13,7 @@ var ( urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`) // regex to detect traling slash in path (not applicable to raw requests) trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`) + // ErrNoMoreRequests is internal error to ) // HasTrailingSlash returns true if path(that has default variables) has trailing slash diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index b3880c05f..ada0aed72 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -1,6 +1,7 @@ package types import ( + "io" "time" "github.com/projectdiscovery/goflags" @@ -9,6 +10,11 @@ import ( fileutil "github.com/projectdiscovery/utils/file" ) +var ( + // ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate + ErrNoMoreRequests = io.EOF +) + // Options contains the configuration options for nuclei scanner. type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths From 2cf574a9ba926d6c61b40f5e7ec6477bd000a711 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jul 2023 18:55:44 +0530 Subject: [PATCH 27/62] chore(deps): bump github.com/projectdiscovery/httpx in /v2 (#3912) Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.3.0 to 1.3.3. - [Release notes](https://github.com/projectdiscovery/httpx/releases) - [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/httpx/compare/v1.3.0...v1.3.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/httpx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 4 ++-- v2/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0c669bdc5..c64fb2659 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -70,7 +70,7 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.10 - github.com/projectdiscovery/httpx v1.3.0 + github.com/projectdiscovery/httpx v1.3.3 github.com/projectdiscovery/mapcidr v1.1.2 github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 @@ -125,7 +125,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/projectdiscovery/asnmap v1.0.4 // indirect - github.com/projectdiscovery/cdncheck v1.0.6 // indirect + github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/refraction-networking/utls v1.3.2 // indirect github.com/sashabaranov/go-openai v1.12.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 8593fb82e..b9902990c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -391,8 +391,8 @@ github.com/projectdiscovery/asnmap v1.0.4 h1:dmXrls7Y0Sdeb6cLlKGwdYX8h1K9q2iYOGX github.com/projectdiscovery/asnmap v1.0.4/go.mod h1:iTLDyYsblEwYHcLiKZCRx8Et+xV7NlvgGLusANpgegc= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6uGfPofVpC4c= -github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= +github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= +github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= @@ -409,8 +409,8 @@ github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoi github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= -github.com/projectdiscovery/httpx v1.3.0 h1:x7XbAUd1i3SL/SqVBZFM5EXv7YRXL58jdwonwxR4bvE= -github.com/projectdiscovery/httpx v1.3.0/go.mod h1:tglYwSUBz1zl2cC3NtjgURrI3cZsGepH1DLyQZZY2/M= +github.com/projectdiscovery/httpx v1.3.3 h1:JINmFbxKcP6xtFtmukaG+Js+ya5I/zY6RV9hkO2j4oI= +github.com/projectdiscovery/httpx v1.3.3/go.mod h1:5fvZBoI39IXwT62X9Q31aXpVWRsRbedCC0XZF4uT/yE= github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= github.com/projectdiscovery/interactsh v1.1.4/go.mod h1:rM8IEm6AAm68fWWExzBHjhBWfRhFYzR9gY5emOFiZCY= github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4tQsLPCrsA= From e5154d362af900e83b30fb79028bfab1a7f0c648 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 14 Jul 2023 16:09:32 +0200 Subject: [PATCH 28/62] fixing payload load (#3927) * fixing payload load * Added tests for load payloads edge-case + fixed error * Added separate flags for network and file sandbox * Fixed tests for payload loader * Fixed integration tests locally * readme update --------- Co-authored-by: Ice3man Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- README.md | 75 +++++----- README_ID.md | 138 ++++++++++++------ v2/cmd/nuclei/main.go | 3 +- .../protocols/common/generators/generators.go | 4 +- v2/pkg/protocols/common/generators/load.go | 10 +- .../protocols/common/generators/load_test.go | 16 +- .../protocols/common/protocolstate/state.go | 2 +- v2/pkg/protocols/dns/dns.go | 2 +- v2/pkg/protocols/headless/headless.go | 2 +- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/websocket/websocket.go | 2 +- v2/pkg/testutils/integration.go | 1 + v2/pkg/types/types.go | 6 +- 14 files changed, 164 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 0e089e3d6..4929cb7ca 100644 --- a/README.md +++ b/README.md @@ -41,11 +41,6 @@ Nuclei is used to send requests across targets based on a template, leading to z We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 300** security researchers and engineers. -| :exclamation: **Disclaimer** | -|---------------------------------| -| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. | -| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. | - ## How it works @@ -54,6 +49,11 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp +| :exclamation: **Disclaimer** | +|---------------------------------| +| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. | +| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. | + # Install Nuclei Nuclei requires **go1.20** to install successfully. Run the following command to install the latest version - @@ -157,7 +157,7 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED] + -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true) -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) -nm, -no-meta disable printing result metadata in cli output -ts, -timestamp enables printing timestamp in cli output @@ -169,35 +169,36 @@ OUTPUT: -jle, -jsonl-export string file to export results in JSONL(ine) format CONFIGURATIONS: - -config string path to the nuclei configuration file - -fr, -follow-redirects enable following redirects for http templates - -fhr, -follow-host-redirects follow redirects on the same host - -mr, -max-redirects int max number of redirects to follow for http templates (default 10) - -dr, -disable-redirects disable redirects for http templates - -rc, -report-config string nuclei reporting module configuration file - -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) - -V, -var value custom vars in key=value format - -r, -resolvers string file containing resolver list for nuclei - -sr, -system-resolvers use system DNS resolving as error fallback - -dc, -disable-clustering disable clustering of requests - -passive enable passive HTTP response processing mode - -fh2, -force-http2 force http2 connection on requests - -ev, -env-vars enable environment variables to be used in template - -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts - -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts - -sml, -show-match-line show match lines for file templates, works with extractors only - -ztls use ztls library with autofallback to standard one for tls13 - -sni string tls sni hostname to use (default: input domain name) - -sandbox sandbox nuclei for safe templates execution - -i, -interface string network interface to use for network scan - -at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb) - -sip, -source-ip string source ip address to use for network scan - -config-directory string override the default config path ($home/.config) - -rsr, -response-size-read int max response size to read in bytes (default 10485760) - -rss, -response-size-save int max response size to read in bytes (default 1048576) - -reset reset removes all nuclei configuration and data files (including nuclei-templates) - -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization + -config string path to the nuclei configuration file + -fr, -follow-redirects enable following redirects for http templates + -fhr, -follow-host-redirects follow redirects on the same host + -mr, -max-redirects int max number of redirects to follow for http templates (default 10) + -dr, -disable-redirects disable redirects for http templates + -rc, -report-config string nuclei reporting module configuration file + -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) + -V, -var value custom vars in key=value format + -r, -resolvers string file containing resolver list for nuclei + -sr, -system-resolvers use system DNS resolving as error fallback + -dc, -disable-clustering disable clustering of requests + -passive enable passive HTTP response processing mode + -fh2, -force-http2 force http2 connection on requests + -ev, -env-vars enable environment variables to be used in template + -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -sml, -show-match-line show match lines for file templates, works with extractors only + -ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default + -sni string tls sni hostname to use (default: input domain name) + -lfa, -allow-local-file-access allows file (payload) access anywhere on the system + -lna, -restrict-local-network-access blocks connections to the local / private network + -i, -interface string network interface to use for network scan + -at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb) + -sip, -source-ip string source ip address to use for network scan + -config-directory string override the default config path ($home/.config) + -rsr, -response-size-read int max response size to read in bytes (default 10485760) + -rss, -response-size-save int max response size to read in bytes (default 1048576) + -reset reset removes all nuclei configuration and data files (including nuclei-templates) + -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) @@ -215,10 +216,10 @@ FUZZING: UNCOVER: -uc, -uncover enable uncover engine -uq, -uncover-query string[] uncover search query - -ue, -uncover-engine string[] uncover search engine (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan) + -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan) -uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port") -ul, -uncover-limit int uncover results to return (default 100) - -ucd, -uncover-delay int delay between uncover query requests in seconds (0 to disable) (default 1) + -ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60) RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) diff --git a/README_ID.md b/README_ID.md index df82bfd38..04fcd3e8c 100644 --- a/README_ID.md +++ b/README_ID.md @@ -90,13 +90,15 @@ Nuclei is a fast, template based vulnerability scanner focusing on extensive configurability, massive extensibility and ease of use. Usage: - nuclei [flags] + ./nuclei [flags] Flags: TARGET: - -u, -target string[] target URLs/hosts to scan - -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) - -resume string Resume scan using resume.cfg (clustering will be disabled) + -u, -target string[] target URLs/hosts to scan + -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) + -resume string resume scan using resume.cfg (clustering will be disabled) + -sa, -scan-all-ips scan all the IP's associated with dns record + -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TEMPLATES: -nt, -new-templates run only new templates added in latest nuclei-templates release @@ -107,7 +109,8 @@ TEMPLATES: -w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file) -wu, -workflow-url string[] list of workflow urls to run (comma-separated, file) -validate validate the passed templates to nuclei - -nss, -no-strict-syntax Disable strict syntax check on templates + -nss, -no-strict-syntax disable strict syntax check on templates + -td, -template-display displays the templates content -tl list all available templates FILTERING: @@ -122,8 +125,8 @@ FILTERING: -em, -exclude-matchers string[] template matchers to exclude in result -s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical, unknown -es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown - -pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois - -ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois + -pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois + -ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois -tc, -template-condition string[] templates to run based on expression condition OUTPUT: @@ -133,41 +136,48 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED] + -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true) -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) -nm, -no-meta disable printing result metadata in cli output - -nts, -no-timestamp disable printing timestamp in cli output + -ts, -timestamp enables printing timestamp in cli output -rdb, -report-db string nuclei reporting database (always use this to persist report data) -ms, -matcher-status display match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format - -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans - -jle, -jsonl-export string file to export results in JSONL(ine) format as a list of line-delimited JSON objects + -je, -json-export string file to export results in JSON format + -jle, -jsonl-export string file to export results in JSONL(ine) format CONFIGURATIONS: - -config string path to the nuclei configuration file - -fr, -follow-redirects enable following redirects for http templates - -fhr, -follow-host-redirects follow redirects on the same host - -mr, -max-redirects int max number of redirects to follow for http templates (default 10) - -dr, -disable-redirects disable redirects for http templates - -rc, -report-config string nuclei reporting module configuration file - -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) - -V, -var value custom vars in key=value format - -r, -resolvers string file containing resolver list for nuclei - -sr, -system-resolvers use system DNS resolving as error fallback - -passive enable passive HTTP response processing mode - -ev, -env-vars enable environment variables to be used in template - -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts - -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts - -sml, -show-match-line show match lines for file templates, works with extractors only - -ztls use ztls library with autofallback to standard one for tls13 - -sni string tls sni hostname to use (default: input domain name) - -i, -interface string network interface to use for network scan - -sip, -source-ip string source ip address to use for network scan - -config-directory string Override the default config path ($home/.config) - -rsr, -response-size-read int max response size to read in bytes (default 10485760) - -rss, -response-size-save int max response size to save in bytes (default 10485760) + -config string path to the nuclei configuration file + -fr, -follow-redirects enable following redirects for http templates + -fhr, -follow-host-redirects follow redirects on the same host + -mr, -max-redirects int max number of redirects to follow for http templates (default 10) + -dr, -disable-redirects disable redirects for http templates + -rc, -report-config string nuclei reporting module configuration file + -H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file) + -V, -var value custom vars in key=value format + -r, -resolvers string file containing resolver list for nuclei + -sr, -system-resolvers use system DNS resolving as error fallback + -dc, -disable-clustering disable clustering of requests + -passive enable passive HTTP response processing mode + -fh2, -force-http2 force http2 connection on requests + -ev, -env-vars enable environment variables to be used in template + -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -sml, -show-match-line show match lines for file templates, works with extractors only + -ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default + -sni string tls sni hostname to use (default: input domain name) + -lfa, -allow-local-file-access allows file (payload) access anywhere on the system + -lna, -restrict-local-network-access blocks connections to the local / private network + -i, -interface string network interface to use for network scan + -at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb) + -sip, -source-ip string source ip address to use for network scan + -config-directory string override the default config path ($home/.config) + -rsr, -response-size-read int max response size to read in bytes (default 10485760) + -rss, -response-size-save int max response size to read in bytes (default 1048576) + -reset reset removes all nuclei configuration and data files (including nuclei-templates) + -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) @@ -178,6 +188,18 @@ INTERACTSH: -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates +FUZZING: + -ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix) + -fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single) + +UNCOVER: + -uc, -uncover enable uncover engine + -uq, -uncover-query string[] uncover search query + -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan) + -uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port") + -ul, -uncover-limit int uncover results to return (default 100) + -ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60) + RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) -rlm, -rate-limit-minute int maximum number of requests to send per minute @@ -189,22 +211,24 @@ RATE-LIMIT: OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 10) -retries int number of times to retry a failed request (default 1) - -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443 + -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443) -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) -te, -track-error string[] adds given error to max-host-error watchlist (standard, file) -nmhe, -no-mhe disable skipping host from scan based on errors -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path - -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + -project-path string set a specific project path (default "/tmp") + -spm, -stop-at-first-match stop processing HTTP requests after the first match (may break template/workflow logic) -stream stream mode - start elaborating without sorting the input + -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout duration timeout on input read (default 3m0s) - -no-stdin Disable Stdin processing + -nh, -no-httpx disable httpx probing for non-url input + -no-stdin disable stdin processing HEADLESS: - -headless enable templates that require headless browser support (root user on linux will disable sandbox) + -headless enable templates that require headless browser support (root user on Linux will disable sandbox) -page-timeout int seconds to wait for each page in headless mode (default 20) -sb, -show-browser show the browser on the screen when running templates with headless mode - -sc, -system-chrome Use local installed chrome browser instead of nuclei installed + -sc, -system-chrome use local installed Chrome browser instead of nuclei installed -lha, -list-headless-action list available headless actions DEBUG: @@ -221,22 +245,46 @@ DEBUG: -v, -verbose show verbose output -profile-mem string optional nuclei memory profile dump file -vv display templates loaded for scan + -svd, -show-var-dump show variables dump for debugging -ep, -enable-pprof enable pprof debugging server -tv, -templates-version shows the version of the installed nuclei-templates -hc, -health-check run diagnostic check up UPDATE: - -update update nuclei engine to the latest released version - -ut, -update-templates update nuclei-templates to latest released version - -ud, -update-directory string overwrite the default directory to install nuclei-templates - -duc, -disable-update-check disable automatic nuclei/templates update check + -up, -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-template-dir string custom directory to install / update nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan - -sj, -stats-json dispaly statistics in JSONL(ines) format + -sj, -stats-json display statistics in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) -m, -metrics expose nuclei metrics on a port -mp, -metrics-port int port to expose nuclei metrics on (default 9092) + +CLOUD: + -cloud run scan on nuclei cloud + -ads, -add-datasource string add specified data source (s3,github) + -atr, -add-target string add target(s) to cloud + -atm, -add-template string add template(s) to cloud + -lsn, -list-scan list previous cloud scans + -lso, -list-output string list scan output by scan id + -ltr, -list-target list cloud target by id + -ltm, -list-template list cloud template by id + -lds, -list-datasource list cloud datasource by id + -lrs, -list-reportsource list reporting sources + -dsn, -delete-scan string delete cloud scan by id + -dtr, -delete-target string delete target(s) from cloud + -dtm, -delete-template string delete template(s) from cloud + -dds, -delete-datasource string delete specified data source + -drs, -disable-reportsource string disable specified reporting source + -ers, -enable-reportsource string enable specified reporting source + -gtr, -get-target string get target content by id + -gtm, -get-template string get template content by id + -nos, -no-store disable scan/output storage on cloud + -no-tables do not display pretty-printed tables + -limit int limit the number of output to display (default 100) ``` ### Menjalankan Nuclei diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 981db8499..bd68cfc30 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -203,7 +203,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"), - flagSet.BoolVar(&options.Sandbox, "sandbox", false, "sandbox nuclei for safe templates execution"), + flagSet.BoolVarP(&options.AllowLocalFileAccess, "allow-local-file-access", "lfa", false, "allows file (payload) access anywhere on the system"), + flagSet.BoolVarP(&options.RestrictLocalNetworkAccess, "restrict-local-network-access", "lna", false, "blocks connections to the local / private network"), flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"), flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"), flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"), diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 9cc8a9761..98fc6aa13 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -17,7 +17,7 @@ type PayloadGenerator struct { } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, sandbox bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, allowLocalFileAccess bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -43,7 +43,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, sandbox) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, allowLocalFileAccess) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 612f4e33e..92ec93201 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -11,7 +11,7 @@ import ( ) // loadPayloads loads the input payloads from a map to a data map -func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, sandbox bool) (map[string][]string, error) { +func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, allowLocalFileAccess bool) (map[string][]string, error) { loadedPayloads := make(map[string][]string) for name, payload := range payloads { @@ -22,9 +22,13 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, if len(elements) >= 2 { loadedPayloads[name] = elements } else { - if sandbox { + if !allowLocalFileAccess { pt = filepath.Clean(pt) - templatePathDir := filepath.Dir(templatePath) + templateAbsPath, err := filepath.Abs(templatePath) + if err != nil { + return nil, errors.Wrap(err, "could not get absolute path") + } + templatePathDir := filepath.Dir(templateAbsPath) if !(templatePathDir != "/" && strings.HasPrefix(pt, templatePathDir)) && !strings.HasPrefix(pt, templateDirectory) { return nil, errors.New("denied payload file path specified") } diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index cecdde9b5..28803b097 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -25,14 +25,20 @@ func TestLoadPayloads(t *testing.T) { t.Run("templates-directory", func(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/test", tempdir, true) + }, "/test", tempdir, false) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) + t.Run("templates-path-relative", func(t *testing.T) { + _, err := generator.loadPayloads(map[string]interface{}{ + "new": "../../../../../../../../../etc/passwd", + }, ".", tempdir, false) + require.Error(t, err, "could load payloads") + }) t.Run("template-directory", func(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, filepath.Join(tempdir, "test.yaml"), "/test", true) + }, filepath.Join(tempdir, "test.yaml"), "/test", false) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) @@ -42,19 +48,19 @@ func TestLoadPayloads(t *testing.T) { } _, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", false) + }, "/random", "/test", true) require.NoError(t, err, "could load payloads") }) t.Run("invalid", func(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", true) + }, "/random", "/test", false) require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") values, err = generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/random", "/test", true) + }, "/random", "/test", false) require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") }) diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index 4e9df494a..95293a519 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -91,7 +91,7 @@ func Init(options *types.Options) error { if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } - if options.Sandbox { + if options.RestrictLocalNetworkAccess { opts.Deny = append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...) } opts.WithDialerHistory = true diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index dad2ef750..1cc8302e5 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -172,7 +172,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 06203d3ed..0d1d09e5e 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -106,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { if len(request.Payloads) > 0 { var err error - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 99afe8c49..552d9229b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -353,7 +353,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 86292a16c..c344538f5 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index e5f03696f..26468fd35 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -106,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index 42777fd8f..56d32d5c3 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -52,6 +52,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro cmd.Args = append(cmd.Args, "-duc") // disable auto updates cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") + cmd.Args = append(cmd.Args, "-allow-local-file-access") if debug { cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index ada0aed72..ad6986638 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -285,8 +285,10 @@ type Options struct { ClientCAFile string // Deprecated: Use ZTLS library ZTLS bool - // Sandbox enables sandboxed nuclei template execution - Sandbox bool + // AllowLocalFileAccess allows local file access from templates payloads + AllowLocalFileAccess bool + // RestrictLocalNetworkAccess restricts local network access from templates requests + RestrictLocalNetworkAccess bool // ShowMatchLine enables display of match line number ShowMatchLine bool // EnablePprof enables exposing pprof runtime information with a webserver. From 66f0dc735ce90fabd16c4d0a3f4e79c1c3ba0999 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 14 Jul 2023 17:54:12 +0200 Subject: [PATCH 29/62] Adding jarm helper via dsl (#3906) * Adding jarm helper via dsl * adding test * removing debug file * fixing tests --------- Co-authored-by: Sandeep Singh --- integration_tests/http/dsl-functions.yaml | 1 + v2/cmd/integration-test/http.go | 2 +- v2/go.mod | 6 +++--- v2/go.sum | 14 +++++++------- v2/pkg/protocols/common/expressions/variables.go | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/integration_tests/http/dsl-functions.yaml b/integration_tests/http/dsl-functions.yaml index 72dffbfc5..5afce178e 100644 --- a/integration_tests/http/dsl-functions.yaml +++ b/integration_tests/http/dsl-functions.yaml @@ -98,6 +98,7 @@ requests: 85: {{split("ab,cd,efg", ",", 2)}} 86: {{ip_format('127.0.0.1', 3)}} 87: {{ip_format('127.0.1.0', 11)}} + 88: {{jarm('scanme.sh:443')}} extractors: - type: regex name: results diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 9a3046b48..a1b67c575 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -362,7 +362,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error { resultPart = stringsutil.TrimPrefixAny(resultPart, "/", " ", "[") extracted := strings.Split(resultPart, ",") - numberOfDslFunctions := 87 + numberOfDslFunctions := 88 if len(extracted) != numberOfDslFunctions { return errors.New("incorrect number of results") } diff --git a/v2/go.mod b/v2/go.mod index c64fb2659..48d7535a9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,7 +31,7 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.23.5 // indirect + github.com/shirou/gopsutil/v3 v3.23.6 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 @@ -66,7 +66,7 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.12 + github.com/projectdiscovery/dsl v0.0.13-0.20230705084047-7ddbef70070b github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.10 @@ -77,7 +77,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.0 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.40 + github.com/projectdiscovery/utils v0.0.41-0.20230705082547-236cfa9298ab github.com/projectdiscovery/wappalyzergo v0.0.104 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 diff --git a/v2/go.sum b/v2/go.sum index b9902990c..d071b43f4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -395,8 +395,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= -github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw= +github.com/projectdiscovery/dsl v0.0.13-0.20230705084047-7ddbef70070b h1:q0kiI23W1MMdxnoN2KVJVib8NJIVBFiZQ8Rf2QlPvPY= +github.com/projectdiscovery/dsl v0.0.13-0.20230705084047-7ddbef70070b/go.mod h1:f5+1OwmqW9kcIkAfSZDqJEnA8/PFrct7bJTIlOjsXpE= github.com/projectdiscovery/fastdialer v0.0.33 h1:FKXZjJme5nhgnnRL0Y4KjZ5YXKg03dX5ZJYFJV3LOwU= github.com/projectdiscovery/fastdialer v0.0.33/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -434,8 +434,8 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.40 h1:vChjk6PnSb6+nUaP3/GS8LjbGGlrUpFiZBbeoBa9Sww= -github.com/projectdiscovery/utils v0.0.40/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= +github.com/projectdiscovery/utils v0.0.41-0.20230705082547-236cfa9298ab h1:KcbRfus364It55dhAUpbqFHfyCuIa8Ls/9QzMWYKq78= +github.com/projectdiscovery/utils v0.0.41-0.20230705082547-236cfa9298ab/go.mod h1:DTFCMSLh8FanDZIrzOwTo3AIv1K4w0PDELi41mjwLiw= github.com/projectdiscovery/wappalyzergo v0.0.104 h1:hdda6WxAzXVpLBbJW1sLqrwOXHn0prP9IYFY7dfCMjE= github.com/projectdiscovery/wappalyzergo v0.0.104/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -460,8 +460,8 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= +github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -496,7 +496,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -704,6 +703,7 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 7b8f0b595..c4d1cb61f 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -120,7 +120,7 @@ func hasLiteralsOnly(data string) bool { if err != nil { return false } - if err == nil && expr != nil { + if expr != nil { _, err = expr.Evaluate(nil) return err == nil } From 08e1ab9dddc848b7e0dc836cf37b0f8b065092a2 Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Mon, 17 Jul 2023 15:23:10 -0400 Subject: [PATCH 30/62] Disable Template Locations (#3705) (#3926) * Download override option definition * Update the variable names for consistency * Add checks for custom template disable flags * Environment variable controlled template downloads * Switch env naming per feedback from @ehsandeep * minor changes --------- Co-authored-by: Tarun Koyalwar --- v2/internal/installer/template.go | 54 +++++++++++-------- v2/internal/runner/options.go | 26 +++++++-- v2/internal/runner/runner.go | 5 +- v2/pkg/catalog/config/constants.go | 9 ++-- v2/pkg/catalog/config/nucleiconfig.go | 2 +- v2/pkg/external/customtemplates/azure_blob.go | 2 +- v2/pkg/external/customtemplates/github.go | 14 +++-- v2/pkg/external/customtemplates/gitlab.go | 2 +- v2/pkg/external/customtemplates/s3.go | 2 +- v2/pkg/types/types.go | 10 ++++ 10 files changed, 85 insertions(+), 41 deletions(-) diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go index 12dd85a55..b9eda1d0e 100644 --- a/v2/internal/installer/template.go +++ b/v2/internal/installer/template.go @@ -57,7 +57,9 @@ func (t *templateUpdateResults) String() string { // TemplateManager is a manager for templates. // It downloads / updates / installs templates. type TemplateManager struct { - CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates + CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates + DisablePublicTemplates bool // if true, + // public templates are not downloaded from the GitHub nuclei-templates repository } // FreshInstallIfNotExists installs templates if they are not already installed @@ -78,7 +80,7 @@ func (t *TemplateManager) FreshInstallIfNotExists() error { // UpdateIfOutdated updates templates if they are outdated func (t *TemplateManager) UpdateIfOutdated() error { - // if folder does not exist, its a fresh install and not update + // if the templates folder does not exist, it's a fresh installation and do not update if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { return t.FreshInstallIfNotExists() } @@ -95,12 +97,16 @@ func (t *TemplateManager) installTemplatesAt(dir string) error { return errorutil.NewWithErr(err).Msgf("failed to create directory at %s", dir) } } + if t.DisablePublicTemplates { + gologger.Info().Msgf("Skipping installation of public nuclei-templates") + return nil + } ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName) if err != nil { return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) } // write templates to disk - if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir) } gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir) @@ -109,10 +115,14 @@ func (t *TemplateManager) installTemplatesAt(dir string) error { // updateTemplatesAt updates templates at given directory func (t *TemplateManager) updateTemplatesAt(dir string) error { - // firstly read checksums from .checksum file these are used to generate stats + if t.DisablePublicTemplates { + gologger.Info().Msgf("Skipping update of public nuclei-templates") + return nil + } + // firstly, read checksums from .checksum file these are used to generate stats oldchecksums, err := t.getChecksumFromDir(dir) if err != nil { - // if something went wrong overwrite all files + // if something went wrong, overwrite all files oldchecksums = make(map[string]string) } @@ -124,7 +134,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error { gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName()) // write templates to disk - if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { return err } @@ -173,9 +183,9 @@ func (t *TemplateManager) summarizeChanges(old, new map[string]string) *template return results } -// getAbsoluteFilePath returns absolute path where a file should be written based on given uri(i.e files in zip) -// if returned path is empty, it means that file should not be written and skipped -func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.FileInfo) string { +// getAbsoluteFilePath returns an absolute path where a file should be written based on given uri(i.e., files in zip) +// if a returned path is empty, it means that file should not be written and skipped +func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.FileInfo) string { // overwrite .nuclei-ignore everytime nuclei-templates are downloaded if f.Name() == config.NucleiIgnoreFileName { return config.DefaultConfig.GetIgnoreFilePath() @@ -194,7 +204,7 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File if index == -1 { // zip files does not have directory at all , in this case log error but continue gologger.Warning().Msgf("failed to get directory name from uri: %s", uri) - return filepath.Join(templatedir, uri) + return filepath.Join(templateDir, uri) } // seperator is also included in rootDir rootDirectory := uri[:index+1] @@ -205,14 +215,14 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File return "" } - newPath := filepath.Clean(filepath.Join(templatedir, relPath)) + newPath := filepath.Clean(filepath.Join(templateDir, relPath)) - if !strings.HasPrefix(newPath, templatedir) { + if !strings.HasPrefix(newPath, templateDir) { // we don't allow LFI return "" } - if newPath == templatedir || newPath == templatedir+string(os.PathSeparator) { + if newPath == templateDir || newPath == templateDir+string(os.PathSeparator) { // skip writing the folder itself since it already exists return "" } @@ -228,12 +238,12 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File } // writeChecksumFileInDir is actual method responsible for writing all templates to directory -func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { - LocaltemplatesIndex, err := config.GetNucleiTemplatesIndex() +func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { + localTemplatesIndex, err := config.GetNucleiTemplatesIndex() if err != nil { gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err) - if LocaltemplatesIndex == nil { - LocaltemplatesIndex = map[string]string{} // no-op + if localTemplatesIndex == nil { + localTemplatesIndex = map[string]string{} // no-op } } @@ -253,10 +263,10 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo // instead of creating it from scratch id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri) if id != "" { - // based on template id, check if we are updating path of official nuclei template - if oldPath, ok := LocaltemplatesIndex[id]; ok { + // based on template id, check if we are updating a path of official nuclei template + if oldPath, ok := localTemplatesIndex[id]; ok { if oldPath != writePath { - // write new template at new path and delete old template + // write new template at a new path and delete old template if err := os.WriteFile(writePath, bin, f.Mode()); err != nil { return errorutil.NewWithErr(err).Msgf("failed to write file %s", uri) } @@ -303,12 +313,12 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index") } - // after installation create and write checksums to .checksum file + // after installation, create and write checksums to .checksum file return t.writeChecksumFileInDir(dir) } // getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) -// if .checksum file does not exist checksums are calculated and returned +// if .checksum file does not exist, checksums are calculated and returned func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, error) { checksumFilePath := config.DefaultConfig.GetChecksumFilePath() if fileutil.FileExists(checksumFilePath) { diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 7c93c46dd..3da37f45c 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -29,7 +29,7 @@ import ( func ConfigureOptions() error { // with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions - // if file has extension `.yaml,.json` we consider those as strings and not files to be read + // if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read isFromFileFunc := func(s string) bool { return !config.IsTemplate(s) } @@ -136,7 +136,7 @@ func validateOptions(options *types.Options) error { validateCertificatePaths(options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile) } // Verify AWS secrets are passed if a S3 template bucket is passed - if options.AwsBucketName != "" && options.UpdateTemplates { + if options.AwsBucketName != "" && options.UpdateTemplates && !options.AwsTemplateDisableDownload { missing := validateMissingS3Options(options) if missing != nil { return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ",")) @@ -144,7 +144,7 @@ func validateOptions(options *types.Options) error { } // Verify Azure connection configuration is passed if the Azure template bucket is passed - if options.AzureContainerName != "" && options.UpdateTemplates { + if options.AzureContainerName != "" && options.UpdateTemplates && !options.AzureTemplateDisableDownload { missing := validateMissingAzureOptions(options) if missing != nil { return fmt.Errorf("azure connection details are missing. Please provide %s", strings.Join(missing, ",")) @@ -152,7 +152,7 @@ func validateOptions(options *types.Options) error { } // Verify that all GitLab options are provided if the GitLab server or token is provided - if len(options.GitLabTemplateRepositoryIDs) != 0 && options.UpdateTemplates { + if len(options.GitLabTemplateRepositoryIDs) != 0 && options.UpdateTemplates && !options.GitLabTemplateDisableDownload { missing := validateMissingGitLabOptions(options) if missing != nil { return fmt.Errorf("gitlab server details are missing. Please provide %s", strings.Join(missing, ",")) @@ -292,7 +292,7 @@ func configureOutput(options *types.Options) { logutil.DisableDefaultLogger() } -// loadResolvers loads resolvers from both user provided flag and file +// loadResolvers loads resolvers from both user-provided flags and file func loadResolvers(options *types.Options) { if options.ResolversFile == "" { return @@ -396,4 +396,20 @@ func readEnvInputVars(options *types.Options) { options.AzureClientID = os.Getenv("AZURE_CLIENT_ID") options.AzureClientSecret = os.Getenv("AZURE_CLIENT_SECRET") options.AzureServiceURL = os.Getenv("AZURE_SERVICE_URL") + + // General options to disable the template download locations from being used. + // This will override the default behavior of downloading templates from the default locations as well as the + // custom locations. + // The primary use-case is when the user wants to use custom templates only and does not want to download any + // templates from the default locations or is unable to connect to the public internet. + options.PublicTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_PUBLIC_DOWNLOAD") + options.GitHubTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITHUB_DOWNLOAD") + options.GitLabTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITLAB_DOWNLOAD") + options.AwsTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AWS_DOWNLOAD") + options.AzureTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AZURE_DOWNLOAD") +} + +func getBoolEnvValue(key string) bool { + value := os.Getenv(key) + return strings.EqualFold(value, "true") } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 3a73fdc9e..711ea769e 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -112,7 +112,10 @@ func New(options *types.Options) (*Runner, error) { // Check for template updates and update if available. // If the custom templates manager is not nil, we will install custom templates if there is a fresh installation - tm := &installer.TemplateManager{CustomTemplates: ctm} + tm := &installer.TemplateManager{ + CustomTemplates: ctm, + DisablePublicTemplates: options.PublicTemplateDisableDownload, + } if err := tm.FreshInstallIfNotExists(); err != nil { gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err) } diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 46166eaa0..464d5f268 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -14,7 +14,7 @@ const ( NucleiTemplatesIndexFileName = ".templates-index" // contains index of official nuclei templates NucleiTemplatesCheckSumFileName = ".checksum" NewTemplateAdditionsFileName = ".new-additions" - CLIConifgFileName = "config.yaml" + CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei Version = `v2.9.8` @@ -25,11 +25,12 @@ const ( CustomGitLabTemplatesDirName = "gitlab" ) -// IsOutdatedVersion compares two versions and returns true if the current version is outdated +// IsOutdatedVersion compares two versions and returns true +// if the current version is outdated func IsOutdatedVersion(current, latest string) bool { if latest == "" { - // if pdtm api call failed it's assumed that current version is outdated - // and it will be confirmed while updating from github + // if pdtm api call failed it's assumed that the current version is outdated + // and it will be confirmed while updating from GitHub // this fixes `version string empty` errors return true } diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index de2e5044a..baf7acb43 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -135,7 +135,7 @@ func (c *Config) GetChecksumFilePath() string { // GetCLIOptsConfigFilePath returns the nuclei cli config file path func (c *Config) GetFlagsConfigFilePath() string { - return filepath.Join(c.configDir, CLIConifgFileName) + return filepath.Join(c.configDir, CLIConfigFileName) } // GetNewAdditions returns new template additions in current template release diff --git a/v2/pkg/external/customtemplates/azure_blob.go b/v2/pkg/external/customtemplates/azure_blob.go index 4e405caf5..442d03746 100644 --- a/v2/pkg/external/customtemplates/azure_blob.go +++ b/v2/pkg/external/customtemplates/azure_blob.go @@ -25,7 +25,7 @@ type customTemplateAzureBlob struct { // NewAzureProviders creates a new Azure Blob Storage provider for downloading custom templates func NewAzureProviders(options *types.Options) ([]*customTemplateAzureBlob, error) { providers := []*customTemplateAzureBlob{} - if options.AzureContainerName != "" { + if options.AzureContainerName != "" && !options.AzureTemplateDisableDownload { // Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL) if err != nil { diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index d9317ca3a..8a498b259 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -58,11 +58,15 @@ func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) { } } -// NewGithubProviders returns new instance of github providers for downloading custom templates +// NewGithubProviders returns new instance of GitHub providers for downloading custom templates func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, error) { providers := []*customTemplateGithubRepo{} gitHubClient := getGHClientIncognito() + if options.GitHubTemplateDisableDownload { + return providers, nil + } + for _, repoName := range options.GithubTemplateRepo { owner, repo, err := getOwnerAndRepo(repoName) if err != nil { @@ -86,8 +90,8 @@ func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, er } // getOwnerAndRepo returns the owner, repo, err from the given string -// eg. it takes input projectdiscovery/nuclei-templates and -// returns owner=> projectdiscovery , repo => nuclei-templates +// e.g., it takes input projectdiscovery/nuclei-templates and +// returns owner => projectdiscovery, repo => nuclei-templates func getOwnerAndRepo(reponame string) (owner string, repo string, err error) { s := strings.Split(reponame, "/") if len(s) != 2 { @@ -118,7 +122,7 @@ getRepo: return repo, nil } -// download the git repo to given path +// download the git repo to a given path func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) error { r, err := git.PlainClone(clonePath, false, &git.CloneOptions{ URL: ctr.gitCloneURL, @@ -127,7 +131,7 @@ func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) er if err != nil { return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error()) } - // Add the user as well in the config. By default user is not set + // Add the user as well in the config. By default, user is not set config, _ := r.Storer.Config() config.User.Name = ctr.owner return r.SetConfig(config) diff --git a/v2/pkg/external/customtemplates/gitlab.go b/v2/pkg/external/customtemplates/gitlab.go index 20e7c41d8..db272c98c 100644 --- a/v2/pkg/external/customtemplates/gitlab.go +++ b/v2/pkg/external/customtemplates/gitlab.go @@ -24,7 +24,7 @@ type customTemplateGitLabRepo struct { // NewGitlabProviders returns a new list of GitLab providers for downloading custom templates func NewGitlabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) { providers := []*customTemplateGitLabRepo{} - if options.GitLabToken != "" { + if options.GitLabToken != "" && !options.GitLabTemplateDisableDownload { // Establish a connection to GitLab and build a client object with which to download templates from GitLab gitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken) if err != nil { diff --git a/v2/pkg/external/customtemplates/s3.go b/v2/pkg/external/customtemplates/s3.go index 0804590d2..f5421672f 100644 --- a/v2/pkg/external/customtemplates/s3.go +++ b/v2/pkg/external/customtemplates/s3.go @@ -61,7 +61,7 @@ func (bk *customTemplateS3Bucket) Update(ctx context.Context) { // NewS3Providers returns a new instances of a s3 providers for downloading custom templates func NewS3Providers(options *types.Options) ([]*customTemplateS3Bucket, error) { providers := []*customTemplateS3Bucket{} - if options.AwsBucketName != "" { + if options.AwsBucketName != "" && !options.AwsTemplateDisableDownload { s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("error downloading s3 bucket %s", options.AwsBucketName) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index ad6986638..10411ea3f 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -337,16 +337,22 @@ type Options struct { ScanAllIPs bool // IPVersion to scan (4,6) IPVersion goflags.StringSlice + // PublicTemplateDisableDownload disables downloading templates from the nuclei-templates public repository + PublicTemplateDisableDownload bool // GitHub token used to clone/pull from private repos for custom templates GithubToken string // GithubTemplateRepo is the list of custom public/private templates GitHub repos GithubTemplateRepo []string + // GitHubTemplateDisableDownload disables downloading templates from custom GitHub repositories + GitHubTemplateDisableDownload bool // GitLabServerURL is the gitlab server to use for custom templates GitLabServerURL string // GitLabToken used to clone/pull from private repos for custom templates GitLabToken string // GitLabTemplateRepositoryIDs is the comma-separated list of custom gitlab repositories IDs GitLabTemplateRepositoryIDs []int + // GitLabTemplateDisableDownload disables downloading templates from custom GitLab repositories + GitLabTemplateDisableDownload bool // AWS access key for downloading templates from S3 bucket AwsAccessKey string // AWS secret key for downloading templates from S3 bucket @@ -355,6 +361,8 @@ type Options struct { AwsBucketName string // AWS Region name where AWS S3 bucket is located AwsRegion string + // AwsTemplateDisableDownload disables downloading templates from AWS S3 buckets + AwsTemplateDisableDownload bool // AzureContainerName for downloading templates from Azure Blob Storage. Example: templates AzureContainerName string // AzureTenantID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000 @@ -365,6 +373,8 @@ type Options struct { AzureClientSecret string // AzureServiceURL for downloading templates from Azure Blob Storage. Example: https://XXXXXXXXXX.blob.core.windows.net/ AzureServiceURL string + // AzureTemplateDisableDownload disables downloading templates from Azure Blob Storage + AzureTemplateDisableDownload bool // Scan Strategy (auto,hosts-spray,templates-spray) ScanStrategy string // Fuzzing Type overrides template level fuzzing-type configuration From 09a1200a68b9ae99ddd3b83c7d3542ea92f4f57e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 01:10:59 +0530 Subject: [PATCH 31/62] chore(deps): bump github.com/projectdiscovery/dsl in /v2 (#3934) Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.13-0.20230705084047-7ddbef70070b to 0.0.14. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/commits/v0.0.14) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 6 +++--- v2/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 48d7535a9..78b7cf623 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -66,10 +66,10 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.13-0.20230705084047-7ddbef70070b + github.com/projectdiscovery/dsl v0.0.14 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 - github.com/projectdiscovery/gologger v1.1.10 + github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/httpx v1.3.3 github.com/projectdiscovery/mapcidr v1.1.2 github.com/projectdiscovery/ratelimit v0.0.9 @@ -128,7 +128,7 @@ require ( github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/refraction-networking/utls v1.3.2 // indirect - github.com/sashabaranov/go-openai v1.12.0 // indirect + github.com/sashabaranov/go-openai v1.13.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index d071b43f4..286c76f72 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -395,8 +395,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.13-0.20230705084047-7ddbef70070b h1:q0kiI23W1MMdxnoN2KVJVib8NJIVBFiZQ8Rf2QlPvPY= -github.com/projectdiscovery/dsl v0.0.13-0.20230705084047-7ddbef70070b/go.mod h1:f5+1OwmqW9kcIkAfSZDqJEnA8/PFrct7bJTIlOjsXpE= +github.com/projectdiscovery/dsl v0.0.14 h1:CAxCoYbIEBCuINiMR1UKA1v6ifmub3P5hCwzBmmkh0c= +github.com/projectdiscovery/dsl v0.0.14/go.mod h1:3K2GmExpriruVHsVJmsTugxR7H9wVpUo8/+jWXXbSSw= github.com/projectdiscovery/fastdialer v0.0.33 h1:FKXZjJme5nhgnnRL0Y4KjZ5YXKg03dX5ZJYFJV3LOwU= github.com/projectdiscovery/fastdialer v0.0.33/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -405,8 +405,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goflags v0.1.10 h1:Gompf8JDy8y+5c4eWlc70KKtPuDH/hqFB3tMeHcMiKk= github.com/projectdiscovery/goflags v0.1.10/go.mod h1:MHEkqm3XgxBf5fK4gr3IXsj6VeLTq4qJYGC/4JRYQ74= -github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoii4P2yDovqck= -github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= +github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= +github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= github.com/projectdiscovery/httpx v1.3.3 h1:JINmFbxKcP6xtFtmukaG+Js+ya5I/zY6RV9hkO2j4oI= @@ -453,8 +453,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.12.0 h1:aRNHH0gtVfrpIaEolD0sWrLLRnYQNK4cH/bIAHwL8Rk= -github.com/sashabaranov/go-openai v1.12.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.13.0 h1:EAusFfnhaMaaUspUZ2+MbB/ZcVeD4epJmTOlZ+8AcAE= +github.com/sashabaranov/go-openai v1.13.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= From 8009fc6a8e70a867b7dbbcaed72f79d921553c73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 01:11:43 +0530 Subject: [PATCH 32/62] chore(deps): bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /v2 (#3935) Bumps [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) from 1.11.71 to 1.11.72. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.11.71...feature/s3/manager/v1.11.72) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 32 ++++++++++++++-------------- v2/go.sum | 64 +++++++++++++++++++++++++++---------------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 78b7cf623..2a952ad18 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -54,11 +54,11 @@ require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 - github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 - github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 + github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go-v2/config v1.18.28 + github.com/aws/aws-sdk-go-v2/credentials v1.13.27 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 + github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/docker/go-units v0.5.0 github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 @@ -93,10 +93,10 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bits-and-blooms/bitset v1.3.1 // indirect github.com/bits-and-blooms/bloom/v3 v3.4.0 // indirect @@ -232,14 +232,14 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/alecthomas/chroma v0.10.0 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect diff --git a/v2/go.sum b/v2/go.sum index 286c76f72..9075d67c8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -69,42 +69,42 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= +github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= -github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= +github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= +github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27/go.mod h1:syOqAek45ZXZp29HlnRS/BNgMIW6uiRmeuQsz4Qh2UE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5/go.mod h1:Gj7tm95r+QsDoN2Fhuz/3npQvcZbkEf5mL70n3Xfluc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 h1:m0MmP89v1B0t3b8W8rtATU76KNsodak69QtiokHyEvo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72/go.mod h1:ylOTxIuoTL+XjH46Omv2iPjHdeGUk3SQ4hxYho4EHMA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27/go.mod h1:ZdjYvJpDlefgh8/hWelJhqgqJeodxu4SmbVsSdBlL7E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30/go.mod h1:qQtIBl5OVMfmeQkz8HaVyh5DzFmmFXyvK27UgIgOr4c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4/go.mod h1:JniVpqvw90sVjNqanGLufrVapWySL28fhBlYgl96Q/w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0/go.mod h1:PwyKKVL0cNkC37QwLcrhyeCrAk+5bY8O2ou7USyAS2A= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13/go.mod h1:BzqsVVFduubEmzrVtUFQQIQdFqvUItF8XUq2EnS8Wog= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2OZgH1ypC114H04U= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= From 22c7422eb8536ced0a007f84e80dd81900e573cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 01:12:24 +0530 Subject: [PATCH 33/62] chore(deps): bump golang from 1.20.5-alpine to 1.20.6-alpine (#3931) Bumps golang from 1.20.5-alpine to 1.20.6-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d75c1c17c..23a40a61e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.20.5-alpine AS build-env +FROM golang:1.20.6-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app From 729424d0dfbd01a577c1fe494229929db83d54ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 01:15:18 +0530 Subject: [PATCH 34/62] chore(deps): bump github.com/Azure/azure-sdk-for-go/sdk/storage/azblob (#3933) Bumps [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/v1.0...v1.1) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 2a952ad18..97b5e8ebe 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -48,7 +48,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/DataDog/gostackparse v0.6.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 diff --git a/v2/go.sum b/v2/go.sum index 9075d67c8..394e4ec9b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -9,8 +9,9 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybI github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1vBI4o= From 0828339de6c161a90dfb63a7cb8864941529decf Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 18 Jul 2023 03:24:39 +0530 Subject: [PATCH 35/62] version update --- v2/pkg/catalog/config/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 464d5f268..fed0319c7 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v2.9.8` + Version = `v2.9.9` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGithubTemplatesDirName = "github" From 9558e22a64f372ff0b8f3a0e0f46f52f1de93334 Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Fri, 21 Jul 2023 16:04:53 -0400 Subject: [PATCH 36/62] Remove unused file (#3962) --- v2/pkg/protocols/common/updatecheck/client.go | 83 ------------------- 1 file changed, 83 deletions(-) delete mode 100644 v2/pkg/protocols/common/updatecheck/client.go diff --git a/v2/pkg/protocols/common/updatecheck/client.go b/v2/pkg/protocols/common/updatecheck/client.go deleted file mode 100644 index a921034f4..000000000 --- a/v2/pkg/protocols/common/updatecheck/client.go +++ /dev/null @@ -1,83 +0,0 @@ -package updatecheck - -import ( - "context" - "io" - "net/http" - "net/url" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" - "github.com/projectdiscovery/retryablehttp-go" -) - -const ( - RegisterServer = "https://version-check.nuclei.sh/" - VersionsCall = "versions" - IgnoreCall = "ignore" -) - -var nucleiVersion string - -// LatestVersion is the latest version info for nuclei and templates repos -type LatestVersion struct { - Nuclei string - Templates string - IgnoreHash string -} - -func InitNucleiVersion(version string) { - nucleiVersion = version -} - -// GetLatestNucleiTemplatesVersion returns the latest version info for nuclei and templates repos -func GetLatestNucleiTemplatesVersion() (*LatestVersion, error) { - body, err := callRegisterServer(VersionsCall) - if err != nil { - return nil, err - } - defer body.Close() - - data := make(map[string]string) - if err := jsoniter.NewDecoder(body).Decode(&data); err != nil { - return nil, err - } - return &LatestVersion{Nuclei: data["nuclei"], Templates: data["templates"], IgnoreHash: data["ignore-hash"]}, nil -} - -// GetLatestIgnoreFile returns the latest version of nuclei ignore -func GetLatestIgnoreFile() ([]byte, error) { - body, err := callRegisterServer(IgnoreCall) - if err != nil { - return nil, err - } - defer body.Close() - - data, err := io.ReadAll(body) - if err != nil { - return nil, err - } - return data, nil -} - -// callRegisterServer makes a request to RegisterServer with a call. -func callRegisterServer(call string) (io.ReadCloser, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, RegisterServer+call, nil) - if err != nil { - return nil, errors.Wrap(err, "could not make request") - } - if nucleiVersion != "" { - query := make(url.Values, 1) - query.Set("v", nucleiVersion) - req.URL.RawQuery = query.Encode() - } - resp, err := retryablehttp.DefaultClient().Do(req) - if err != nil { - return nil, errors.Wrap(err, "could not do request") - } - return resp.Body, nil -} From 759ee3d5f8821c8134ff02800d7a52a98d584817 Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Fri, 21 Jul 2023 16:54:06 -0400 Subject: [PATCH 37/62] Markdown Export Sorting (#3961) * Sort markdown exports by host, severity, or template * Switch default to empty string * use fileutil to create folder --------- Co-authored-by: Tarun Koyalwar --- v2/internal/runner/options.go | 7 ++++ v2/internal/runner/runner.go | 2 ++ .../reporting/exporters/markdown/markdown.go | 36 +++++++++++++++++-- v2/pkg/types/types.go | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 3da37f45c..ebbe3e66d 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -407,6 +407,13 @@ func readEnvInputVars(options *types.Options) { options.GitLabTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITLAB_DOWNLOAD") options.AwsTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AWS_DOWNLOAD") options.AzureTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AZURE_DOWNLOAD") + + // Options to modify the behavior of exporters + options.MarkdownExportSortMode = strings.ToLower(os.Getenv("MARKDOWN_EXPORT_SORT_MODE")) + // If the user has not specified a valid sort mode, use the default + if options.MarkdownExportSortMode != "template" && options.MarkdownExportSortMode != "severity" && options.MarkdownExportSortMode != "host" { + options.MarkdownExportSortMode = "" + } } func getBoolEnvValue(key string) bool { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 711ea769e..01c49c9e9 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -346,12 +346,14 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) reportingOptions.MarkdownExporter = &markdown.Options{ Directory: options.MarkdownExportDirectory, IncludeRawPayload: !options.OmitRawRequests, + SortMode: options.MarkdownExportSortMode, } } else { reportingOptions = &reporting.Options{} reportingOptions.MarkdownExporter = &markdown.Options{ Directory: options.MarkdownExportDirectory, IncludeRawPayload: !options.OmitRawRequests, + SortMode: options.MarkdownExportSortMode, } } } diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 362c65718..e3458f767 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -6,9 +6,12 @@ import ( "path/filepath" "strings" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -25,6 +28,7 @@ type Options struct { // Directory is the directory to export found results to Directory string `yaml:"directory"` IncludeRawPayload bool `yaml:"include-raw-payload"` + SortMode string `yaml:"sort-mode"` } // New creates a new markdown exporter integration client based on options. @@ -69,7 +73,35 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { defer file.Close() filename := createFileName(event) - host := util.CreateLink(event.Host, filename) + + // If the sort mode is set to severity, host, or template, then we need to get a safe version of the name for a + // subdirectory to store the file in. + // This will allow us to sort the files into subdirectories based on the sort mode. The subdirectory will need to + // be created if it does not exist. + fileUrl := filename + subdirectory := "" + switch exporter.options.SortMode { + case "severity": + subdirectory = event.Info.SeverityHolder.Severity.String() + case "host": + subdirectory = event.Host + case "template": + subdirectory = event.TemplateID + } + if subdirectory != "" { + // Sanitize the subdirectory name to remove any characters that are not allowed in a directory name + subdirectory = sanitizeFilename(subdirectory) + + // Prepend the subdirectory name to the filename for the fileUrl + fileUrl = filepath.Join(subdirectory, filename) + + // Create the subdirectory if it does not exist + if err = fileutil.CreateFolders(filepath.Join(exporter.directory, subdirectory)); err != nil { + gologger.Warning().Msgf("Could not create subdirectory for markdown report: %s", err) + } + } + + host := util.CreateLink(event.Host, fileUrl) finding := event.TemplateID + " " + event.MatcherName severity := event.Info.SeverityHolder.Severity.String() @@ -85,7 +117,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{})) data := dataBuilder.Bytes() - return os.WriteFile(filepath.Join(exporter.directory, filename), data, 0644) + return os.WriteFile(filepath.Join(exporter.directory, subdirectory, filename), data, 0644) } func createFileName(event *output.ResultEvent) string { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 10411ea3f..48347a600 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -93,6 +93,8 @@ type Options struct { ReportingConfig string // MarkdownExportDirectory is the directory to export reports in Markdown format MarkdownExportDirectory string + // MarkdownExportSortMode is the method to sort the markdown reports (options: severity, template, host, none) + MarkdownExportSortMode string // SarifExport is the file to export sarif output format to SarifExport string // CloudURL is the URL for the nuclei cloud endpoint From ac51b571aeb7fb1cda319ed30323be2946db83cf Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 22 Jul 2023 00:49:52 +0200 Subject: [PATCH 38/62] adding set with httpx probe (#3955) --- v2/examples/simple.go | 20 ++++++++++++++++---- v2/internal/runner/inputs.go | 35 ++++------------------------------- v2/pkg/core/inputs/inputs.go | 15 ++++++++++++++- v2/pkg/utils/http_probe.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 36 deletions(-) create mode 100644 v2/pkg/utils/http_probe.go diff --git a/v2/examples/simple.go b/v2/examples/simple.go index ea7fe130c..e1f9a4cd4 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -5,12 +5,13 @@ import ( "fmt" "log" "os" - "path" + "path/filepath" "time" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" @@ -47,7 +48,7 @@ func main() { protocolstate.Init(defaultOpts) protocolinit.Init(defaultOpts) - defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"} + defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) @@ -58,7 +59,7 @@ func main() { defer interactClient.Close() home, _ := os.UserHomeDir() - catalog := disk.NewCatalog(path.Join(home, "nuclei-templates")) + catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates")) executerOpts := protocols.ExecutorOptions{ Output: outputWriter, Options: defaultOpts, @@ -86,9 +87,20 @@ func main() { } store.Load() + // flat input without probe inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}} - input := &inputs.SimpleInputProvider{Inputs: inputArgs} + + httpxOptions := httpx.DefaultOptions + httpxOptions.Timeout = 5 * time.Second + httpxClient, err := httpx.New(&httpxOptions) + if err != nil { + log.Fatal(err) + } + + // use httpx to probe the URL => https://scanme.sh + input.SetWithProbe("scanme.sh", httpxClient) + _ = engine.Execute(store.Templates(), input) engine.WorkPool().Wait() // Wait for the scan to finish } diff --git a/v2/internal/runner/inputs.go b/v2/internal/runner/inputs.go index 1bce3f879..191972a03 100644 --- a/v2/internal/runner/inputs.go +++ b/v2/internal/runner/inputs.go @@ -1,18 +1,16 @@ package runner import ( - "fmt" - "net/http" - "strings" "sync/atomic" "time" - "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/remeh/sizedwaitgroup" ) @@ -25,7 +23,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { if err != nil { return nil, errors.Wrap(err, "could not create temporary input file") } - gologger.Info().Msgf("Running httpx on input host") var bulkSize = probeBulkSize @@ -45,7 +42,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { swg := sizedwaitgroup.New(bulkSize) count := int32(0) r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { - if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") { + if stringsutil.HasPrefixAny(value.Input, "http://", "https://") { return true } @@ -53,7 +50,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { go func(input *contextargs.MetaInput) { defer swg.Done() - if result := probeURL(input.Input, httpxClient); result != "" { + if result := utils.ProbeURL(input.Input, httpxClient); result != "" { atomic.AddInt32(&count, 1) _ = hm.Set(input.Input, []byte(result)) } @@ -65,27 +62,3 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count)) return hm, nil } - -var ( - httpSchemes = []string{"https", "http"} -) - -// probeURL probes the scheme for a URL. first HTTPS is tried -// and if any errors occur http is tried. If none succeeds, probing -// is abandoned for such URLs. -func probeURL(input string, httpxclient *httpx.HTTPX) string { - for _, scheme := range httpSchemes { - formedURL := fmt.Sprintf("%s://%s", scheme, input) - req, err := httpxclient.NewRequest(http.MethodHead, formedURL) - if err != nil { - continue - } - req.Header.Set("User-Agent", uarand.GetRandom()) - - if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { - continue - } - return formedURL - } - return "" -} diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go index bb75b6e9c..5e04d7128 100644 --- a/v2/pkg/core/inputs/inputs.go +++ b/v2/pkg/core/inputs/inputs.go @@ -1,6 +1,10 @@ package inputs -import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +import ( + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" +) type SimpleInputProvider struct { Inputs []*contextargs.MetaInput @@ -24,3 +28,12 @@ func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) b func (s *SimpleInputProvider) Set(value string) { s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) } + +// SetWithProbe adds item to input provider with http probing +func (s *SimpleInputProvider) SetWithProbe(value string, httpxClient *httpx.HTTPX) { + valueToAppend := value + if result := utils.ProbeURL(value, httpxClient); result != "" { + valueToAppend = result + } + s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: valueToAppend}) +} diff --git a/v2/pkg/utils/http_probe.go b/v2/pkg/utils/http_probe.go new file mode 100644 index 000000000..f79567d75 --- /dev/null +++ b/v2/pkg/utils/http_probe.go @@ -0,0 +1,33 @@ +package utils + +import ( + "fmt" + "net/http" + + "github.com/corpix/uarand" + "github.com/projectdiscovery/httpx/common/httpx" +) + +var ( + HttpSchemes = []string{"https", "http"} +) + +// probeURL probes the scheme for a URL. first HTTPS is tried +// and if any errors occur http is tried. If none succeeds, probing +// is abandoned for such URLs. +func ProbeURL(input string, httpxclient *httpx.HTTPX) string { + for _, scheme := range HttpSchemes { + formedURL := fmt.Sprintf("%s://%s", scheme, input) + req, err := httpxclient.NewRequest(http.MethodHead, formedURL) + if err != nil { + continue + } + req.Header.Set("User-Agent", uarand.GetRandom()) + + if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { + continue + } + return formedURL + } + return "" +} From ae667a52d228e4777fc43e334f5f6acde73dd695 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 23 Jul 2023 18:42:32 +0530 Subject: [PATCH 39/62] Create CONTRIBUTING.md (#3968) --- CONTRIBUTING.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..ad05027e8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,30 @@ +# Contributing to ProjectDiscovery/Nuclei + +We appreciate your interest in contributing to the projectdiscovery/nuclei! This document provides some basic guidelines for contributors. + +## Getting Started + +- Always base your work from the `dev` branch, which is the development branch with the latest code. +- Before creating a Pull Request (PR), make sure there is a corresponding issue for your contribution. If there isn't one already, please create one. +- Include the problem description in the issue. + +## Pull Requests + +When creating a PR, please follow these guidelines: + +- Link your PR to the corresponding issue. +- Provide context in the PR description to help reviewers understand the changes. The more information you provide, the faster the review process will be. +- Include an example of running the tool with the changed code, if applicable. Provide 'before' and 'after' examples if possible. +- Include steps for functional testing or replication. +- If you're adding a new feature, make sure to include unit tests. + +## Code Style + +Please adhere to the existing coding style for consistency. + +## Questions + +If you have any questions or need further guidance, please feel free to ask in the issue or PR, or [reach out to the maintainers](https://discord.gg/projectdiscovery). + +Thank you for your contribution! + From 5074722f17e517ae6ea35a6a9762f571b9ac1aca Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 24 Jul 2023 16:50:28 +0200 Subject: [PATCH 40/62] Reducing data race via SyncLockMap (#3959) * replacing custom map with synclockmap * fixing initialization * removing unused code --- v2/pkg/protocols/common/contextargs/args.go | 31 ------------ .../common/contextargs/contextargs.go | 50 ++++--------------- 2 files changed, 11 insertions(+), 70 deletions(-) delete mode 100644 v2/pkg/protocols/common/contextargs/args.go diff --git a/v2/pkg/protocols/common/contextargs/args.go b/v2/pkg/protocols/common/contextargs/args.go deleted file mode 100644 index e3bde44a4..000000000 --- a/v2/pkg/protocols/common/contextargs/args.go +++ /dev/null @@ -1,31 +0,0 @@ -package contextargs - -// Args is a generic map with helpers -type Args map[string]interface{} - -// Set a key with value -func (args Args) Set(key string, value interface{}) { - args[key] = value -} - -// Get the value associated to a key -func (args Args) Get(key string) (interface{}, bool) { - value, ok := args[key] - return value, ok -} - -// Has verifies if the map contains the key -func (args Args) Has(key string) bool { - _, ok := args[key] - return ok -} - -// IsEmpty verifies if the map is empty -func (Args Args) IsEmpty() bool { - return len(Args) == 0 -} - -// create a new args map instance -func newArgs() map[string]interface{} { - return make(map[string]interface{}) -} diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index f847ca112..875230750 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -2,9 +2,8 @@ package contextargs import ( "net/http/cookiejar" - "sync" - "golang.org/x/exp/maps" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Context implements a shared context struct to share information across multiple templates within a workflow @@ -15,10 +14,8 @@ type Context struct { // CookieJar shared within workflow's http templates CookieJar *cookiejar.Jar - // Access to Args must use lock strategies to prevent data races - *sync.RWMutex // Args is a workflow shared key-value store - args Args + args *mapsutil.SyncLockMap[string, interface{}] } // Create a new contextargs instance @@ -32,15 +29,7 @@ func NewWithInput(input string) *Context { } func (ctx *Context) initialize() { - ctx.args = newArgs() - ctx.RWMutex = &sync.RWMutex{} -} - -func (ctx *Context) set(key string, value interface{}) { - ctx.Lock() - defer ctx.Unlock() - - ctx.args.Set(key, value) + ctx.args = &mapsutil.SyncLockMap[string, interface{}]{Map: mapsutil.Map[string, interface{}]{}} } // Set the specific key-value pair @@ -49,7 +38,7 @@ func (ctx *Context) Set(key string, value interface{}) { ctx.initialize() } - ctx.set(key, value) + _ = ctx.args.Set(key, value) } func (ctx *Context) isInitialized() bool { @@ -60,49 +49,33 @@ func (ctx *Context) hasArgs() bool { return ctx.isInitialized() && !ctx.args.IsEmpty() } -func (ctx *Context) get(key string) (interface{}, bool) { - ctx.RLock() - defer ctx.RUnlock() - - return ctx.args.Get(key) -} - // Get the value with specific key if exists func (ctx *Context) Get(key string) (interface{}, bool) { if !ctx.hasArgs() { return nil, false } - return ctx.get(key) + return ctx.args.Get(key) } -func (ctx *Context) GetAll() Args { +func (ctx *Context) GetAll() *mapsutil.SyncLockMap[string, interface{}] { if !ctx.hasArgs() { return nil } - return maps.Clone(ctx.args) + return ctx.args.Clone() } func (ctx *Context) ForEach(f func(string, interface{})) { - ctx.RLock() - defer ctx.RUnlock() - - for k, v := range ctx.args { + _ = ctx.args.Iterate(func(k string, v interface{}) error { f(k, v) - } -} - -func (ctx *Context) has(key string) bool { - ctx.RLock() - defer ctx.RUnlock() - - return ctx.args.Has(key) + return nil + }) } // Has check if the key exists func (ctx *Context) Has(key string) bool { - return ctx.hasArgs() && ctx.has(key) + return ctx.hasArgs() && ctx.args.Has(key) } func (ctx *Context) HasArgs() bool { @@ -112,7 +85,6 @@ func (ctx *Context) HasArgs() bool { func (ctx *Context) Clone() *Context { newCtx := &Context{ MetaInput: ctx.MetaInput.Clone(), - RWMutex: ctx.RWMutex, args: ctx.args, CookieJar: ctx.CookieJar, } From 4005b8bf3f34dafa4b52221ba0312c9092bcd6da Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:55:21 +0530 Subject: [PATCH 41/62] dep workflow update --- .github/dependabot.yml | 50 ++++++++++--------- .../{auto-merge.yml => dep-auto-merge.yml} | 3 +- 2 files changed, 28 insertions(+), 25 deletions(-) rename .github/workflows/{auto-merge.yml => dep-auto-merge.yml} (89%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 24bdcc299..dbe5721d0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,38 +6,40 @@ version: 2 updates: - # Maintain dependencies for GitHub Actions - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - target-branch: "dep" - commit-message: - prefix: "chore" - include: "scope" - labels: - - "Type: Maintenance" - # Maintain dependencies for go modules - package-ecosystem: "gomod" directory: "v2/" schedule: interval: "weekly" - target-branch: "dep" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" labels: - "Type: Maintenance" + allow: + - dependency-name: "github.com/projectdiscovery/*" - # Maintain dependencies for docker - - package-ecosystem: "docker" - directory: "/" - schedule: - interval: "weekly" - target-branch: "dep" - commit-message: - prefix: "chore" - include: "scope" - labels: - - "Type: Maintenance" +# # Maintain dependencies for GitHub Actions +# - package-ecosystem: "github-actions" +# directory: "/" +# schedule: +# interval: "weekly" +# target-branch: "dev" +# commit-message: +# prefix: "chore" +# include: "scope" +# labels: +# - "Type: Maintenance" +# +# # Maintain dependencies for docker +# - package-ecosystem: "docker" +# directory: "/" +# schedule: +# interval: "weekly" +# target-branch: "dev" +# commit-message: +# prefix: "chore" +# include: "scope" +# labels: +# - "Type: Maintenance" diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/dep-auto-merge.yml similarity index 89% rename from .github/workflows/auto-merge.yml rename to .github/workflows/dep-auto-merge.yml index 0e56f0a03..84b26e1fe 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/dep-auto-merge.yml @@ -3,7 +3,7 @@ name: 🤖 dep auto merge on: pull_request: branches: - - dep + - dev workflow_dispatch: permissions: @@ -14,6 +14,7 @@ permissions: jobs: automerge: runs-on: ubuntu-latest + if: github.actor == 'dependabot[bot]' steps: - uses: actions/checkout@v3 with: From d40ecc5b625dc870eb73e28bba8c2c8db1eed3a2 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:56:34 +0530 Subject: [PATCH 42/62] workflow clean up --- .github/workflows/sync-dep.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/sync-dep.yml diff --git a/.github/workflows/sync-dep.yml b/.github/workflows/sync-dep.yml deleted file mode 100644 index 73798b78d..000000000 --- a/.github/workflows/sync-dep.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: 🤖 sync dep - -on: - push: - branches: - - dev - -permissions: - pull-requests: write - issues: write - repository-projects: write - -jobs: - sync-dep-with-dev: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Node - uses: actions/setup-node@v3 - with: - node-version: 12 - - - name: Opening pull request - id: pull - uses: tretuna/sync-branches@1.4.0 - with: - GITHUB_TOKEN: "${{ secrets.DEPENDABOT_PAT }}" - FROM_BRANCH: "${{ github.ref_name }}" - TO_BRANCH: "dep" \ No newline at end of file From 16894cf0e07467aa5f1d2b4e2cd9fcca19b215e2 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 28 Jul 2023 15:52:14 +0200 Subject: [PATCH 43/62] fixing certificate expiration date (#3995) --- .../auth/certificate/assets/client.crt | 32 +++---- .../auth/certificate/assets/client.key | 52 +++++------ .../auth/certificate/assets/server.crt | 32 +++---- v2/cmd/integration-test/generic.go | 88 +++++++++---------- 4 files changed, 102 insertions(+), 102 deletions(-) diff --git a/integration_tests/generic/auth/certificate/assets/client.crt b/integration_tests/generic/auth/certificate/assets/client.crt index 4bbdee24b..8e92ac8ce 100644 --- a/integration_tests/generic/auth/certificate/assets/client.crt +++ b/integration_tests/generic/auth/certificate/assets/client.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDETCCAfkCFFSLOinkkPWOfwuzMHF0B9EZIIghMA0GCSqGSIb3DQEBCwUAMEUx +MIIDEzCCAfsCFBDZsFEIb3QwKLzXLoqR/oaDwakYMA0GCSqGSIb3DQEBCwUAMEUx CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MzA2WhcNMjMwNzIxMDA0 -MzA2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAp187MX/sPGlTl8WklyTSh8+kYURy2zWmp2FglMvo058NanCD -rjYJvS2FNM2ZwRZH52qBr6rpqIsa8QDFjnlFJK5y4FNr104ZjDr1YXDfX5ftQeeX -wCavFRDdH/s43kGyNjNeSH78EhqmC3z7LxCJT5k8rXRAyC02uBRqQTrRHfNuVvGI -JZ59jxESzRJt89kWuYymkAj16LHffMvcq6HLwr/KG8IyrJJRj5KcDzYorFonTPe9 -rwBlAbU3LpQ4ZlEP1mQA2PdCg3t85pz3n+57Iw839bWrwbjfijXHY3yjbqfEAaqa -md08nByTg8TZlt9UzjWgj0K1DXDMDDkJKqi39QIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQAH0MhznadrMFuY2ZYA69FbsvOygMctv8qZW1HrHS0X13IXeW+8uxfb5+gk -yKFgXNMFueyd5PoN9vyC1t9AOBPnI56gaYm/MbDtwEqGo8C+9fjJasY23J90p54t -G6vxcXwo33HVpWBeRBkVF/SePeCn+MKk0jd/JgJS0T0s1Ih6wkn84/83hDk4M2M2 -/yhc3wuNYdf/WB8QAfJAc2YpIfkMOoxGPTsxvREiZrPUyGiWa507hrHcQU0GV8qC -KcnS7UUCT9TtJvQIKHwW68XjBudWpaILBj1TS8hOGseOJydJqbk8wyMTE6fgc1Ss -KfTrfa0HOHIkAU/TfE22Zqfw4z70 +cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwNzI4MTAwODIyWhgPMzAwMzA5Mjkx +MDA4MjJaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCp8/P9JAyE90ZrE1LZcJ/B24f79aazY8S/eeRRZsTvUP73 +NrOznv1zhvJ9TKHUNcOouZ/NPQanNOiqkoigQwP7L2FA2bPOPAPIWBPWGdjSkeyZ +8MYbA7Or+16k2ZYvKsCarG/PgGeL0UFLe6INvZRMnk1s+iF0upcHv5BhjIfBwzh4 +o2pLY1d9bbnEsuSNagOzIkQS3mI22d1YbJKxXP0m+tBk1gTqhUhwEAXNaIBCRscs +xyv9pW7ZSjPabf/L0Md2yMcVs0+oK6rkQbAWrTTjN1lJ603BHh+keIDMwQnbMB0U +AStJdyQpwa7hZ+5767+GxR7n85Twe1rSexmTl9/fAgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAIOQE2DWqwse0srtG+7IS0EO3iP27lRKxd387wY1xq00o3depKReVpYm +R8sZM1meumniH1QKoVFJpBHYoPzQMi8vMmI9AV3KWNFcCyf+jwc69Qab2erDNVsw +5mCCGXkrzLbCzmbPFZoyvMmBlsQSmOjwyGGIeXwfqKv/TPwOzKfSM/KkQmgRyUro +GDT+TI5VhgvQyNLmkWNRhnI30DnlsQ1Bc0MEQ1hismOYxD4mCqufCOS3BmakDRNK +QBz0xl0i5Dbf+e4o3rEaCGW/rzKkL1mm1TXqpDEy3UAwj+jIOZu5yByw5djfgSIX +OEVuqklUASYAPeVdSyf/VAflLV9nGKI= -----END CERTIFICATE----- diff --git a/integration_tests/generic/auth/certificate/assets/client.key b/integration_tests/generic/auth/certificate/assets/client.key index 96a88abbc..a31f753cf 100644 --- a/integration_tests/generic/auth/certificate/assets/client.key +++ b/integration_tests/generic/auth/certificate/assets/client.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnXzsxf+w8aVOX -xaSXJNKHz6RhRHLbNaanYWCUy+jTnw1qcIOuNgm9LYU0zZnBFkfnaoGvqumoixrx -AMWOeUUkrnLgU2vXThmMOvVhcN9fl+1B55fAJq8VEN0f+zjeQbI2M15IfvwSGqYL -fPsvEIlPmTytdEDILTa4FGpBOtEd825W8Yglnn2PERLNEm3z2Ra5jKaQCPXosd98 -y9yrocvCv8obwjKsklGPkpwPNiisWidM972vAGUBtTculDhmUQ/WZADY90KDe3zm -nPef7nsjDzf1tavBuN+KNcdjfKNup8QBqpqZ3TycHJODxNmW31TONaCPQrUNcMwM -OQkqqLf1AgMBAAECggEAEZiXdorGYUuJeElVFnbOk2ynEXrKwHURgkNgjgQqBCWS -mYAet/ACchsZCAYdhgk7of62h6tmSUvmlzPHkUT6mfKlLSRYEBir4uxH4+ij8z7b -uLQKZi8q9QIC3VviDKvHep9H6ENBaP3YOxj2p2oLpYysrmesb98hA5VR3m26knVb -sfYwLdsrqQ5Tiuzm1Mdaca2HEYq0iXoZqjQHi31h33rw4HFBKcGX6w6PJOon+i2m -eSSaCAJMYFkoS6NafJYwQDuwIwp+IyrYdI/vaAR6s2ufJ4doIjJC7YuPO5jpLpJc -IBBl5e7pu4/rlwJCDARfBWuXFbjtoMAvmM1MojdNOQKBgQDYCIl97ef19LmeYfwk -RW2xhQTVCWwwBBhSl06iB9SeFkzCCRb+AFjhVhcwe4xIWh9GXWZawZC2XSaoTXte -hxcZRFbnpjcpZ6sYKiP4fB/GXGqYDUWovxu8gmXKrXtfwqJMXklfFE7WblGgK5gG -l7OfbuczaRaIQHoIQYzfmeCwmwKBgQDGVhCSNFGiRaDG7k2VfXElaIyL7m7FlsXf -EptolOeoGv5GvVr3CB5TvUqXN3haLZvUbBKRpWrDbP1n1i+77VICOLtq1qf6SEog -1p2PAccGhXXvrL7LJLUr9Hk831D4fSX9TqVzdxwfMdGFepYoS4vm8fkGVCuaytDa -fniJl8TarwKBgQCpTreCrAsY5bz7dcuIGamIcLmCxKm3T95IDDEiJ4ToiI2LnFga -pOcDYtc1tf4RTiAoo1ZuVjk10vdS+7ZuNO1Tbg216rxchNTAUXZzbcPxT8hydiRb -xbrVGFTybNe+CunrdBGIpH/M6hSqtL+mmwm5L8+eqQNxsSZyhf0D2LMRdQKBgD8J -CXk+MZfOY1v2Tygs1zIZeVnb7M7VrYvJYSUq9jliYuBevDN5HBJnPfazhYe7qSQp -OPmbRkRYNm2zEDa9JWxZVY+OK5MLOKwZKbhSy0uSTTpgf78WqpIOwB2NqDFhrRpF -zaXV/FUZw0qV/HVQFWXQD+JoC/fFb/2RZoPsfX83AoGBAM4VPDUa1I16MuuW+FNZ -cVENZK/qsXFMvm3xutezvMSgmCxVSnXy9GP8QbqkfMrDJd3v/HnwrC4ORTlU7rim -AWSvC6CYO1c2RantleA46T90uWW8kP24TK1yWOrRGKuaQYvfokiiLVExq1nA1iSR -/QPLg6vEoPMOLhB7BQBpsFkq +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCp8/P9JAyE90Zr +E1LZcJ/B24f79aazY8S/eeRRZsTvUP73NrOznv1zhvJ9TKHUNcOouZ/NPQanNOiq +koigQwP7L2FA2bPOPAPIWBPWGdjSkeyZ8MYbA7Or+16k2ZYvKsCarG/PgGeL0UFL +e6INvZRMnk1s+iF0upcHv5BhjIfBwzh4o2pLY1d9bbnEsuSNagOzIkQS3mI22d1Y +bJKxXP0m+tBk1gTqhUhwEAXNaIBCRscsxyv9pW7ZSjPabf/L0Md2yMcVs0+oK6rk +QbAWrTTjN1lJ603BHh+keIDMwQnbMB0UAStJdyQpwa7hZ+5767+GxR7n85Twe1rS +exmTl9/fAgMBAAECggEAPZzaVGhQPZgjqEfeHkQtNqtuthJNd/Vwa3Y2JqiaNqRn +epoTNcgq3EoM+Q3iETvYjf+VhmNcWRveSZBMBcWl2NdJa6hA/kBVorkDn/fI2jXa +z8gxGbQS3AOKQTs8ribSooBnHJPRdifLgyD0FAUpkUlGin53yIionj99iU/YG48g +4dwkBIFHRcxertQyhu3YQ+XleJ35n7mNFwGzC7curRBPUHMImPASzVYQhVdN8OBt +TZEoJw+2lmH4fIJYult27hcl2/pLs1FPvQFSLTIoqzaEzRhKdANkclmnhJjCBXzB +7RLUpKOv1Q28u+P5KH1nFBV/UuuxXrjFt4jhRdji2QKBgQDvv+W0GJWX5POfyRHT +pAROclgVPEgS5vXQIelMdR76a72L/4Vm2/xeolWW1h5qmJF479V/+P+ppxb1IrUy +6+yGtkMiQE4CizhFGWivfXUTPZbdeeSpHMUl9tRZdBZWi3aXzJ/8DfCzD+ZVS4Vx ++y62V4ymQyAqBWv2ast/ElEbowKBgQC1ePQgR+MNfz7/BaatCcLPwFG/kkqPVuzH +//6HB+gAYTyuZsbLrYhCQsbsTjvQz0ExmTnNSeCjHTntQ+pZ8Tnuet9bHxKTRbvG +9Ol/J402EnY2tO/b8jKXHNNyLNImvWpJ4PpaLRKQVxLPei+JcEHyz4MVMrhIjX1b +EhhDCZ6ulQKBgEUy+jX1MphY+QiRnJedq7CIyGu4roTmLOUaJKBw3bQiDN+vrO13 +aWxXJqUWwEi8KKDjeJgrYn+xPqsajXpZJjfru4zTSrDpRiCLqO+eIoCfMkBSwnEd +YLoIeFopa5knP9+orDSwQV0tpanQ1n+DpIP02R/UGCCI2BST1pCi1M5JAoGAC/+E +PIIkO+c21gucmoIztCKmBQF6FoILw6lkPa9DIotLRMicyiieAquBlWwSvlqFl+7m +iHEi/gXXp50+6FVvnBnZnJ+wTbZllODqczK9Pl74G+PYm/UmbSFFxZ27Az6wwVOz +mbSzLoHjR35vmCmo4pHfu84PqxRXvmay3fPL3wUCgYEA0yZcvQqiTs+f4S/mZbhp +fyPgurmowXUNgdijyeFoH+DMkwdWUJeBrinelQaXADUSXkKiA8gaoNGOIkDIBcve +gdUhrY204MeoTYxnIb1dw6/KReya4YdRSMlYiX2hYEURIxdaJV5HcwW5ySMOzP87 +t2+YVr4faAv4AS8k21pBGrc= -----END PRIVATE KEY----- diff --git a/integration_tests/generic/auth/certificate/assets/server.crt b/integration_tests/generic/auth/certificate/assets/server.crt index aa818acb4..b406cf943 100644 --- a/integration_tests/generic/auth/certificate/assets/server.crt +++ b/integration_tests/generic/auth/certificate/assets/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDETCCAfkCFHA1RpGfOY5p/vQmeMQ1oRFqH+CGMA0GCSqGSIb3DQEBCwUAMEUx +MIIDEzCCAfsCFC21Zw7U0tGDyLyMalwfo9cWbL6dMA0GCSqGSIb3DQEBCwUAMEUx CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MjQ2WhcNMjMwNzIxMDA0 -MjQ2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA3VdrKR5hmZ+vyvg6NB2dOL5vEIQ/9DevivnKWqX5mserYLMj -Wq0knVfogewZnrDe+zVC3kOogBQvYk8Z53kTY9qpJT85dMCuW4xDx0JU+cWHul9a -pzF+bvws4paCWIcsGONyocPAx5g07LbPU9civC80QkQqELo1zYiRU1bX8vRJJqbN -TW2mzl9MN3AnCAYTwq8WhVG/1QR3LPQhPR68/1LWrFefQaEWaXT2s+Xv7K7NDXro -WSba4SgKdFd6fyUVMVr/ioT1KT45TP5jbRrW5JJUTdpkiXaIucrZg39f6F5gTZGA -U7bNROUMkqrJJngN9+Hp+YH1GpkKgu9EKA30EQIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQAw91bxiAi7DIVsKL3k4B0I+50ZKq9VMVNE3YCTPygpfuRiGQvlITZ5I8I5 -3Ok2wWltgKx6EnicHIlLg42yRj7j3mdgOLMFMrUCfJmdogwnS+k6veG3G1RHUs9r -ATfX49u/hEX2pe7Rvx2VYVIugwrQESgQ21iaf6uUMsrq6W8eYZ31as1nJKpqIGbu -W1fZMSi0RIUJP+mpVBE82IW+gJRi3uKU4HKPqyrU3dviBFdBxb3lNbh34/vdNkIw -4H2CfBxEvdwLYAhWDerlm4wWCmjkMiHfBHPBhhOICTkR25a7NFy27h/UDHjVC/6m -fGshVSBtxVPJP7kcvZ1scIctvFZZ +cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwNzI4MTAwNzI3WhgPMzAwMzA5Mjkx +MDA3MjdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCjMlvOKQX9yn9SOYPJ8p+jeDUU/JWPwT4LRfqaxvvKSnS7 +NZzd7lS4AR0YTjyjiRj3+t0QnEDHVKBD8cMh9kMXkQ2S0r7psCURLvvZOYt4v6KM +CyZpBbp8b/pG3aJQHDZjRDOApQrXhx62XJDIs64YKA8NybYOLqNisrWGrfqF4uEz +RMgVGlthuQcXo3n2HzobuYN7RsHBzCWGLn9fRMDC2j3IAnQLf4YOznOJ57CjMd2W +mn/yhHK8h9s4iU5zw3+PK+X/IM4GeAfeJMx8c5uq2A8A24uzMidyhxJCK7VUprjK +/ckdNYya6dkG2De+LR7W82ygfWbFDOnZKM26cPG/AgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAH5+Wdb/1jgBhihN6Pb6SWJmDvwkOEP3t00E3fBao4TDqdDOhPsLYrAm +8gt16OcGrrXDQA3bi79mAVqAqCvaf4hk0vSI0L4rNcCSP4D3fUBjRO3fY3fM4Qw8 +xg9AusF5hRrvzFbEak7lPJ01kLOJEgBA1l457HrLnXcpDTml8Y46WqdWa6yVM33l +7tNaXWrPwYZYMTcRumIytsYtIJXp/sMLBIT0AO/QR4yarvVOeMSJ1va459PjKLBG +JGGmf2rigaT050e71QOrGyMXgT6xsNjJgzeVhUgPO422mPT692kDi2oB5DA0Fau0 +4qm5CMFgmYcC3zQoN53aDs1mHyWeroc= -----END CERTIFICATE----- diff --git a/v2/cmd/integration-test/generic.go b/v2/cmd/integration-test/generic.go index dfd24b165..b22ffd166 100644 --- a/v2/cmd/integration-test/generic.go +++ b/v2/cmd/integration-test/generic.go @@ -19,53 +19,53 @@ var genericTestcases = map[string]testutils.TestCase{ var ( serverCRT = `-----BEGIN CERTIFICATE----- -MIIDETCCAfkCFHA1RpGfOY5p/vQmeMQ1oRFqH+CGMA0GCSqGSIb3DQEBCwUAMEUx +MIIDEzCCAfsCFC21Zw7U0tGDyLyMalwfo9cWbL6dMA0GCSqGSIb3DQEBCwUAMEUx CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MjQ2WhcNMjMwNzIxMDA0 -MjQ2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA3VdrKR5hmZ+vyvg6NB2dOL5vEIQ/9DevivnKWqX5mserYLMj -Wq0knVfogewZnrDe+zVC3kOogBQvYk8Z53kTY9qpJT85dMCuW4xDx0JU+cWHul9a -pzF+bvws4paCWIcsGONyocPAx5g07LbPU9civC80QkQqELo1zYiRU1bX8vRJJqbN -TW2mzl9MN3AnCAYTwq8WhVG/1QR3LPQhPR68/1LWrFefQaEWaXT2s+Xv7K7NDXro -WSba4SgKdFd6fyUVMVr/ioT1KT45TP5jbRrW5JJUTdpkiXaIucrZg39f6F5gTZGA -U7bNROUMkqrJJngN9+Hp+YH1GpkKgu9EKA30EQIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQAw91bxiAi7DIVsKL3k4B0I+50ZKq9VMVNE3YCTPygpfuRiGQvlITZ5I8I5 -3Ok2wWltgKx6EnicHIlLg42yRj7j3mdgOLMFMrUCfJmdogwnS+k6veG3G1RHUs9r -ATfX49u/hEX2pe7Rvx2VYVIugwrQESgQ21iaf6uUMsrq6W8eYZ31as1nJKpqIGbu -W1fZMSi0RIUJP+mpVBE82IW+gJRi3uKU4HKPqyrU3dviBFdBxb3lNbh34/vdNkIw -4H2CfBxEvdwLYAhWDerlm4wWCmjkMiHfBHPBhhOICTkR25a7NFy27h/UDHjVC/6m -fGshVSBtxVPJP7kcvZ1scIctvFZZ ------END CERTIFICATE----- +cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwNzI4MTAwNzI3WhgPMzAwMzA5Mjkx +MDA3MjdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCjMlvOKQX9yn9SOYPJ8p+jeDUU/JWPwT4LRfqaxvvKSnS7 +NZzd7lS4AR0YTjyjiRj3+t0QnEDHVKBD8cMh9kMXkQ2S0r7psCURLvvZOYt4v6KM +CyZpBbp8b/pG3aJQHDZjRDOApQrXhx62XJDIs64YKA8NybYOLqNisrWGrfqF4uEz +RMgVGlthuQcXo3n2HzobuYN7RsHBzCWGLn9fRMDC2j3IAnQLf4YOznOJ57CjMd2W +mn/yhHK8h9s4iU5zw3+PK+X/IM4GeAfeJMx8c5uq2A8A24uzMidyhxJCK7VUprjK +/ckdNYya6dkG2De+LR7W82ygfWbFDOnZKM26cPG/AgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAH5+Wdb/1jgBhihN6Pb6SWJmDvwkOEP3t00E3fBao4TDqdDOhPsLYrAm +8gt16OcGrrXDQA3bi79mAVqAqCvaf4hk0vSI0L4rNcCSP4D3fUBjRO3fY3fM4Qw8 +xg9AusF5hRrvzFbEak7lPJ01kLOJEgBA1l457HrLnXcpDTml8Y46WqdWa6yVM33l +7tNaXWrPwYZYMTcRumIytsYtIJXp/sMLBIT0AO/QR4yarvVOeMSJ1va459PjKLBG +JGGmf2rigaT050e71QOrGyMXgT6xsNjJgzeVhUgPO422mPT692kDi2oB5DA0Fau0 +4qm5CMFgmYcC3zQoN53aDs1mHyWeroc= +-----END CERTIFICATE----- ` serverKey = `-----BEGIN PRIVATE KEY----- -MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDdV2spHmGZn6/K -+Do0HZ04vm8QhD/0N6+K+cpapfmax6tgsyNarSSdV+iB7BmesN77NULeQ6iAFC9i -TxnneRNj2qklPzl0wK5bjEPHQlT5xYe6X1qnMX5u/CziloJYhywY43Khw8DHmDTs -ts9T1yK8LzRCRCoQujXNiJFTVtfy9Ekmps1NbabOX0w3cCcIBhPCrxaFUb/VBHcs -9CE9Hrz/UtasV59BoRZpdPaz5e/srs0NeuhZJtrhKAp0V3p/JRUxWv+KhPUpPjlM -/mNtGtbkklRN2mSJdoi5ytmDf1/oXmBNkYBTts1E5QySqskmeA334en5gfUamQqC -70QoDfQRAgMBAAECggEBALtPsHMSr9vW5Giq2m6iJRwRJGJg2NJukZLVwuYlkW7n -zGNAFgo1fkfdTfks+Z1u5rTGJPl9XkpNSrAyaqSVtNALCptnvtLMAIGe2Pj2bH0X -Kb6R1WCqJOn9ZGq4nkQW2D2Ttb2psCn458jvB9NWu6FvfRUbJFIVk1SFXx6c3pFN -kPCUudAiscaldUDCiz4FccKGXdRjq6HIeeWqvdErteb6JPTs9QXCHfBql9Esl4rK -SHt9RmAFNY+CLExHiFPBR15hHZRtiVkAVrgnPg1CPGAyVG0hGXj7YMMWpAyfFWpn -8gWVt7XJ4UX2knUwfU8p8dWe6qwf+AMrhravYJyccoUCgYEA8Ts0kHFnLga8Ewao -nyDQs5uYGG0PWkbXqnFVYnMeSbXzyC4ouInIk/eOQABCxdjy3NF9QuYvVLpfLJ+9 -a97q1Vyg6lZ4PPuK8ZcPrHFSNNaj4eWNTOMo/Qdzz4bfflTsv8vjeeMxsqb6woXV -+E23UKCPlQPf86jugZVdaMtvZKsCgYEA6uR7glji70pVoG/f3soX1vllmVTtiLnh -zYMmwPyTRDvoGgg/nGK+GCq//Xyn8D900hbX8KKqGX7ca5FGk5pOpW/QE9uLcuWK -xcy8KAc05k1u4VaS5loWKnPGWreIpj3RbCfbPs5X/jBC+fPIA4Q8Qor5ZGdqVBvW -IKejnNqasjMCgYEAqltPUbpkTWLAKweGyWnZOR3mmUlbkDt7Toje7bmyaAew82t1 -omzbU3N958DHZwVA7aSbu0TnpARB9jeRA77XRHo3wYXzP828X8R4cyVMEriJ35vG -38eESLyckrAC4SqETyZjrM4/aJT3fawaYVIw5SWegHPOEjr4xFaBMuKH9iUCgYEA -wFpC2kc374UMAcobpjIQu7aYAKyPqDuwMb+I6NjtMB9uvoKqtMIXsWqwtkBytkcA -v1p9k01hxmcg0eWxygW/CbM6zkgnNfvLXJeALbdZFo+qkVV4DrMPG8ybToalnJ1a -9hrda91GKZ4T+uQrktWjE0sDV7loVWBGRY+CaFyL+gkCgYEA3Z0j8VOLJnAKdCDp -3N74460pykwJ2suEYSJG6glXfU3fZ5VwAYjimxgD0S2VU4qK8PYBfa/oFH2vRX5p -11dWQWbfBdREO70UmJD4Pr6g3q9AF6DXLXb7dVm4y+hX065Xshk8oIuITVyO/XVK -wWqBD5GScI+Q7PLMes7aqtsDDJI= ------END PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCjMlvOKQX9yn9S +OYPJ8p+jeDUU/JWPwT4LRfqaxvvKSnS7NZzd7lS4AR0YTjyjiRj3+t0QnEDHVKBD +8cMh9kMXkQ2S0r7psCURLvvZOYt4v6KMCyZpBbp8b/pG3aJQHDZjRDOApQrXhx62 +XJDIs64YKA8NybYOLqNisrWGrfqF4uEzRMgVGlthuQcXo3n2HzobuYN7RsHBzCWG +Ln9fRMDC2j3IAnQLf4YOznOJ57CjMd2Wmn/yhHK8h9s4iU5zw3+PK+X/IM4GeAfe +JMx8c5uq2A8A24uzMidyhxJCK7VUprjK/ckdNYya6dkG2De+LR7W82ygfWbFDOnZ +KM26cPG/AgMBAAECggEAFtRko2J5xBcf2JDTLt0SF/wo8Nak1Ydi9pDDjgNoFdR0 +n/vQBfvhPhxpxYysTvRO2eHuKvSw2zGredXIRmf82r8f9vokWuyZQt4fvTOfnzSv +uIeWx/pVLDM9/8vhePN5aEmSKtzrt1rfoQMx/eGk6RwxfuxI25MKqDP30O9lrHTn +Y0lW7dthgdDMlQnSpOqUm2ldDsykYCBFteh4i5RDzAhiGx1ryaz3FMg+/y0VTTk0 +BM43qW6H9PD8P4iOau3DGIPNqtIlFSnWoYaM6Ta2osfzzdsnFbe5F7JbdMrf5MBc +Jq3VMUqffRmHubz7di03qRsRqGYQn2cJeiuVC+y6gQKBgQDYpq3MfMjwzPeoB1Ay +ZQdzx+T290XRxFZwkiv3uugsYMlFGEabdAMFx5oIIOdjWSBLI92RvXbg7qMd/xMC +ya/GzbKQd+5GbRLW+TZ0odGkMFkTo+DEkt07yEM8mrPJ6XePUndHbiNFSdpVKx4g +KdmiRHinm3R8Lr5/puvISrOdcwKBgQDA1kln9aD1mvIdObI6MubPitb+NuNcpVDo +myc1UrEJbcn8nBbLb+0Q+7gckjau2C8GN7Olnd8RCYLc7kU1On2pY+f19Ru/PdZX +cjCCTcxqCJvWkNWOzw14ag6UrDTF5nxtoVl/eXbHxWqFjdt0a211sa1mp3Gn3ZNq +m/teImYHhQKBgQCzWUA1MPPzi+pU2kEEhugla8xauha9cUiRhiAJw1uiKTlVDqSc +2ewKo9MaeYqzjruSGI26sVqxGDxGf7tQKoBuFiiFOhMxj+fxuHrhEHiI8FE9VgOj +F2U3sTAgAn1lX/VO21jM9BsUp++rY7dbrulwUDiFn8ZNazDeYeN8eoK4iwKBgQCb +cqJN+YW9NyCBSqdPnwTMvSE+YES7xFAKkjfzFiu8bBJtXe5KJHm4PRJXhc4q9/5A +Rtq8YR0WgNJLApArrnDqAa1Vajbp3RFSAKz1/X0Q5MurFanxqxsyvFvwoTkRZxFa +1rxstB96Prv12TrVCFx+ibI8lDJcnZNeV0s0wQn6eQKBgQDXkfPuX5TFBpNe1bWI +KUFmw9R1ynmUlIOaU3ITLv9C+w8zaJSpxFDZgJdv3uT8PfnXrsHm+lWjaOunvjri +quZSc06mLlEbggYoIFQNPeNPRyN0+GLvefMS3mCotzanZTmD5GrH9XG451tVPiH9 +G/lpNA1ccRCCsLslcG/aaa5PQw== +-----END PRIVATE KEY----- ` ) From beb1bf6d2cc23b258c29c05d81b8b625e28c3a8d Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Fri, 28 Jul 2023 19:28:20 +0530 Subject: [PATCH 44/62] headless: automerge and other improvements (#3958) * headless: automerge and other improvements * fix typo in function signature --- v2/pkg/protocols/headless/engine/instance.go | 10 +- v2/pkg/protocols/headless/engine/page.go | 2 +- .../protocols/headless/engine/page_actions.go | 98 +++++++++++-------- v2/pkg/protocols/headless/request.go | 38 +++++-- v2/pkg/protocols/http/build_request.go | 12 +-- .../{http/utils => utils/http}/requtils.go | 2 +- .../utils => utils/http}/requtils_test.go | 2 +- 7 files changed, 104 insertions(+), 60 deletions(-) rename v2/pkg/protocols/{http/utils => utils/http}/requtils.go (98%) rename v2/pkg/protocols/{http/utils => utils/http}/requtils_test.go (99%) diff --git a/v2/pkg/protocols/headless/engine/instance.go b/v2/pkg/protocols/headless/engine/instance.go index 90ffccac2..f0dae7828 100644 --- a/v2/pkg/protocols/headless/engine/instance.go +++ b/v2/pkg/protocols/headless/engine/instance.go @@ -17,6 +17,7 @@ type Instance struct { // redundant due to dependency cycle interactsh *interactsh.Client + requestLog map[string]string // contains actual request that was sent } // NewInstance creates a new instance for the current browser. @@ -35,7 +36,14 @@ func (b *Browser) NewInstance() (*Instance, error) { // We use a custom sleeper that sleeps from 100ms to 500 ms waiting // for an interaction. Used throughout rod for clicking, etc. browser = browser.Sleeper(func() utils.Sleeper { return maxBackoffSleeper(10) }) - return &Instance{browser: b, engine: browser}, nil + return &Instance{browser: b, engine: browser, requestLog: map[string]string{}}, nil +} + +// returns a map of [template-defined-urls] -> [actual-request-sent] +// Note: this does not include CORS or other requests while rendering that were not explicitly +// specified in template +func (i *Instance) GetRequestLog() map[string]string { + return i.requestLog } // Close closes all the tabs and pages for a browser instance diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index e23d2eb25..a566ab37a 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -134,7 +134,7 @@ func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads m } } - data, err := createdPage.ExecuteActions(input, actions) + data, err := createdPage.ExecuteActions(input, actions, payloads) if err != nil { return nil, nil, err } diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index b96d4f264..93cd5018c 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -2,11 +2,8 @@ package engine import ( "context" - "net" - "net/url" "os" "path/filepath" - "regexp" "strconv" "strings" "sync" @@ -19,17 +16,21 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" "github.com/segmentio/ksuid" ) var ( errinvalidArguments = errors.New("invalid arguments provided") - reUrlWithPort = regexp.MustCompile(`{{BaseURL}}:(\d+)`) ) const ( @@ -39,17 +40,13 @@ const ( ) // ExecuteActions executes a list of actions on a page. -func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (map[string]string, error) { - baseURL, err := url.Parse(input.MetaInput.Input) - if err != nil { - return nil, err - } - +func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (map[string]string, error) { outData := make(map[string]string) + var err error for _, act := range actions { switch act.ActionType.ActionType { case ActionNavigate: - err = p.NavigateURL(act, outData, baseURL) + err = p.NavigateURL(act, outData, variables) case ActionScript: err = p.RunScript(act, outData) case ActionClick: @@ -237,25 +234,57 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { } // NavigateURL executes an ActionLoadURL actions loading a URL for the page. -func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error { - URL := p.getActionArgWithDefaultValues(action, "url") - if URL == "" { +func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[string]interface{}) error { + // input <- is input url from cli + // target <- is the url from template (ex: {{BaseURL}}/test) + input, err := urlutil.Parse(p.input.MetaInput.Input) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not parse url %s", p.input.MetaInput.Input) + } + target := p.getActionArgWithDefaultValues(action, "url") + if target == "" { return errinvalidArguments } - // Handle the dynamic value substitution here. - URL, parsed = baseURLWithTemplatePrefs(URL, parsed) - if strings.HasSuffix(parsed.Path, "/") && strings.Contains(URL, "{{BaseURL}}/") { - parsed.Path = strings.TrimSuffix(parsed.Path, "/") - } - parsedString := parsed.String() - final := replaceWithValues(URL, map[string]interface{}{ - "Hostname": parsed.Hostname(), - "BaseURL": parsedString, - }) + // if target contains port ex: {{BaseURL}}:8080 use port specified in input + input, target = httputil.UpdateURLPortFromPayload(input, target) + hasTrailingSlash := httputil.HasTrailingSlash(target) - if err := p.page.Navigate(final); err != nil { - return errors.Wrap(err, "could not navigate") + // create vars from input url + defaultReqVars := protocolutils.GenerateVariables(input, hasTrailingSlash, contextargs.GenerateVariables(p.input)) + // merge all variables + // Note: ideally we should evaluate all available variables with reqvars + // but due to cyclic dependency between packages `engine` and `protocols` + // allvars are evaluated,merged and passed from headless package itself + // TODO: remove cyclic dependency between packages `engine` and `protocols` + allvars = generators.MergeMaps(allvars, defaultReqVars) + + if vardump.EnableVarDump { + gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) + } + + // Evaluate the target url with all variables + target, err = expressions.Evaluate(target, allvars) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not evaluate url %s", target) + } + + reqURL, err := urlutil.ParseURL(target, true) + if err != nil { + return errorutil.NewWithTag("http", "failed to parse url %v while creating http request", target) + } + + // ===== parameter automerge ===== + // while merging parameters first preference is given to target params + finalparams := input.Params.Clone() + finalparams.Merge(reqURL.Params.Encode()) + reqURL.Params = finalparams + + // log all navigated requests + p.instance.requestLog[action.GetArg("url")] = reqURL.String() + + if err := p.page.Navigate(reqURL.String()); err != nil { + return errorutil.NewWithErr(err).Msgf("could not navigate to url %s", reqURL.String()) } return nil } @@ -609,23 +638,6 @@ func selectorBy(selector string) rod.SelectorType { } } -// baseURLWithTemplatePrefs returns the url for BaseURL keeping -// the template port and path preference over the user provided one. -func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { - // template port preference over input URL port if template has a port - matches := reUrlWithPort.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return data, parsed - } - port := matches[0][1] - parsed.Host = net.JoinHostPort(parsed.Hostname(), port) - data = strings.ReplaceAll(data, ":"+port, "") - if parsed.Path == "" { - parsed.Path = "/" - } - return data, parsed -} - func (p *Page) getActionArg(action *Action, arg string) string { return p.getActionArgWithValues(action, arg, nil) } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index b7f40eb41..f9ce8e6a9 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -1,6 +1,7 @@ package headless import ( + "fmt" "net/url" "strings" "time" @@ -119,21 +120,31 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p } defer page.Close() + reqLog := instance.GetRequestLog() + navigatedURL := request.getLastNavigationURLWithLog(reqLog) // also known as matchedURL if there is a match + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil) request.options.Progress.IncrementRequests() - gologger.Verbose().Msgf("Sent Headless request to %s", input.MetaInput.Input) + gologger.Verbose().Msgf("Sent Headless request to %s", navigatedURL) reqBuilder := &strings.Builder{} if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input.MetaInput.Input) + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, navigatedURL) for _, act := range request.Steps { - actStepStr := act.String() - actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", input.MetaInput.Input) - reqBuilder.WriteString("\t" + actStepStr + "\n") + if act.ActionType.ActionType == engine.ActionNavigate { + value := act.GetArg("url") + if reqLog[value] != "" { + reqBuilder.WriteString(fmt.Sprintf("\tnavigate => %v\n", reqLog[value])) + } else { + reqBuilder.WriteString(fmt.Sprintf("%v not found in %v\n", value, reqLog)) + } + } else { + actStepStr := act.String() + reqBuilder.WriteString("\t" + actStepStr + "\n") + } } gologger.Debug().Msgf(reqBuilder.String()) - } var responseBody string @@ -142,7 +153,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory()) + outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory()) for k, v := range out { outputEvent[k] = v } @@ -215,3 +226,16 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads } return nil } + +// getLastNaviationURL returns last successfully navigated URL +func (request *Request) getLastNavigationURLWithLog(reqLog map[string]string) string { + for i := len(request.Steps) - 1; i >= 0; i-- { + if request.Steps[i].ActionType.ActionType == engine.ActionNavigate { + templateURL := request.Steps[i].GetArg("url") + if reqLog[templateURL] != "" { + return reqLog[templateURL] + } + } + } + return "" +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index eecd9eadf..af551d09c 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -17,8 +17,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/utils" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" @@ -97,8 +97,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, hasTrailingSlash := false if !isRawRequest { // if path contains port ex: {{BaseURL}}:8080 use port specified in reqData - parsed, reqData = utils.UpdateURLPortFromPayload(parsed, reqData) - hasTrailingSlash = utils.HasTrailingSlash(reqData) + parsed, reqData = httputil.UpdateURLPortFromPayload(parsed, reqData) + hasTrailingSlash = httputil.HasTrailingSlash(reqData) } // defaultreqvars are vars generated from request/input ex: {{baseURL}}, {{Host}} etc @@ -362,13 +362,13 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st req.Body = bodyReader } if !r.request.Unsafe { - utils.SetHeader(req, "User-Agent", uarand.GetRandom()) + httputil.SetHeader(req, "User-Agent", uarand.GetRandom()) } // Only set these headers on non-raw requests if len(r.request.Raw) == 0 && !r.request.Unsafe { - utils.SetHeader(req, "Accept", "*/*") - utils.SetHeader(req, "Accept-Language", "en") + httputil.SetHeader(req, "Accept", "*/*") + httputil.SetHeader(req, "Accept-Language", "en") } if !LeaveDefaultPorts { diff --git a/v2/pkg/protocols/http/utils/requtils.go b/v2/pkg/protocols/utils/http/requtils.go similarity index 98% rename from v2/pkg/protocols/http/utils/requtils.go rename to v2/pkg/protocols/utils/http/requtils.go index 7be1f2259..97bb49f6e 100644 --- a/v2/pkg/protocols/http/utils/requtils.go +++ b/v2/pkg/protocols/utils/http/requtils.go @@ -1,4 +1,4 @@ -package utils +package httputil import ( "regexp" diff --git a/v2/pkg/protocols/http/utils/requtils_test.go b/v2/pkg/protocols/utils/http/requtils_test.go similarity index 99% rename from v2/pkg/protocols/http/utils/requtils_test.go rename to v2/pkg/protocols/utils/http/requtils_test.go index 635d62855..056b13b81 100644 --- a/v2/pkg/protocols/http/utils/requtils_test.go +++ b/v2/pkg/protocols/utils/http/requtils_test.go @@ -1,4 +1,4 @@ -package utils +package httputil import ( "testing" From fd13225b27288dbf82b461fe789bcfec96f78156 Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Fri, 28 Jul 2023 10:18:15 -0400 Subject: [PATCH 45/62] Omit Empty Fields (#3977) * Don't show Lines/matched-line on null * Remove unused "info.references" property * Revert "Remove unused "info.references" property" This reverts commit 6466644bcac6952ece8d2bc880ea9157f2e10c16. * Switch to pointer so omitempty works properly * keeping matcher status output in jsonl output always * rename function to NewRawStringSlice --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Tarun Koyalwar --- v2/pkg/model/model.go | 2 +- v2/pkg/model/model_test.go | 4 ++-- v2/pkg/model/types/stringslice/stringslice_raw.go | 6 +++--- v2/pkg/output/output.go | 2 +- v2/pkg/reporting/format/format_utils_test.go | 5 +++-- v2/pkg/templates/templates_doc_examples.go | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index 63c0e1ee2..c3cf98c43 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -47,7 +47,7 @@ type Info struct { // examples: // - value: > // []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"} - Reference stringslice.RawStringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` + Reference *stringslice.RawStringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` // description: | // Severity of the template. SeverityHolder severity.Holder `json:"severity,omitempty" yaml:"severity,omitempty"` diff --git a/v2/pkg/model/model_test.go b/v2/pkg/model/model_test.go index aba1cf1c9..7a0ca188e 100644 --- a/v2/pkg/model/model_test.go +++ b/v2/pkg/model/model_test.go @@ -18,7 +18,7 @@ func TestInfoJsonMarshal(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.NewRaw("Reference1"), + Reference: stringslice.NewRawStringSlice("Reference1"), Metadata: map[string]interface{}{ "string_key": "string_value", "array_key": []string{"array_value1", "array_value2"}, @@ -42,7 +42,7 @@ func TestInfoYamlMarshal(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.NewRaw("Reference1"), + Reference: stringslice.NewRawStringSlice("Reference1"), Metadata: map[string]interface{}{ "string_key": "string_value", "array_key": []string{"array_value1", "array_value2"}, diff --git a/v2/pkg/model/types/stringslice/stringslice_raw.go b/v2/pkg/model/types/stringslice/stringslice_raw.go index 8ca38c5e8..494731ca5 100644 --- a/v2/pkg/model/types/stringslice/stringslice_raw.go +++ b/v2/pkg/model/types/stringslice/stringslice_raw.go @@ -4,10 +4,10 @@ type RawStringSlice struct { StringSlice } -func NewRaw(value interface{}) RawStringSlice { - return RawStringSlice{StringSlice: StringSlice{Value: value}} +func NewRawStringSlice(value interface{}) *RawStringSlice { + return &RawStringSlice{StringSlice: StringSlice{Value: value}} } -func (rawStringSlice RawStringSlice) Normalize(value string) string { +func (rawStringSlice *RawStringSlice) Normalize(value string) string { return value } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index a752eee18..3d3b8e847 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -144,7 +144,7 @@ type ResultEvent struct { // MatcherStatus is the status of the match MatcherStatus bool `json:"matcher-status"` // Lines is the line count for the specified match - Lines []int `json:"matched-line"` + Lines []int `json:"matched-line,omitempty"` FileToIndexPosition map[string]int `json:"-"` } diff --git a/v2/pkg/reporting/format/format_utils_test.go b/v2/pkg/reporting/format/format_utils_test.go index 2a950b50b..97f258aa9 100644 --- a/v2/pkg/reporting/format/format_utils_test.go +++ b/v2/pkg/reporting/format/format_utils_test.go @@ -1,10 +1,11 @@ package format import ( - "github.com/stretchr/testify/assert" "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" @@ -18,7 +19,7 @@ func TestToMarkdownTableString(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.NewRaw("reference1"), + Reference: stringslice.NewRawStringSlice("reference1"), Metadata: map[string]interface{}{ "customDynamicKey1": "customDynamicValue1", "customDynamicKey2": "customDynamicValue2", diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 496bc10b8..c2650d091 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -20,7 +20,7 @@ var ( Name: "Argument Injection in Ruby Dragonfly", Authors: stringslice.StringSlice{Value: "0xspara"}, SeverityHolder: severity.Holder{Severity: severity.High}, - Reference: stringslice.NewRaw("https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"), + Reference: stringslice.NewRawStringSlice("https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"), Tags: stringslice.StringSlice{Value: "cve,cve2021,rce,ruby"}, } exampleNormalHTTPRequest = &http.Request{ From 9adce978b41e37d9d91327f5b843f2191bad651f Mon Sep 17 00:00:00 2001 From: lu4nx Date: Fri, 28 Jul 2023 23:31:28 +0800 Subject: [PATCH 46/62] Support wildcard matching for the `-template-id` parameter (#3967) * Support wildcard matching for the `-template-id' parameter * Remove _ identifier * Fix: If tagFilter.allowedIds is empty, included should be true. --- v2/cmd/nuclei/main.go | 2 +- v2/pkg/catalog/loader/filter/tag_filter.go | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index bd68cfc30..67f15b2e4 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -151,7 +151,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVar(&options.Tags, "tags", nil, "templates to run based on tags (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", nil, "templates to exclude based on tags (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.IncludeTags, "include-tags", "itags", nil, "tags to be executed even if they are excluded either by default or configuration", goflags.FileNormalizedStringSliceOptions), // TODO show default deny list - flagSet.StringSliceVarP(&options.IncludeIds, "template-id", "id", nil, "templates to run based on template ids (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.IncludeIds, "template-id", "id", nil, "templates to run based on template ids (comma-separated, file, allow-wildcard)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", nil, "templates to exclude based on template ids (comma-separated, file)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", nil, "templates to be executed even if they are excluded either by default or configuration", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", nil, "template or template directory to exclude (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index 1da6090e3..638da945b 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "strings" + "path/filepath" "github.com/Knetic/govaluate" "github.com/projectdiscovery/gologger" @@ -170,9 +171,18 @@ func isIdMatch(tagFilter *TagFilter, templateId string) bool { if len(tagFilter.excludeIds) == 0 && len(tagFilter.allowedIds) == 0 { return true } - included := true - if len(tagFilter.allowedIds) > 0 { - _, included = tagFilter.allowedIds[templateId] + + included := len(tagFilter.allowedIds) == 0 + for id := range tagFilter.allowedIds { + match, err := filepath.Match(id, templateId) + if err != nil { + continue + } + + if match { + included = true + break + } } excluded := false From 6bdef687345d0fef6ca3e11c42ae98042b9a152e Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:04:02 +0530 Subject: [PATCH 47/62] ignore version parsing error (#3984) * ignore version parsing error * hide no parameter error * integration test+ DEBUG.md * typo fix in DEBUG.md * go mod tidy --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- DEBUG.md | 42 +++++++++++++++++ .../dsl/hide-version-warning.yaml | 18 +++++++ .../dsl/show-version-warning.yaml | 18 +++++++ v2/cmd/integration-test/dsl.go | 47 +++++++++++++++++++ v2/cmd/integration-test/integration-test.go | 1 + v2/go.mod | 18 +++---- v2/go.sum | 36 +++++++------- v2/pkg/operators/matchers/match.go | 24 ++++++++-- v2/pkg/testutils/integration.go | 30 ++++++++++++ 9 files changed, 204 insertions(+), 30 deletions(-) create mode 100644 DEBUG.md create mode 100644 integration_tests/dsl/hide-version-warning.yaml create mode 100644 integration_tests/dsl/show-version-warning.yaml create mode 100644 v2/cmd/integration-test/dsl.go diff --git a/DEBUG.md b/DEBUG.md new file mode 100644 index 000000000..d217a1258 --- /dev/null +++ b/DEBUG.md @@ -0,0 +1,42 @@ +## Debugging Nuclei + +While Adding new features or fixing bugs or writing new templates to properly understand the behavior of that component, it is essential to understand what debugging options are available in nuclei. This guide lists all the debugging options available in nuclei. + +### Template related debugging + +- `-debug` flag + +When this flag is provided, nuclei will print all requests that are being sent by nuclei to the target as well as the response received from the target. + +- `-debug-req` flag + +When this flag is provided, nuclei will print all requests that are being sent by nuclei to the target. + +- `-debug-resp` flag + +When this flag is provided, nuclei will print all responses that are being received by nuclei from the target. + +- `-ldf` flag + +When this flag is provided, nuclei will print the list of all helper functions available in this release of nuclei and exit. + +- `-svd` flag + +When this flag is provided, nuclei will print all `variables` pre and post execution of a request for a template. This is useful to understand what variables are available for a template and what values they have. + +- `-elog = errors.txt` flag + +When this flag is provided, nuclei will log all errors to the file specified. This is helpful when running large scans. + + + +### Environment Variable Switches + +Nuclei was built with some environment variables in mind to help with debugging. These environment variables can be set to enable debugging of a particular component/functionality for nuclei. + +| Environment Variable | Description | +| ---------------------- | -------------------------------------------------------- | +| `DEBUG=true` | Enables Printing Stack Traces for all errors | +| `SHOW_DSL_ERRORS=true` | Enables Printing DSL Errors (that are hidden by default) | + + diff --git a/integration_tests/dsl/hide-version-warning.yaml b/integration_tests/dsl/hide-version-warning.yaml new file mode 100644 index 000000000..32df2954f --- /dev/null +++ b/integration_tests/dsl/hide-version-warning.yaml @@ -0,0 +1,18 @@ +id: basic-example + +info: + name: Test HTTP Template + author: pdteam + severity: info + reference: | + test case for default behaviour of version warning (dsl parsing error) + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: dsl + dsl: + - compare_versions("GG", '< 4.8.5') \ No newline at end of file diff --git a/integration_tests/dsl/show-version-warning.yaml b/integration_tests/dsl/show-version-warning.yaml new file mode 100644 index 000000000..b0d5aa8b7 --- /dev/null +++ b/integration_tests/dsl/show-version-warning.yaml @@ -0,0 +1,18 @@ +id: basic-example + +info: + name: Test HTTP Template + author: pdteam + severity: info + reference: | + test case where version warning is shown when env `SHOW_DSL_ERRORS=true` is set + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: dsl + dsl: + - compare_versions("GG", '< 4.8.5') \ No newline at end of file diff --git a/v2/cmd/integration-test/dsl.go b/v2/cmd/integration-test/dsl.go new file mode 100644 index 000000000..7d60e8f5f --- /dev/null +++ b/v2/cmd/integration-test/dsl.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var dslTestcases = map[string]testutils.TestCase{ + "dsl/hide-version-warning.yaml": &dslVersionWarning{}, + "dsl/show-version-warning.yaml": &dslShowVersionWarning{}, +} + +type dslVersionWarning struct{} + +func (d *dslVersionWarning) Execute(templatePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "DSL version parsing warning test") + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiArgsAndGetErrors(debug, nil, "-t", templatePath, "-target", ts.URL, "-v") + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + +type dslShowVersionWarning struct{} + +func (d *dslShowVersionWarning) Execute(templatePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "DSL version parsing warning test") + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiArgsAndGetErrors(debug, []string{"SHOW_DSL_ERRORS=true"}, "-t", templatePath, "-target", ts.URL, "-v") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 2012f1132..381a62775 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -40,6 +40,7 @@ var ( "customConfigDir": customConfigDirTestCases, "fuzzing": fuzzingTestCases, "generic": genericTestcases, + "dsl": dslTestcases, } // For debug purposes diff --git a/v2/go.mod b/v2/go.mod index 97b5e8ebe..d2cb85319 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -66,7 +66,7 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.14 + github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.11 @@ -77,7 +77,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.0 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.41-0.20230705082547-236cfa9298ab + github.com/projectdiscovery/utils v0.0.44 github.com/projectdiscovery/wappalyzergo v0.0.104 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -101,11 +101,11 @@ require ( github.com/bits-and-blooms/bitset v1.3.1 // indirect github.com/bits-and-blooms/bloom/v3 v3.4.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect - github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaukas/godicttls v0.0.3 // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect @@ -128,7 +128,7 @@ require ( github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/refraction-networking/utls v1.3.2 // indirect - github.com/sashabaranov/go-openai v1.13.0 // indirect + github.com/sashabaranov/go-openai v1.14.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect @@ -190,7 +190,7 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mholt/acmez v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.24 // indirect + github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -216,11 +216,11 @@ require ( go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect golang.org/x/crypto v0.11.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/mod v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 + golang.org/x/mod v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.10.0 // indirect + golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/v2/go.sum b/v2/go.sum index 394e4ec9b..d09bc9227 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -126,8 +126,8 @@ github.com/caddyserver/certmagic v0.17.2 h1:o30seC1T/dBqBCNNGNHWwj2i5/I/FMjBbTAh github.com/caddyserver/certmagic v0.17.2/go.mod h1:ouWUuC490GOLJzkyN35eXfV8bSbwMwSf4bdhkIxtdQE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= -github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= -github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= +github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -158,8 +158,8 @@ github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3O github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -339,8 +339,8 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= -github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= +github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= +github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= @@ -396,8 +396,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.14 h1:CAxCoYbIEBCuINiMR1UKA1v6ifmub3P5hCwzBmmkh0c= -github.com/projectdiscovery/dsl v0.0.14/go.mod h1:3K2GmExpriruVHsVJmsTugxR7H9wVpUo8/+jWXXbSSw= +github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= +github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= github.com/projectdiscovery/fastdialer v0.0.33 h1:FKXZjJme5nhgnnRL0Y4KjZ5YXKg03dX5ZJYFJV3LOwU= github.com/projectdiscovery/fastdialer v0.0.33/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -435,8 +435,8 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.41-0.20230705082547-236cfa9298ab h1:KcbRfus364It55dhAUpbqFHfyCuIa8Ls/9QzMWYKq78= -github.com/projectdiscovery/utils v0.0.41-0.20230705082547-236cfa9298ab/go.mod h1:DTFCMSLh8FanDZIrzOwTo3AIv1K4w0PDELi41mjwLiw= +github.com/projectdiscovery/utils v0.0.44 h1:F/LNgBw53RNM/3mRZ1ji+prM1yDnehDRBf13TPk3WBM= +github.com/projectdiscovery/utils v0.0.44/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= github.com/projectdiscovery/wappalyzergo v0.0.104 h1:hdda6WxAzXVpLBbJW1sLqrwOXHn0prP9IYFY7dfCMjE= github.com/projectdiscovery/wappalyzergo v0.0.104/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -454,8 +454,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.13.0 h1:EAusFfnhaMaaUspUZ2+MbB/ZcVeD4epJmTOlZ+8AcAE= -github.com/sashabaranov/go-openai v1.13.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.14.1 h1:jqfkdj8XHnBF84oi2aNtT8Ktp3EJ0MfuVjvcMkfI0LA= +github.com/sashabaranov/go-openai v1.14.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -619,15 +619,15 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -739,8 +739,8 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index abd8bf336..aae6f3a94 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -1,13 +1,21 @@ package matchers import ( + "os" "strings" "github.com/Knetic/govaluate" + dslRepo "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + stringsutil "github.com/projectdiscovery/utils/strings" +) + +var ( + // showDSLErr controls whether to show hidden DSL errors or not + showDSLErr = strings.EqualFold(os.Getenv("SHOW_DSL_ERRORS"), "true") ) // MatchStatusCode matches a status code check against a corpus @@ -185,10 +193,8 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { if matcher.condition == ANDCondition { return false } - if strings.Contains(err.Error(), "No parameter") { + if !matcher.ignoreErr(err) { gologger.Warning().Msgf("[%s] %s", data["template-id"], err.Error()) - } else { - gologger.Error().Label("WRN").Msgf("[%s] %s", data["template-id"], err.Error()) } continue } @@ -219,3 +225,15 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { } return false } + +// ignoreErr checks if the error is to be ignored or not +// Reference: https://github.com/projectdiscovery/nuclei/issues/3950 +func (m *Matcher) ignoreErr(err error) bool { + if showDSLErr { + return false + } + if stringsutil.ContainsAny(err.Error(), "No parameter", dslRepo.ErrParsingArg.Error()) { + return true + } + return false +} diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index 56d32d5c3..bd773a9c3 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -77,6 +77,36 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro return parts, nil } +// RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) +func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei") + extra = append(extra, ExtraDebugArgs...) + cmd.Env = append(os.Environ(), env...) + cmd.Args = append(cmd.Args, extra...) + cmd.Args = append(cmd.Args, "-duc") // disable auto updates + cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") + cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") + cmd.Args = append(cmd.Args, "-allow-local-file-access") + cmd.Args = append(cmd.Args, "-nc") // disable color + data, err := cmd.CombinedOutput() + if debug { + fmt.Println(string(data)) + } + results := []string{} + for _, v := range strings.Split(string(data), "\n") { + line := strings.TrimSpace(v) + switch { + case strings.HasPrefix(line, "[ERR]"): + results = append(results, line) + case strings.HasPrefix(line, "[WRN]"): + results = append(results, line) + case strings.HasPrefix(line, "[FTL]"): + results = append(results, line) + } + } + return results, err +} + var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) // RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template From 163bc22281f8c05961f78e8f1987959387587c78 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 28 Jul 2023 18:50:57 +0300 Subject: [PATCH 48/62] add headless options flag (#3951) * add headless options flag * disable some tests for windows * disable interactsh tests on darwin * disable network/hex.yaml on windows * make DisableOn func --- README.md | 11 ++- v2/cmd/integration-test/code.go | 6 +- v2/cmd/integration-test/custom-dir.go | 4 +- v2/cmd/integration-test/dns.go | 16 +-- v2/cmd/integration-test/file.go | 10 +- v2/cmd/integration-test/fuzz.go | 10 +- v2/cmd/integration-test/generic.go | 4 +- v2/cmd/integration-test/headless.go | 16 +-- v2/cmd/integration-test/http.go | 104 ++++++++++---------- v2/cmd/integration-test/integration-test.go | 48 +++++---- v2/cmd/integration-test/interactsh.go | 10 +- v2/cmd/integration-test/loader.go | 14 +-- v2/cmd/integration-test/network.go | 15 +-- v2/cmd/integration-test/offline-http.go | 8 +- v2/cmd/integration-test/ssl.go | 12 +-- v2/cmd/integration-test/template-dir.go | 4 +- v2/cmd/integration-test/template-path.go | 10 +- v2/cmd/integration-test/websocket.go | 10 +- v2/cmd/integration-test/whois.go | 4 +- v2/cmd/integration-test/workflow.go | 16 +-- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/options.go | 4 + v2/internal/runner/options_test.go | 61 ++++++++++++ v2/pkg/protocols/headless/engine/engine.go | 6 ++ v2/pkg/types/types.go | 17 ++++ 25 files changed, 255 insertions(+), 166 deletions(-) create mode 100644 v2/internal/runner/options_test.go diff --git a/README.md b/README.md index 4929cb7ca..e60b5e202 100644 --- a/README.md +++ b/README.md @@ -246,11 +246,12 @@ OPTIMIZATIONS: -no-stdin disable stdin processing HEADLESS: - -headless enable templates that require headless browser support (root user on Linux will disable sandbox) - -page-timeout int seconds to wait for each page in headless mode (default 20) - -sb, -show-browser show the browser on the screen when running templates with headless mode - -sc, -system-chrome use local installed Chrome browser instead of nuclei installed - -lha, -list-headless-action list available headless actions + -headless enable templates that require headless browser support (root user on Linux will disable sandbox) + -page-timeout int seconds to wait for each page in headless mode (default 20) + -sb, -show-browser show the browser on the screen when running templates with headless mode + -ho, -headless-options string[] start headless chrome with additional options + -sc, -system-chrome use local installed Chrome browser instead of nuclei installed + -lha, -list-headless-action list available headless actions DEBUG: -debug show all requests and responses diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index 6835f86b2..b2b5f94cc 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -34,9 +34,9 @@ import ( "github.com/projectdiscovery/ratelimit" ) -var codeTestcases = map[string]testutils.TestCase{ - "code/test.yaml": &goIntegrationTest{}, - "code/test.json": &goIntegrationTest{}, +var codeTestcases = []TestCaseInfo{ + {Path: "code/test.yaml", TestCase: &goIntegrationTest{}}, + {Path: "code/test.json", TestCase: &goIntegrationTest{}}, } type goIntegrationTest struct{} diff --git a/v2/cmd/integration-test/custom-dir.go b/v2/cmd/integration-test/custom-dir.go index 83f2b1e96..774a367d4 100644 --- a/v2/cmd/integration-test/custom-dir.go +++ b/v2/cmd/integration-test/custom-dir.go @@ -8,8 +8,8 @@ import ( type customConfigDirTest struct{} -var customConfigDirTestCases = map[string]testutils.TestCase{ - "dns/cname-fingerprint.yaml": &customConfigDirTest{}, +var customConfigDirTestCases = []TestCaseInfo{ + {Path: "dns/cname-fingerprint.yaml", TestCase: &customConfigDirTest{}}, } // Execute executes a test case and returns an error if occurred diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index 08fac28c9..570f734a2 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -4,14 +4,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var dnsTestCases = map[string]testutils.TestCase{ - "dns/basic.yaml": &dnsBasic{}, - "dns/ptr.yaml": &dnsPtr{}, - "dns/caa.yaml": &dnsCAA{}, - "dns/tlsa.yaml": &dnsTLSA{}, - "dns/variables.yaml": &dnsVariables{}, - "dns/payload.yaml": &dnsPayload{}, - "dns/dsl-matcher-variable.yaml": &dnsDSLMatcherVariable{}, +var dnsTestCases = []TestCaseInfo{ + {Path: "dns/basic.yaml", TestCase: &dnsBasic{}}, + {Path: "dns/ptr.yaml", TestCase: &dnsPtr{}}, + {Path: "dns/caa.yaml", TestCase: &dnsCAA{}}, + {Path: "dns/tlsa.yaml", TestCase: &dnsTLSA{}}, + {Path: "dns/variables.yaml", TestCase: &dnsVariables{}}, + {Path: "dns/payload.yaml", TestCase: &dnsPayload{}}, + {Path: "dns/dsl-matcher-variable.yaml", TestCase: &dnsDSLMatcherVariable{}}, } type dnsBasic struct{} diff --git a/v2/cmd/integration-test/file.go b/v2/cmd/integration-test/file.go index 079b3a372..c5351f305 100644 --- a/v2/cmd/integration-test/file.go +++ b/v2/cmd/integration-test/file.go @@ -4,11 +4,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var fileTestcases = map[string]testutils.TestCase{ - "file/matcher-with-or.yaml": &fileWithOrMatcher{}, - "file/matcher-with-and.yaml": &fileWithAndMatcher{}, - "file/matcher-with-nested-and.yaml": &fileWithAndMatcher{}, - "file/extract.yaml": &fileWithExtractor{}, +var fileTestcases = []TestCaseInfo{ + {Path: "file/matcher-with-or.yaml", TestCase: &fileWithOrMatcher{}}, + {Path: "file/matcher-with-and.yaml", TestCase: &fileWithAndMatcher{}}, + {Path: "file/matcher-with-nested-and.yaml", TestCase: &fileWithAndMatcher{}}, + {Path: "file/extract.yaml", TestCase: &fileWithExtractor{}}, } type fileWithOrMatcher struct{} diff --git a/v2/cmd/integration-test/fuzz.go b/v2/cmd/integration-test/fuzz.go index 2a9573d23..5dd048d32 100644 --- a/v2/cmd/integration-test/fuzz.go +++ b/v2/cmd/integration-test/fuzz.go @@ -12,11 +12,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var fuzzingTestCases = map[string]testutils.TestCase{ - "fuzz/fuzz-mode.yaml": &fuzzModeOverride{}, - "fuzz/fuzz-type.yaml": &fuzzTypeOverride{}, - "fuzz/fuzz-query.yaml": &httpFuzzQuery{}, - "fuzz/fuzz-headless.yaml": &HeadlessFuzzingQuery{}, +var fuzzingTestCases = []TestCaseInfo{ + {Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}}, + {Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}}, + {Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}}, + {Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}}, } type httpFuzzQuery struct{} diff --git a/v2/cmd/integration-test/generic.go b/v2/cmd/integration-test/generic.go index b22ffd166..8b66257a0 100644 --- a/v2/cmd/integration-test/generic.go +++ b/v2/cmd/integration-test/generic.go @@ -13,8 +13,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var genericTestcases = map[string]testutils.TestCase{ - "generic/auth/certificate/http-get.yaml": &clientCertificate{}, +var genericTestcases = []TestCaseInfo{ + {Path: "generic/auth/certificate/http-get.yaml", TestCase: &clientCertificate{}}, } var ( diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go index e4c70543d..185cdc87f 100644 --- a/v2/cmd/integration-test/headless.go +++ b/v2/cmd/integration-test/headless.go @@ -10,14 +10,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var headlessTestcases = map[string]testutils.TestCase{ - "headless/headless-basic.yaml": &headlessBasic{}, - "headless/headless-header-action.yaml": &headlessHeaderActions{}, - "headless/headless-extract-values.yaml": &headlessExtractValues{}, - "headless/headless-payloads.yaml": &headlessPayloads{}, - "headless/variables.yaml": &headlessVariables{}, - "headless/file-upload.yaml": &headlessFileUpload{}, - "headless/headless-header-status-test.yaml": &headlessHeaderStatus{}, +var headlessTestcases = []TestCaseInfo{ + {Path: "headless/headless-basic.yaml", TestCase: &headlessBasic{}}, + {Path: "headless/headless-header-action.yaml", TestCase: &headlessHeaderActions{}}, + {Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}}, + {Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}}, + {Path: "headless/variables.yaml", TestCase: &headlessVariables{}}, + {Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}}, + {Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}}, } type headlessBasic struct{} diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index a1b67c575..75e80e186 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -26,60 +26,60 @@ import ( stringsutil "github.com/projectdiscovery/utils/strings" ) -var httpTestcases = map[string]testutils.TestCase{ +var httpTestcases = []TestCaseInfo{ // TODO: excluded due to parsing errors with console // "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, - "http/get-headers.yaml": &httpGetHeaders{}, - "http/get-query-string.yaml": &httpGetQueryString{}, - "http/get-redirects.yaml": &httpGetRedirects{}, - "http/get-host-redirects.yaml": &httpGetHostRedirects{}, - "http/disable-redirects.yaml": &httpDisableRedirects{}, - "http/get.yaml": &httpGet{}, - "http/post-body.yaml": &httpPostBody{}, - "http/post-json-body.yaml": &httpPostJSONBody{}, - "http/post-multipart-body.yaml": &httpPostMultipartBody{}, - "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{}, - "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{}, - "http/raw-get-query.yaml": &httpRawGetQuery{}, - "http/raw-get.yaml": &httpRawGet{}, - "http/raw-with-params.yaml": &httpRawWithParams{}, - "http/raw-unsafe-with-params.yaml": &httpRawWithParams{}, // Not a typo, functionality is same as above - "http/raw-path-trailing-slash.yaml": &httpRawPathTrailingSlash{}, - "http/raw-payload.yaml": &httpRawPayload{}, - "http/raw-post-body.yaml": &httpRawPostBody{}, - "http/raw-unsafe-path.yaml": &httpRawUnsafePath{}, - "http/http-paths.yaml": &httpPaths{}, - "http/request-condition.yaml": &httpRequestCondition{}, - "http/request-condition-new.yaml": &httpRequestCondition{}, - "http/self-contained.yaml": &httpRequestSelfContained{}, - "http/self-contained-with-path.yaml": &httpRequestSelfContained{}, // Not a typo, functionality is same as above - "http/self-contained-with-params.yaml": &httpRequestSelfContainedWithParams{}, - "http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{}, - "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, - "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, - "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, - "http/dsl-matcher-variable.yaml": &httpDSLVariable{}, - "http/dsl-functions.yaml": &httpDSLFunctions{}, - "http/race-simple.yaml": &httpRaceSimple{}, - "http/race-multiple.yaml": &httpRaceMultiple{}, - "http/stop-at-first-match.yaml": &httpStopAtFirstMatch{}, - "http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{}, - "http/variables.yaml": &httpVariables{}, - "http/variable-dsl-function.yaml": &httpVariableDSLFunction{}, - "http/get-override-sni.yaml": &httpSniAnnotation{}, - "http/get-sni.yaml": &customCLISNI{}, - "http/redirect-match-url.yaml": &httpRedirectMatchURL{}, - "http/get-sni-unsafe.yaml": &customCLISNIUnsafe{}, - "http/annotation-timeout.yaml": &annotationTimeout{}, - "http/custom-attack-type.yaml": &customAttackType{}, - "http/get-all-ips.yaml": &scanAllIPS{}, - "http/get-without-scheme.yaml": &httpGetWithoutScheme{}, - "http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{}, - "http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, - "http/save-extractor-values-to-file.yaml": &httpSaveExtractorValuesToFile{}, - "http/cli-with-constants.yaml": &ConstantWithCliVar{}, - "http/matcher-status.yaml": &matcherStatusTest{}, - "http/disable-path-automerge.yaml": &httpDisablePathAutomerge{}, + {Path: "http/get-headers.yaml", TestCase: &httpGetHeaders{}}, + {Path: "http/get-query-string.yaml", TestCase: &httpGetQueryString{}}, + {Path: "http/get-redirects.yaml", TestCase: &httpGetRedirects{}}, + {Path: "http/get-host-redirects.yaml", TestCase: &httpGetHostRedirects{}}, + {Path: "http/disable-redirects.yaml", TestCase: &httpDisableRedirects{}}, + {Path: "http/get.yaml", TestCase: &httpGet{}}, + {Path: "http/post-body.yaml", TestCase: &httpPostBody{}}, + {Path: "http/post-json-body.yaml", TestCase: &httpPostJSONBody{}}, + {Path: "http/post-multipart-body.yaml", TestCase: &httpPostMultipartBody{}}, + {Path: "http/raw-cookie-reuse.yaml", TestCase: &httpRawCookieReuse{}}, + {Path: "http/raw-dynamic-extractor.yaml", TestCase: &httpRawDynamicExtractor{}}, + {Path: "http/raw-get-query.yaml", TestCase: &httpRawGetQuery{}}, + {Path: "http/raw-get.yaml", TestCase: &httpRawGet{}}, + {Path: "http/raw-with-params.yaml", TestCase: &httpRawWithParams{}}, + {Path: "http/raw-unsafe-with-params.yaml", TestCase: &httpRawWithParams{}}, // Not a typo, functionality is same as above + {Path: "http/raw-path-trailing-slash.yaml", TestCase: &httpRawPathTrailingSlash{}}, + {Path: "http/raw-payload.yaml", TestCase: &httpRawPayload{}}, + {Path: "http/raw-post-body.yaml", TestCase: &httpRawPostBody{}}, + {Path: "http/raw-unsafe-path.yaml", TestCase: &httpRawUnsafePath{}}, + {Path: "http/http-paths.yaml", TestCase: &httpPaths{}}, + {Path: "http/request-condition.yaml", TestCase: &httpRequestCondition{}}, + {Path: "http/request-condition-new.yaml", TestCase: &httpRequestCondition{}}, + {Path: "http/self-contained.yaml", TestCase: &httpRequestSelfContained{}}, + {Path: "http/self-contained-with-path.yaml", TestCase: &httpRequestSelfContained{}}, // Not a typo, functionality is same as above + {Path: "http/self-contained-with-params.yaml", TestCase: &httpRequestSelfContainedWithParams{}}, + {Path: "http/self-contained-file-input.yaml", TestCase: &httpRequestSelfContainedFileInput{}}, + {Path: "http/get-case-insensitive.yaml", TestCase: &httpGetCaseInsensitive{}}, + {Path: "http/get.yaml,http/get-case-insensitive.yaml", TestCase: &httpGetCaseInsensitiveCluster{}}, + {Path: "http/get-redirects-chain-headers.yaml", TestCase: &httpGetRedirectsChainHeaders{}}, + {Path: "http/dsl-matcher-variable.yaml", TestCase: &httpDSLVariable{}}, + {Path: "http/dsl-functions.yaml", TestCase: &httpDSLFunctions{}}, + {Path: "http/race-simple.yaml", TestCase: &httpRaceSimple{}}, + {Path: "http/race-multiple.yaml", TestCase: &httpRaceMultiple{}}, + {Path: "http/stop-at-first-match.yaml", TestCase: &httpStopAtFirstMatch{}}, + {Path: "http/stop-at-first-match-with-extractors.yaml", TestCase: &httpStopAtFirstMatchWithExtractors{}}, + {Path: "http/variables.yaml", TestCase: &httpVariables{}}, + {Path: "http/variable-dsl-function.yaml", TestCase: &httpVariableDSLFunction{}}, + {Path: "http/get-override-sni.yaml", TestCase: &httpSniAnnotation{}}, + {Path: "http/get-sni.yaml", TestCase: &customCLISNI{}}, + {Path: "http/redirect-match-url.yaml", TestCase: &httpRedirectMatchURL{}}, + {Path: "http/get-sni-unsafe.yaml", TestCase: &customCLISNIUnsafe{}}, + {Path: "http/annotation-timeout.yaml", TestCase: &annotationTimeout{}}, + {Path: "http/custom-attack-type.yaml", TestCase: &customAttackType{}}, + {Path: "http/get-all-ips.yaml", TestCase: &scanAllIPS{}}, + {Path: "http/get-without-scheme.yaml", TestCase: &httpGetWithoutScheme{}}, + {Path: "http/cl-body-without-header.yaml", TestCase: &httpCLBodyWithoutHeader{}}, + {Path: "http/cl-body-with-header.yaml", TestCase: &httpCLBodyWithHeader{}}, + {Path: "http/save-extractor-values-to-file.yaml", TestCase: &httpSaveExtractorValuesToFile{}}, + {Path: "http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}}, + {Path: "http/matcher-status.yaml", TestCase: &matcherStatusTest{}}, + {Path: "http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}}, } type httpInteractshRequest struct{} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 381a62775..f878a76e8 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -4,7 +4,7 @@ import ( "flag" "fmt" "os" - "sort" + "runtime" "strings" "github.com/logrusorgru/aurora" @@ -13,6 +13,12 @@ import ( sliceutil "github.com/projectdiscovery/utils/slice" ) +type TestCaseInfo struct { + Path string + TestCase testutils.TestCase + DisableOn func() bool +} + var ( debug = os.Getenv("DEBUG") == "true" githubAction = os.Getenv("GH_ACTION") == "true" @@ -21,7 +27,7 @@ var ( success = aurora.Green("[✓]").String() failed = aurora.Red("[✘]").String() - protocolTests = map[string]map[string]testutils.TestCase{ + protocolTests = map[string][]TestCaseInfo{ "http": httpTestcases, "interactsh": interactshTestCases, "network": networkTestcases, @@ -99,18 +105,18 @@ func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCou } func debugTests() { - keys := getMapKeys(protocolTests[runProtocol]) - for _, tpath := range keys { - testcase := protocolTests[runProtocol][tpath] - if runTemplate != "" && !strings.Contains(tpath, runTemplate) { + testCaseInfos := protocolTests[runProtocol] + for _, testCaseInfo := range testCaseInfos { + if (runTemplate != "" && !strings.Contains(testCaseInfo.Path, runTemplate)) || + (testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn()) { continue } if runProtocol == "interactsh" { - if _, err := executeWithRetry(testcase, tpath, interactshRetryCount); err != nil { + if _, err := executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount); err != nil { fmt.Printf("\n%v", err.Error()) } } else { - if _, err := execute(testcase, tpath); err != nil { + if _, err := execute(testCaseInfo.TestCase, testCaseInfo.Path); err != nil { fmt.Printf("\n%v", err.Error()) } } @@ -120,21 +126,22 @@ func debugTests() { func runTests(customTemplatePaths []string) []string { var failedTestTemplatePaths []string - for proto, testCases := range protocolTests { + for proto, testCaseInfos := range protocolTests { if len(customTemplatePaths) == 0 { fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto)) } - keys := getMapKeys(testCases) - - for _, templatePath := range keys { - testCase := testCases[templatePath] - if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, templatePath) { + for _, testCaseInfo := range testCaseInfos { + if testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn() { + fmt.Printf("skipping test case %v. disabled on %v.\n", aurora.Blue(testCaseInfo.Path), runtime.GOOS) + continue + } + if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, testCaseInfo.Path) { var failedTemplatePath string var err error if proto == "interactsh" { - failedTemplatePath, err = executeWithRetry(testCase, templatePath, interactshRetryCount) + failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount) } else { - failedTemplatePath, err = execute(testCase, templatePath) + failedTemplatePath, err = execute(testCaseInfo.TestCase, testCaseInfo.Path) } if err != nil { failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath) @@ -169,12 +176,3 @@ func normalizeSplit(str string) []string { return r == ',' }) } - -func getMapKeys[T any](testcases map[string]T) []string { - keys := make([]string, 0, len(testcases)) - for k := range testcases { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} diff --git a/v2/cmd/integration-test/interactsh.go b/v2/cmd/integration-test/interactsh.go index 035f844d0..d163fef80 100644 --- a/v2/cmd/integration-test/interactsh.go +++ b/v2/cmd/integration-test/interactsh.go @@ -1,10 +1,10 @@ package main -import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +import osutils "github.com/projectdiscovery/utils/os" // All Interactsh related testcases -var interactshTestCases = map[string]testutils.TestCase{ - "http/interactsh.yaml": &httpInteractshRequest{}, - "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, - "http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{}, +var interactshTestCases = []TestCaseInfo{ + {Path: "http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, + {Path: "http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, + {Path: "http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, } diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 4f4c06056..6c0ccae61 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -12,13 +12,13 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var loaderTestcases = map[string]testutils.TestCase{ - "loader/template-list.yaml": &remoteTemplateList{}, - "loader/workflow-list.yaml": &remoteWorkflowList{}, - "loader/excluded-template.yaml": &excludedTemplate{}, - "loader/nonexistent-template-list.yaml": &nonExistentTemplateList{}, - "loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{}, - "loader/template-list-not-allowed.yaml": &remoteTemplateListNotAllowed{}, +var loaderTestcases = []TestCaseInfo{ + {Path: "loader/template-list.yaml", TestCase: &remoteTemplateList{}}, + {Path: "loader/workflow-list.yaml", TestCase: &remoteWorkflowList{}}, + {Path: "loader/excluded-template.yaml", TestCase: &excludedTemplate{}}, + {Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}}, + {Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}}, + {Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}}, } type remoteTemplateList struct{} diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index ff4d72a8c..6335fd003 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -4,15 +4,16 @@ import ( "net" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + osutils "github.com/projectdiscovery/utils/os" ) -var networkTestcases = map[string]testutils.TestCase{ - "network/basic.yaml": &networkBasic{}, - "network/hex.yaml": &networkBasic{}, - "network/multi-step.yaml": &networkMultiStep{}, - "network/self-contained.yaml": &networkRequestSelContained{}, - "network/variables.yaml": &networkVariables{}, - "network/same-address.yaml": &networkBasic{}, +var networkTestcases = []TestCaseInfo{ + {Path: "network/basic.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }}, + {Path: "network/hex.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }}, + {Path: "network/multi-step.yaml", TestCase: &networkMultiStep{}}, + {Path: "network/self-contained.yaml", TestCase: &networkRequestSelContained{}}, + {Path: "network/variables.yaml", TestCase: &networkVariables{}}, + {Path: "network/same-address.yaml", TestCase: &networkBasic{}}, } const defaultStaticPort = 5431 diff --git a/v2/cmd/integration-test/offline-http.go b/v2/cmd/integration-test/offline-http.go index b563cbc0b..8e442112c 100644 --- a/v2/cmd/integration-test/offline-http.go +++ b/v2/cmd/integration-test/offline-http.go @@ -6,10 +6,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var offlineHttpTestcases = map[string]testutils.TestCase{ - "offlinehttp/rfc-req-resp.yaml": &RfcRequestResponse{}, - "offlinehttp/offline-allowed-paths.yaml": &RequestResponseWithAllowedPaths{}, - "offlinehttp/offline-raw.yaml": &RawRequestResponse{}, +var offlineHttpTestcases = []TestCaseInfo{ + {Path: "offlinehttp/rfc-req-resp.yaml", TestCase: &RfcRequestResponse{}}, + {Path: "offlinehttp/offline-allowed-paths.yaml", TestCase: &RequestResponseWithAllowedPaths{}}, + {Path: "offlinehttp/offline-raw.yaml", TestCase: &RawRequestResponse{}}, } type RfcRequestResponse struct{} diff --git a/v2/cmd/integration-test/ssl.go b/v2/cmd/integration-test/ssl.go index 7f3374816..03b2e6bcb 100644 --- a/v2/cmd/integration-test/ssl.go +++ b/v2/cmd/integration-test/ssl.go @@ -7,12 +7,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var sslTestcases = map[string]testutils.TestCase{ - "ssl/basic.yaml": &sslBasic{}, - "ssl/basic-ztls.yaml": &sslBasicZtls{}, - "ssl/custom-cipher.yaml": &sslCustomCipher{}, - "ssl/custom-version.yaml": &sslCustomVersion{}, - "ssl/ssl-with-vars.yaml": &sslWithVars{}, +var sslTestcases = []TestCaseInfo{ + {Path: "ssl/basic.yaml", TestCase: &sslBasic{}}, + {Path: "ssl/basic-ztls.yaml", TestCase: &sslBasicZtls{}}, + {Path: "ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}}, + {Path: "ssl/custom-version.yaml", TestCase: &sslCustomVersion{}}, + {Path: "ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}}, } type sslBasic struct{} diff --git a/v2/cmd/integration-test/template-dir.go b/v2/cmd/integration-test/template-dir.go index 8b318ad89..fe629fff1 100644 --- a/v2/cmd/integration-test/template-dir.go +++ b/v2/cmd/integration-test/template-dir.go @@ -7,8 +7,8 @@ import ( errorutil "github.com/projectdiscovery/utils/errors" ) -var templatesDirTestCases = map[string]testutils.TestCase{ - "dns/cname-fingerprint.yaml": &templateDirWithTargetTest{}, +var templatesDirTestCases = []TestCaseInfo{ + {Path: "dns/cname-fingerprint.yaml", TestCase: &templateDirWithTargetTest{}}, } type templateDirWithTargetTest struct{} diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 9633558d8..97cab7aa1 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -12,15 +12,15 @@ func getTemplatePath() string { return config.DefaultConfig.TemplatesDirectory } -var templatesPathTestCases = map[string]testutils.TestCase{ +var templatesPathTestCases = []TestCaseInfo{ //template folder path issue - "http/get.yaml": &folderPathTemplateTest{}, + {Path: "http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd - "./dns/cname-fingerprint.yaml": &cwdTemplateTest{}, + {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, //relative path - "dns/cname-fingerprint.yaml": &relativePathTemplateTest{}, + {Path: "dns/cname-fingerprint.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path - fmt.Sprintf("%v/dns/cname-fingerprint.yaml", getTemplatePath()): &absolutePathTemplateTest{}, + {Path: fmt.Sprintf("%v/dns/cname-fingerprint.yaml", getTemplatePath()), TestCase: &absolutePathTemplateTest{}}, } type cwdTemplateTest struct{} diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index 73de9e481..9d0873acc 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -9,11 +9,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var websocketTestCases = map[string]testutils.TestCase{ - "websocket/basic.yaml": &websocketBasic{}, - "websocket/cswsh.yaml": &websocketCswsh{}, - "websocket/no-cswsh.yaml": &websocketNoCswsh{}, - "websocket/path.yaml": &websocketWithPath{}, +var websocketTestCases = []TestCaseInfo{ + {Path: "websocket/basic.yaml", TestCase: &websocketBasic{}}, + {Path: "websocket/cswsh.yaml", TestCase: &websocketCswsh{}}, + {Path: "websocket/no-cswsh.yaml", TestCase: &websocketNoCswsh{}}, + {Path: "websocket/path.yaml", TestCase: &websocketWithPath{}}, } type websocketBasic struct{} diff --git a/v2/cmd/integration-test/whois.go b/v2/cmd/integration-test/whois.go index edb534e43..5caa846b4 100644 --- a/v2/cmd/integration-test/whois.go +++ b/v2/cmd/integration-test/whois.go @@ -4,8 +4,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var whoisTestCases = map[string]testutils.TestCase{ - "whois/basic.yaml": &whoisBasic{}, +var whoisTestCases = []TestCaseInfo{ + {Path: "whois/basic.yaml", TestCase: &whoisBasic{}}, } type whoisBasic struct{} diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index fc22d8d8f..4cfb95b16 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -11,14 +11,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var workflowTestcases = map[string]testutils.TestCase{ - "workflow/basic.yaml": &workflowBasic{}, - "workflow/condition-matched.yaml": &workflowConditionMatched{}, - "workflow/condition-unmatched.yaml": &workflowConditionUnmatch{}, - "workflow/matcher-name.yaml": &workflowMatcherName{}, - "workflow/http-value-share-workflow.yaml": &workflowHttpKeyValueShare{}, - "workflow/dns-value-share-workflow.yaml": &workflowDnsKeyValueShare{}, - "workflow/shared-cookie.yaml": &workflowSharedCookies{}, +var workflowTestcases = []TestCaseInfo{ + {Path: "workflow/basic.yaml", TestCase: &workflowBasic{}}, + {Path: "workflow/condition-matched.yaml", TestCase: &workflowConditionMatched{}}, + {Path: "workflow/condition-unmatched.yaml", TestCase: &workflowConditionUnmatch{}}, + {Path: "workflow/matcher-name.yaml", TestCase: &workflowMatcherName{}}, + {Path: "workflow/http-value-share-workflow.yaml", TestCase: &workflowHttpKeyValueShare{}}, + {Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}}, + {Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}}, } type workflowBasic struct{} diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 67f15b2e4..301326c80 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -272,6 +272,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support (root user on Linux will disable sandbox)"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), + flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "use local installed Chrome browser instead of nuclei installed"), flagSet.BoolVarP(&options.ShowActions, "list-headless-action", "lha", false, "list available headless actions"), ) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index ebbe3e66d..914f72a8f 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -114,6 +114,10 @@ func validateOptions(options *types.Options) error { return errors.New("both verbose and silent mode specified") } + if (options.HeadlessOptionalArguments != nil || options.ShowBrowser || options.UseInstalledChrome) && !options.Headless { + return errors.New("headless mode (-headless) is required if -ho, -sb, -sc or -lha are set") + } + if options.FollowHostRedirects && options.FollowRedirects { return errors.New("both follow host redirects and follow redirects specified") } diff --git a/v2/internal/runner/options_test.go b/v2/internal/runner/options_test.go new file mode 100644 index 000000000..72a9adb38 --- /dev/null +++ b/v2/internal/runner/options_test.go @@ -0,0 +1,61 @@ +package runner + +import ( + "strings" + "testing" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/stretchr/testify/require" +) + +func TestParseHeadlessOptionalArguments(t *testing.T) { + tests := []struct { + name string + input string + want map[string]string + }{ + { + name: "single value", + input: "a=b", + want: map[string]string{"a": "b"}, + }, + { + name: "empty string", + input: "", + want: map[string]string{}, + }, + { + name: "empty key", + input: "=b", + want: map[string]string{}, + }, + { + name: "empty value", + input: "a=", + want: map[string]string{}, + }, + { + name: "double input", + input: "a=b,c=d", + want: map[string]string{"a": "b", "c": "d"}, + }, + { + name: "duplicated input", + input: "a=b,a=b", + want: map[string]string{"a": "b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strsl := goflags.StringSlice{} + for _, v := range strings.Split(tt.input, ",") { + //nolint + strsl.Set(v) + } + opt := types.Options{HeadlessOptionalArguments: strsl} + got := opt.ParseHeadlessOptionalArguments() + require.Equal(t, tt.want, got) + }) + } +} diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 85e7af7b0..de43211d5 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -8,6 +8,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/launcher/flags" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -75,6 +76,11 @@ func New(options *types.Options) (*Browser, error) { if types.ProxyURL != "" { chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } + + for k, v := range options.ParseHeadlessOptionalArguments() { + chromeLauncher.Set(flags.Flag(k), v) + } + launcherURL, err := chromeLauncher.Launch() if err != nil { return nil, err diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 48347a600..c3842bbb6 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -2,6 +2,7 @@ package types import ( "io" + "strings" "time" "github.com/projectdiscovery/goflags" @@ -197,6 +198,8 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool + // HeadlessOptionalArguments specifies optional arguments to pass to Chrome + HeadlessOptionalArguments goflags.StringSlice // NoTables disables pretty printing of cloud results in tables NoTables bool // DisableClustering disables clustering of templates @@ -441,3 +444,17 @@ func (options *Options) HasCloudOptions() bool { func (options *Options) ShouldUseHostError() bool { return options.MaxHostError > 0 && !options.NoHostErrors } + +func (options *Options) ParseHeadlessOptionalArguments() map[string]string { + optionalArguments := make(map[string]string) + for _, v := range options.HeadlessOptionalArguments { + if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { + key := strings.TrimSpace(argParts[0]) + value := strings.TrimSpace(argParts[1]) + if key != "" && value != "" { + optionalArguments[key] = value + } + } + } + return optionalArguments +} From 3894d466ada5d940f19cdfc82f58182f86dfadae Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:43:09 +0530 Subject: [PATCH 49/62] version update --- README.md | 25 +------------------------ v2/pkg/catalog/config/constants.go | 2 +- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index e60b5e202..7007930a0 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ FILTERING: -tags string[] templates to run based on tags (comma-separated, file) -etags, -exclude-tags string[] templates to exclude based on tags (comma-separated, file) -itags, -include-tags string[] tags to be executed even if they are excluded either by default or configuration - -id, -template-id string[] templates to run based on template ids (comma-separated, file) + -id, -template-id string[] templates to run based on template ids (comma-separated, file, allow-wildcard) -eid, -exclude-id string[] templates to exclude based on template ids (comma-separated, file) -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration -et, -exclude-templates string[] template or template directory to exclude (comma-separated, file) @@ -284,29 +284,6 @@ STATISTICS: -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) -m, -metrics expose nuclei metrics on a port -mp, -metrics-port int port to expose nuclei metrics on (default 9092) - -CLOUD: - -cloud run scan on nuclei cloud - -ads, -add-datasource string add specified data source (s3,github) - -atr, -add-target string add target(s) to cloud - -atm, -add-template string add template(s) to cloud - -lsn, -list-scan list previous cloud scans - -lso, -list-output string list scan output by scan id - -ltr, -list-target list cloud target by id - -ltm, -list-template list cloud template by id - -lds, -list-datasource list cloud datasource by id - -lrs, -list-reportsource list reporting sources - -dsn, -delete-scan string delete cloud scan by id - -dtr, -delete-target string delete target(s) from cloud - -dtm, -delete-template string delete template(s) from cloud - -dds, -delete-datasource string delete specified data source - -drs, -disable-reportsource string disable specified reporting source - -ers, -enable-reportsource string enable specified reporting source - -gtr, -get-target string get target content by id - -gtm, -get-template string get template content by id - -nos, -no-store disable scan/output storage on cloud - -no-tables do not display pretty-printed tables - -limit int limit the number of output to display (default 100) ``` ### Running Nuclei diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index fed0319c7..db104ec49 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v2.9.9` + Version = `v2.9.10` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGithubTemplatesDirName = "github" From d83c2dd8dd3fddba722c7e14b0fb2963f3c0240f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 28 Jul 2023 19:20:52 +0200 Subject: [PATCH 50/62] converting dsl tests to new struct (#4003) --- v2/cmd/integration-test/dsl.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/cmd/integration-test/dsl.go b/v2/cmd/integration-test/dsl.go index 7d60e8f5f..76c6dc0fc 100644 --- a/v2/cmd/integration-test/dsl.go +++ b/v2/cmd/integration-test/dsl.go @@ -9,9 +9,9 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) -var dslTestcases = map[string]testutils.TestCase{ - "dsl/hide-version-warning.yaml": &dslVersionWarning{}, - "dsl/show-version-warning.yaml": &dslShowVersionWarning{}, +var dslTestcases = []TestCaseInfo{ + {Path: "dsl/hide-version-warning.yaml", TestCase: &dslVersionWarning{}}, + {Path: "dsl/show-version-warning.yaml", TestCase: &dslShowVersionWarning{}}, } type dslVersionWarning struct{} From b2fa789622241312fc1e0ed7fd8285299089fa37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:23:49 +0000 Subject: [PATCH 51/62] chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.104 to 0.0.106. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.104...v0.0.106) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index d2cb85319..d88324c51 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.0 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 github.com/projectdiscovery/utils v0.0.44 - github.com/projectdiscovery/wappalyzergo v0.0.104 + github.com/projectdiscovery/wappalyzergo v0.0.106 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/v2/go.sum b/v2/go.sum index d09bc9227..dc51c0ff9 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -437,8 +437,8 @@ github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6L github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= github.com/projectdiscovery/utils v0.0.44 h1:F/LNgBw53RNM/3mRZ1ji+prM1yDnehDRBf13TPk3WBM= github.com/projectdiscovery/utils v0.0.44/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= -github.com/projectdiscovery/wappalyzergo v0.0.104 h1:hdda6WxAzXVpLBbJW1sLqrwOXHn0prP9IYFY7dfCMjE= -github.com/projectdiscovery/wappalyzergo v0.0.104/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/projectdiscovery/wappalyzergo v0.0.106 h1:Mb7EOjb2qWCHv28HOJXdmGYDHg/Eh0XIRboxuhzUyo0= +github.com/projectdiscovery/wappalyzergo v0.0.106/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= From b02a8ce3abcd9c646a6477faa6231e0223fd25e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:30:39 +0000 Subject: [PATCH 52/62] chore(deps): bump github.com/projectdiscovery/retryablehttp-go in /v2 Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.18 to 1.0.20. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.18...v1.0.20) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 6 +++--- v2/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index d2cb85319..484281469 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,12 +21,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.33 + github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.16 - github.com/projectdiscovery/retryabledns v1.0.30 - github.com/projectdiscovery/retryablehttp-go v1.0.18 + github.com/projectdiscovery/retryabledns v1.0.31 + github.com/projectdiscovery/retryablehttp-go v1.0.20 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/v2/go.sum b/v2/go.sum index d09bc9227..515b4b6c3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -398,8 +398,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= -github.com/projectdiscovery/fastdialer v0.0.33 h1:FKXZjJme5nhgnnRL0Y4KjZ5YXKg03dX5ZJYFJV3LOwU= -github.com/projectdiscovery/fastdialer v0.0.33/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo= +github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12 h1:w9EHSIuPyksPv0pWmXBwUmMlR2arrGHVE/s9LJMNP5Y= +github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -424,10 +424,10 @@ github.com/projectdiscovery/rawhttp v0.1.16 h1:2wggRuVUty2Sypy1MKWM/NNmfbC97WkG6 github.com/projectdiscovery/rawhttp v0.1.16/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0= -github.com/projectdiscovery/retryabledns v1.0.30/go.mod h1:+Aqc0TjKGcTtP0HtXE8o1GzrjAHhSno6hSF+L63TBtI= -github.com/projectdiscovery/retryablehttp-go v1.0.18 h1:3IUxyIOOUVSGEBm4pV0cQSk1i/DausZdHePdGDip0Lg= -github.com/projectdiscovery/retryablehttp-go v1.0.18/go.mod h1:oE3dmYWMadFWzaIfG1IqINsYAzUWYUtdI4PJ2xo7cXg= +github.com/projectdiscovery/retryabledns v1.0.31 h1:hvFKyHYQEKD4JYK6NngT00iexX/TJRFulnR+oVMxHBE= +github.com/projectdiscovery/retryabledns v1.0.31/go.mod h1:v+Ft2rq8cGPv2+k3/7JlOx32Icd2xJ/ZTfRCWYmmmw0= +github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE= +github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= From b3f288b395ac28a3cd85286c090b0779afbc6b2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:31:28 +0000 Subject: [PATCH 53/62] chore(deps): bump github.com/projectdiscovery/goflags in /v2 Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.10 to 0.1.12. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.10...v0.1.12) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index d2cb85319..7001c563a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -68,7 +68,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.10 + github.com/projectdiscovery/goflags v0.1.12 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/httpx v1.3.3 github.com/projectdiscovery/mapcidr v1.1.2 diff --git a/v2/go.sum b/v2/go.sum index d09bc9227..b81686ba5 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -404,8 +404,8 @@ github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOi github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.10 h1:Gompf8JDy8y+5c4eWlc70KKtPuDH/hqFB3tMeHcMiKk= -github.com/projectdiscovery/goflags v0.1.10/go.mod h1:MHEkqm3XgxBf5fK4gr3IXsj6VeLTq4qJYGC/4JRYQ74= +github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0= +github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= From 93a87c21487a38662febd43ce243d4ee641a9625 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:49:52 +0000 Subject: [PATCH 54/62] chore(deps): bump github.com/projectdiscovery/fastdialer in /v2 Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.33 to 0.0.35. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.33...v0.0.35) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index d819916aa..cf831a425 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,7 +21,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12 + github.com/projectdiscovery/fastdialer v0.0.35 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.16 diff --git a/v2/go.sum b/v2/go.sum index 90f088e01..d75d83741 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -398,8 +398,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= -github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12 h1:w9EHSIuPyksPv0pWmXBwUmMlR2arrGHVE/s9LJMNP5Y= -github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= +github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= +github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= From b3bf71e05ebc79d12cedd623386f5511eb493178 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:55:53 +0000 Subject: [PATCH 55/62] chore(deps): bump github.com/projectdiscovery/httpx in /v2 Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.3.3 to 1.3.4. - [Release notes](https://github.com/projectdiscovery/httpx/releases) - [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/httpx/compare/v1.3.3...v1.3.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/httpx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 10 +++++----- v2/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5cfaa6f2b..0ea26b1b2 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/go-playground/validator/v10 v10.14.1 - github.com/go-rod/rod v0.113.4 + github.com/go-rod/rod v0.114.0 github.com/gobwas/ws v1.2.1 github.com/google/go-github v17.0.0+incompatible github.com/itchyny/gojq v0.12.13 @@ -21,10 +21,10 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12 + github.com/projectdiscovery/fastdialer v0.0.35 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 - github.com/projectdiscovery/rawhttp v0.1.16 + github.com/projectdiscovery/rawhttp v0.1.18 github.com/projectdiscovery/retryabledns v1.0.31 github.com/projectdiscovery/retryablehttp-go v1.0.20 github.com/projectdiscovery/yamldoc-go v1.0.4 @@ -70,12 +70,12 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.12 github.com/projectdiscovery/gologger v1.1.11 - github.com/projectdiscovery/httpx v1.3.3 + github.com/projectdiscovery/httpx v1.3.4 github.com/projectdiscovery/mapcidr v1.1.2 github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.0 + github.com/projectdiscovery/tlsx v1.1.1 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 github.com/projectdiscovery/utils v0.0.44 github.com/projectdiscovery/wappalyzergo v0.0.106 diff --git a/v2/go.sum b/v2/go.sum index 0fabb594b..770dac75f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -188,8 +188,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-rod/rod v0.113.4 h1:O5a/VTl1h6a08ecXkOF9oVJBi8fjBnF0FtyMSgd3KaY= -github.com/go-rod/rod v0.113.4/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= +github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -398,8 +398,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= -github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12 h1:w9EHSIuPyksPv0pWmXBwUmMlR2arrGHVE/s9LJMNP5Y= -github.com/projectdiscovery/fastdialer v0.0.35-0.20230712222521-f5386e7cfc12/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= +github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= +github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -410,8 +410,8 @@ github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqV github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= -github.com/projectdiscovery/httpx v1.3.3 h1:JINmFbxKcP6xtFtmukaG+Js+ya5I/zY6RV9hkO2j4oI= -github.com/projectdiscovery/httpx v1.3.3/go.mod h1:5fvZBoI39IXwT62X9Q31aXpVWRsRbedCC0XZF4uT/yE= +github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= +github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= github.com/projectdiscovery/interactsh v1.1.4/go.mod h1:rM8IEm6AAm68fWWExzBHjhBWfRhFYzR9gY5emOFiZCY= github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4tQsLPCrsA= @@ -420,8 +420,8 @@ github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzV github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8= github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I= -github.com/projectdiscovery/rawhttp v0.1.16 h1:2wggRuVUty2Sypy1MKWM/NNmfbC97WkG6YtmoxzCOVc= -github.com/projectdiscovery/rawhttp v0.1.16/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY= +github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt31kQO6P+41s= +github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.31 h1:hvFKyHYQEKD4JYK6NngT00iexX/TJRFulnR+oVMxHBE= @@ -431,8 +431,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpWBJBG6I= -github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= +github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= +github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= github.com/projectdiscovery/utils v0.0.44 h1:F/LNgBw53RNM/3mRZ1ji+prM1yDnehDRBf13TPk3WBM= From 5ab39fca25c3dff74a464f7de009830ec8e97ff5 Mon Sep 17 00:00:00 2001 From: lu4nx Date: Sun, 30 Jul 2023 15:32:50 +0800 Subject: [PATCH 56/62] Fix 'reference' info nil pointer dereference. (#4005) (#4006) * Fix 'reference' info nil pointer dereference. (#4005) * test path update * test fixes --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- v2/cmd/integration-test/template-path.go | 4 ++-- v2/pkg/reporting/format/format_utils.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 97cab7aa1..13c79d508 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -18,9 +18,9 @@ var templatesPathTestCases = []TestCaseInfo{ //cwd {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, //relative path - {Path: "dns/cname-fingerprint.yaml", TestCase: &relativePathTemplateTest{}}, + {Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path - {Path: fmt.Sprintf("%v/dns/cname-fingerprint.yaml", getTemplatePath()), TestCase: &absolutePathTemplateTest{}}, + {Path: fmt.Sprintf("%v/dns/dns-saas-service-detection.yaml", getTemplatePath()), TestCase: &absolutePathTemplateTest{}}, } type cwdTemplateTest struct{} diff --git a/v2/pkg/reporting/format/format_utils.go b/v2/pkg/reporting/format/format_utils.go index c02f00622..c9af7b119 100644 --- a/v2/pkg/reporting/format/format_utils.go +++ b/v2/pkg/reporting/format/format_utils.go @@ -113,7 +113,7 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte } reference := event.Info.Reference - if !reference.IsEmpty() { + if reference != nil && !reference.IsEmpty() { builder.WriteString("\nReferences: \n") referenceSlice := reference.ToSlice() From aeefb581b0f81ff75faa00ec6ac9ba8ed174a5e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 05:38:31 +0000 Subject: [PATCH 57/62] chore(deps): bump github.com/projectdiscovery/retryabledns in /v2 Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.31 to 1.0.32. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.31...v1.0.32) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0ea26b1b2..28ac597eb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -25,7 +25,7 @@ require ( github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.18 - github.com/projectdiscovery/retryabledns v1.0.31 + github.com/projectdiscovery/retryabledns v1.0.32 github.com/projectdiscovery/retryablehttp-go v1.0.20 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/v2/go.sum b/v2/go.sum index 770dac75f..82c16c20a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -424,8 +424,8 @@ github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt3 github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.31 h1:hvFKyHYQEKD4JYK6NngT00iexX/TJRFulnR+oVMxHBE= -github.com/projectdiscovery/retryabledns v1.0.31/go.mod h1:v+Ft2rq8cGPv2+k3/7JlOx32Icd2xJ/ZTfRCWYmmmw0= +github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo= +github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c= github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE= github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= From 6490fb8d97d6a2895eb6c99745ed7f05e981c380 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 05:38:44 +0000 Subject: [PATCH 58/62] chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.106 to 0.0.107. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.106...v0.0.107) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0ea26b1b2..8b7cb6f33 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.1 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 github.com/projectdiscovery/utils v0.0.44 - github.com/projectdiscovery/wappalyzergo v0.0.106 + github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/v2/go.sum b/v2/go.sum index 770dac75f..962f658ae 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -437,8 +437,8 @@ github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6L github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= github.com/projectdiscovery/utils v0.0.44 h1:F/LNgBw53RNM/3mRZ1ji+prM1yDnehDRBf13TPk3WBM= github.com/projectdiscovery/utils v0.0.44/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= -github.com/projectdiscovery/wappalyzergo v0.0.106 h1:Mb7EOjb2qWCHv28HOJXdmGYDHg/Eh0XIRboxuhzUyo0= -github.com/projectdiscovery/wappalyzergo v0.0.106/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= +github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= From 63a449665b75646496e26b4c6417ebdea3fc0bb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 06:02:15 +0000 Subject: [PATCH 59/62] chore(deps): bump github.com/projectdiscovery/utils in /v2 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.44 to 0.0.45. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.44...v0.0.45) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 8b7cb6f33..fadf4a523 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -77,7 +77,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.1 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.44 + github.com/projectdiscovery/utils v0.0.45 github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 diff --git a/v2/go.sum b/v2/go.sum index 962f658ae..8c8b53b65 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -435,8 +435,8 @@ github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJK github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.44 h1:F/LNgBw53RNM/3mRZ1ji+prM1yDnehDRBf13TPk3WBM= -github.com/projectdiscovery/utils v0.0.44/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/utils v0.0.45 h1:i0xwdpaGrTIw8hVtutjpVXvPCMcKKwb9VbqWxlwL08M= +github.com/projectdiscovery/utils v0.0.45/go.mod h1:kJu+OqAWsoilLto06ajSp4U3gWuz51GEVLL/PogJXPk= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From 4c1c5301b990bbe222e78857a6b8f89a447a3042 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:33:43 -0400 Subject: [PATCH 60/62] Spelling (#4008) * spelling: addresses Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: asynchronous Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: basic Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: brute force Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: constant Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: disables Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: engine Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: every time Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: execution Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: false positives Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: from Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: further Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: github Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: gitlab Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: highlight Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: hygiene Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: ignore Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: input Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: item Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: itself Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: latestxxx Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: navigation Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: negative Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: nonexistent Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: occurred Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: override Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: overrides Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: payload Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: performed Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: respective Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: retrieve Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: scanlist Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: separated Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: separator Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: severity Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: source Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: strategy Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: string Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: templates Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: terminal Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: timeout Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: trailing slash Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: trailing Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: websocket Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- .github/workflows/build-test.yml | 2 +- .github/workflows/dockerhub-push.yml | 2 +- DESIGN.md | 4 +-- SYNTAX-REFERENCE.md | 2 +- integration_tests/debug.sh | 2 +- integration_tests/workflow/shared-cookie.yaml | 2 +- nuclei-jsonschema.json | 2 +- v2/cmd/integration-test/http.go | 2 +- v2/cmd/nuclei/main.go | 6 ++-- v2/cmd/tmc/main.go | 2 +- v2/gh_retry.sh | 4 +-- v2/internal/installer/template.go | 4 +-- v2/internal/installer/template_test.go | 2 +- v2/internal/installer/util.go | 4 +-- v2/internal/runner/cloud.go | 4 +-- v2/internal/runner/options.go | 18 +++++----- v2/internal/runner/proxy.go | 2 +- v2/internal/runner/templates.go | 4 +-- v2/pkg/catalog/aws/catalog.go | 2 +- v2/pkg/catalog/aws/catalog_test.go | 2 +- v2/pkg/catalog/config/constants.go | 2 +- v2/pkg/catalog/config/nucleiconfig.go | 12 +++---- v2/pkg/catalog/disk/find.go | 4 +-- v2/pkg/catalog/disk/path.go | 2 +- v2/pkg/catalog/loader/loader.go | 4 +-- v2/pkg/core/execute_options.go | 2 +- v2/pkg/core/inputs/hybrid/hmap.go | 2 +- v2/pkg/external/customtemplates/github.go | 34 +++++++++---------- .../external/customtemplates/github_test.go | 4 +-- v2/pkg/external/customtemplates/gitlab.go | 4 +-- .../customtemplates/templates_provider.go | 4 +-- v2/pkg/input/input.go | 2 +- v2/pkg/parsers/parser_test.go | 4 +-- v2/pkg/protocols/common/automaticscan/doc.go | 2 +- .../common/helpers/deserialization/java.go | 8 ++--- .../protocols/common/interactsh/interactsh.go | 2 +- v2/pkg/protocols/common/interactsh/options.go | 2 +- .../protocols/common/protocolstate/state.go | 4 +-- .../common/randomip/randomip_test.go | 2 +- v2/pkg/protocols/common/uncover/uncover.go | 2 +- v2/pkg/protocols/common/variables/doc.go | 2 +- v2/pkg/protocols/headless/request.go | 2 +- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/http/raw/raw.go | 2 +- v2/pkg/protocols/http/request_annotations.go | 14 ++++---- v2/pkg/protocols/network/request.go | 6 ++-- v2/pkg/protocols/utils/http/requtils.go | 2 +- v2/pkg/protocols/utils/variables_test.go | 2 +- v2/pkg/protocols/websocket/websocket.go | 2 +- v2/pkg/templates/cluster.go | 2 +- v2/pkg/templates/templates.go | 2 +- v2/pkg/templates/templates_doc.go | 4 +-- v2/pkg/testutils/testutils.go | 4 +-- v2/pkg/types/types.go | 10 +++--- 54 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 599940d7a..8173b810d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -25,7 +25,7 @@ jobs: - name: Check out code uses: actions/checkout@v3 - - name: Go Mod hygine + - name: Go Mod hygiene run: | go clean -modcache go mod tidy diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 428add3e4..da0887e79 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -14,7 +14,7 @@ jobs: - name: Git Checkout uses: actions/checkout@v3 - - name: Get Github tag + - name: Get GitHub tag id: meta run: | curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT diff --git a/DESIGN.md b/DESIGN.md index 969dc84ed..b02067f24 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -145,7 +145,7 @@ type Exporter interface { } ``` -Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `Gitlab` and `Jira`. +Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `GitLab` and `Jira`. Each exporter and trackers implement their own configuration in YAML format and are very modular in nature, so adding new ones is easy. @@ -484,7 +484,7 @@ $ go tool pprof -http=:8081 mem.pprof - [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter - [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/es) - Elasticsearch Result Exporter - [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results -- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter +- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - GitLab Issue Tracker Exporter - [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter - [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - GitHub Issue Tracker Exporter - [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index a0406daa6..9df9152c4 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -358,7 +358,7 @@ Variables contains any variables for the current request.
-Constants contains any scalar costant for the current template +Constants contains any scalar constant for the current template
diff --git a/integration_tests/debug.sh b/integration_tests/debug.sh index c3a968a82..57bfd0382 100755 --- a/integration_tests/debug.sh +++ b/integration_tests/debug.sh @@ -8,7 +8,7 @@ if [ $1 = "-h" ]; then printf " \$ ./debug.sh http self\n\n" printf "3. To run all integration tests of 'x' protocol that contains 'y' in template name and pass extra args to nuclei:\n" printf " \$ ./debug.sh http self -svd -debug-req\n\n" - printf "nuclei binary is created everytime script is run but integration-test binary is not" + printf "nuclei binary is created every time script is run but integration-test binary is not" exit 0 fi diff --git a/integration_tests/workflow/shared-cookie.yaml b/integration_tests/workflow/shared-cookie.yaml index f2d68be2c..802be7818 100644 --- a/integration_tests/workflow/shared-cookie.yaml +++ b/integration_tests/workflow/shared-cookie.yaml @@ -11,5 +11,5 @@ workflows: - template: workflow/http-2.yaml # store cookie in native browser context - template: workflow/headless-1.yaml - # retrive 2 standard library cookies + headless cookie + # retrieve 2 standard library cookies + headless cookie - template: workflow/http-3.yaml \ No newline at end of file diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index bb6506ad8..579db7082 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -1311,7 +1311,7 @@ } }, "type": "object", - "title": "payloads for the webosocket request", + "title": "payloads for the websocket request", "description": "Payloads contains any payloads for the current request" } }, diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 75e80e186..84e4c8d9b 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -172,7 +172,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error { if err != nil { return err } - // polling is asyncronous, so the interactions may be retrieved after the first request + // polling is asynchronous, so the interactions may be retrieved after the first request return expectResultsCount(results, 1) } diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 301326c80..fa83b520e 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -422,11 +422,11 @@ func printTemplateVersion() { if fileutil.FolderExists(cfg.CustomS3TemplatesDirectory) { gologger.Info().Msgf("Custom S3 templates location: %s\n", cfg.CustomS3TemplatesDirectory) } - if fileutil.FolderExists(cfg.CustomGithubTemplatesDirectory) { - gologger.Info().Msgf("Custom Github templates location: %s ", cfg.CustomGithubTemplatesDirectory) + if fileutil.FolderExists(cfg.CustomGitHubTemplatesDirectory) { + gologger.Info().Msgf("Custom GitHub templates location: %s ", cfg.CustomGitHubTemplatesDirectory) } if fileutil.FolderExists(cfg.CustomGitLabTemplatesDirectory) { - gologger.Info().Msgf("Custom Gitlab templates location: %s ", cfg.CustomGitLabTemplatesDirectory) + gologger.Info().Msgf("Custom GitLab templates location: %s ", cfg.CustomGitLabTemplatesDirectory) } if fileutil.FolderExists(cfg.CustomAzureTemplatesDirectory) { gologger.Info().Msgf("Custom Azure templates location: %s ", cfg.CustomAzureTemplatesDirectory) diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go index c334372da..850ef5529 100644 --- a/v2/cmd/tmc/main.go +++ b/v2/cmd/tmc/main.go @@ -90,7 +90,7 @@ type options struct { func main() { opts := options{} flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`TemplateMan CLI is baisc utility built on the TemplateMan API to standardize nuclei templates.`) + flagSet.SetDescription(`TemplateMan CLI is basic utility built on the TemplateMan API to standardize nuclei templates.`) flagSet.CreateGroup("Input", "input", flagSet.StringVarP(&opts.input, "input", "i", "", "Templates to annotate"), diff --git a/v2/gh_retry.sh b/v2/gh_retry.sh index b8516ec92..bf212e129 100755 --- a/v2/gh_retry.sh +++ b/v2/gh_retry.sh @@ -17,7 +17,7 @@ LIMIT=30 BEFORE="30 mins ago" WORKFLOW="Build Test" -# You can add multiple patterns seperated by | +# You can add multiple patterns separated by | GREP_ERROR_PATTERN='Test "http/interactsh.yaml" failed' #Set fonts for Help. @@ -81,7 +81,7 @@ function retry_failed_jobs() { select ( .conclusion=="failure" ) | select ( .updatedAt > $date) ' --arg date "$date" --arg branch "$BRANCH" --arg workflow "$WORKFLOW" | jq .databaseId) - # convert line seperated by space to array + # convert line separated by space to array eval "arr=($workflowIds)" if [[ -z $arr ]] diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go index b9eda1d0e..22caa5d7f 100644 --- a/v2/internal/installer/template.go +++ b/v2/internal/installer/template.go @@ -186,7 +186,7 @@ func (t *TemplateManager) summarizeChanges(old, new map[string]string) *template // getAbsoluteFilePath returns an absolute path where a file should be written based on given uri(i.e., files in zip) // if a returned path is empty, it means that file should not be written and skipped func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.FileInfo) string { - // overwrite .nuclei-ignore everytime nuclei-templates are downloaded + // overwrite .nuclei-ignore every time nuclei-templates are downloaded if f.Name() == config.NucleiIgnoreFileName { return config.DefaultConfig.GetIgnoreFilePath() } @@ -206,7 +206,7 @@ func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.File gologger.Warning().Msgf("failed to get directory name from uri: %s", uri) return filepath.Join(templateDir, uri) } - // seperator is also included in rootDir + // separator is also included in rootDir rootDirectory := uri[:index+1] relPath := strings.TrimPrefix(uri, rootDirectory) diff --git a/v2/internal/installer/template_test.go b/v2/internal/installer/template_test.go index e6b0345e2..95f44f856 100644 --- a/v2/internal/installer/template_test.go +++ b/v2/internal/installer/template_test.go @@ -53,7 +53,7 @@ func TestTemplateInstallation(t *testing.T) { // we should have at least 1000 templates require.Greater(t, counter, 1000) - // everytime we install templates, it should override the ignore file with latest one + // every time we install templates, it should override the ignore file with latest one require.FileExists(t, config.DefaultConfig.GetIgnoreFilePath()) t.Logf("Installed %d templates", counter) } diff --git a/v2/internal/installer/util.go b/v2/internal/installer/util.go index c9b1b7c4b..024202a34 100644 --- a/v2/internal/installer/util.go +++ b/v2/internal/installer/util.go @@ -36,7 +36,7 @@ func GetNewTemplatesInVersions(versions ...string) []string { continue } - arr, err := getNewAdditionsFileFromGithub(v) + arr, err := getNewAdditionsFileFromGitHub(v) if err != nil { gologger.Error().Msgf("failed to fetch new additions for %v got: %v", v, err) continue @@ -46,7 +46,7 @@ func GetNewTemplatesInVersions(versions ...string) []string { return allTemplates } -func getNewAdditionsFileFromGithub(version string) ([]string, error) { +func getNewAdditionsFileFromGitHub(version string) ([]string, error) { resp, err := retryableHttpClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version)) if err != nil { return nil, err diff --git a/v2/internal/runner/cloud.go b/v2/internal/runner/cloud.go index 707652227..4d173bf28 100644 --- a/v2/internal/runner/cloud.go +++ b/v2/internal/runner/cloud.go @@ -375,8 +375,8 @@ func (r *Runner) addCloudDataSource(source string) error { return err } case "github": - for _, repo := range r.options.GithubTemplateRepo { - if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil { + for _, repo := range r.options.GitHubTemplateRepo { + if _, err := r.processDataSourceItem(repo, r.options.GitHubToken, "github"); err != nil { return err } } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 914f72a8f..f943dfef4 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -80,9 +80,9 @@ func ParseOptions(options *types.Options) { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } - // Set Github token in env variable. runner.getGHClientWithToken() reads token from env - if options.GithubToken != "" && os.Getenv("GITHUB_TOKEN") != options.GithubToken { - os.Setenv("GITHUB_TOKEN", options.GithubToken) + // Set GitHub token in env variable. runner.getGHClientWithToken() reads token from env + if options.GitHubToken != "" && os.Getenv("GITHUB_TOKEN") != options.GitHubToken { + os.Setenv("GITHUB_TOKEN", options.GitHubToken) } if options.UncoverQuery != nil { @@ -203,7 +203,7 @@ func validateCloudOptions(options *types.Options) error { case "s3": missing = validateMissingS3Options(options) case "github": - missing = validateMissingGithubOptions(options) + missing = validateMissingGitHubOptions(options) case "gitlab": missing = validateMissingGitLabOptions(options) case "azure": @@ -253,12 +253,12 @@ func validateMissingAzureOptions(options *types.Options) []string { return missing } -func validateMissingGithubOptions(options *types.Options) []string { +func validateMissingGitHubOptions(options *types.Options) []string { var missing []string - if options.GithubToken == "" { + if options.GitHubToken == "" { missing = append(missing, "GITHUB_TOKEN") } - if len(options.GithubTemplateRepo) == 0 { + if len(options.GitHubTemplateRepo) == 0 { missing = append(missing, "GITHUB_TEMPLATE_REPO") } return missing @@ -360,10 +360,10 @@ func readEnvInputVars(options *types.Options) { } options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_API") - options.GithubToken = os.Getenv("GITHUB_TOKEN") + options.GitHubToken = os.Getenv("GITHUB_TOKEN") repolist := os.Getenv("GITHUB_TEMPLATE_REPO") if repolist != "" { - options.GithubTemplateRepo = append(options.GithubTemplateRepo, stringsutil.SplitAny(repolist, ",")...) + options.GitHubTemplateRepo = append(options.GitHubTemplateRepo, stringsutil.SplitAny(repolist, ",")...) } // GitLab options for downloading templates from a repository diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index 5bf464b7a..a4323eeba 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -14,7 +14,7 @@ import ( proxyutils "github.com/projectdiscovery/utils/proxy" ) -// loadProxyServers load list of proxy servers from file or comma seperated +// loadProxyServers load list of proxy servers from file or comma separated func loadProxyServers(options *types.Options) error { if len(options.Proxy) == 0 { return nil diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index de667f745..559448732 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -57,7 +57,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { path = aurora.Cyan(tpl.Path).String() tplBody, err = r.highlightTemplate(&tplBody) if err != nil { - gologger.Error().Msgf("Could not hihglight the template %s: %s", tpl.Path, err) + gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err) continue } @@ -74,7 +74,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { func (r *Runner) highlightTemplate(body *[]byte) ([]byte, error) { var buf bytes.Buffer - // YAML lexer, true color terminar formatter and monokai style + // YAML lexer, true color terminal formatter and monokai style err := quick.Highlight(&buf, string(*body), "yaml", "terminal16m", "monokai") if err != nil { return nil, err diff --git a/v2/pkg/catalog/aws/catalog.go b/v2/pkg/catalog/aws/catalog.go index 1303a4f49..5dfa86a56 100644 --- a/v2/pkg/catalog/aws/catalog.go +++ b/v2/pkg/catalog/aws/catalog.go @@ -129,7 +129,7 @@ func (c Catalog) ResolvePath(templateName, second string) (string, error) { // if c second path is given, it's c folder and we join the two and check against keys if second != "" { - // Note: Do not replace `path` with `filepath` since filepath is aware of Os path seperator + // Note: Do not replace `path` with `filepath` since filepath is aware of Os path separator // and we only see `/` in s3 paths changing it to filepath cause build fail and other errors target := path.Join(path.Dir(second), templateName) for _, key := range keys { diff --git a/v2/pkg/catalog/aws/catalog_test.go b/v2/pkg/catalog/aws/catalog_test.go index fa19a1423..57dac391a 100644 --- a/v2/pkg/catalog/aws/catalog_test.go +++ b/v2/pkg/catalog/aws/catalog_test.go @@ -141,7 +141,7 @@ func TestCatalog_OpenFile(t *testing.T) { false, }, { - "non-existent key", + "nonexistent key", "something/that-doesnt-exist.yaml", true, }, diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index db104ec49..48553045e 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -20,7 +20,7 @@ const ( Version = `v2.9.10` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" - CustomGithubTemplatesDirName = "github" + CustomGitHubTemplatesDirName = "github" CustomAzureTemplatesDirName = "azure" CustomGitLabTemplatesDirName = "gitlab" ) diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index baf7acb43..7dafbb8dc 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -23,16 +23,16 @@ type Config struct { TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` // customtemplates exists in templates directory with the name of custom-templates provider - // below custom paths are absolute paths to respecitive custom-templates directories + // below custom paths are absolute paths to respective custom-templates directories CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` - CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` + CustomGitHubTemplatesDirectory string `json:"custom-github-templates-directory"` CustomGitLabTemplatesDirectory string `json:"custom-gitlab-templates-directory"` CustomAzureTemplatesDirectory string `json:"custom-azure-templates-directory"` TemplateVersion string `json:"nuclei-templates-version,omitempty"` NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` - // Latestxxx are not meant to be used directly and is used as + // LatestXXX are not meant to be used directly and is used as // local cache of nuclei version check endpoint // these fields are only update during nuclei version check // TODO: move these fields to a separate unexported struct as they are not meant to be used directly @@ -83,7 +83,7 @@ func (c *Config) NeedsTemplateUpdate() bool { return !c.disableUpdates && (c.TemplateVersion == "" || IsOutdatedVersion(c.TemplateVersion, c.LatestNucleiTemplatesVersion) || !fileutil.FolderExists(c.TemplatesDirectory)) } -// NeedsIngoreFileUpdate returns true if Ignore file hash is different (aka ignore file is outdated) +// NeedsIgnoreFileUpdate returns true if Ignore file hash is different (aka ignore file is outdated) func (c *Config) NeedsIgnoreFileUpdate() bool { return c.NucleiIgnoreHash == "" || c.NucleiIgnoreHash != c.LatestNucleiIgnoreHash } @@ -111,7 +111,7 @@ func (c *Config) GetConfigDir() string { // GetAllCustomTemplateDirs returns all custom template directories func (c *Config) GetAllCustomTemplateDirs() []string { - return []string{c.CustomS3TemplatesDirectory, c.CustomGithubTemplatesDirectory, c.CustomGitLabTemplatesDirectory, c.CustomAzureTemplatesDirectory} + return []string{c.CustomS3TemplatesDirectory, c.CustomGitHubTemplatesDirectory, c.CustomGitLabTemplatesDirectory, c.CustomAzureTemplatesDirectory} } // GetReportingConfigFilePath returns the nuclei reporting config file path @@ -188,7 +188,7 @@ func (c *Config) SetTemplatesDir(dirPath string) { } c.TemplatesDirectory = dirPath // Update the custom templates directory - c.CustomGithubTemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) + c.CustomGitHubTemplatesDirectory = filepath.Join(dirPath, CustomGitHubTemplatesDirName) c.CustomS3TemplatesDirectory = filepath.Join(dirPath, CustomS3TemplatesDirName) c.CustomGitLabTemplatesDirectory = filepath.Join(dirPath, CustomGitLabTemplatesDirName) c.CustomAzureTemplatesDirectory = filepath.Join(dirPath, CustomAzureTemplatesDirName) diff --git a/v2/pkg/catalog/disk/find.go b/v2/pkg/catalog/disk/find.go index 77ff0cb24..98b8cf5be 100644 --- a/v2/pkg/catalog/disk/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -48,7 +48,7 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri } } } - // purge all falsepositivies + // purge all false positives filteredTemplates := []string{} for _, v := range allTemplates { // TODO: this is a temporary fix to avoid treating these files as templates @@ -129,7 +129,7 @@ func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) { // findGlobPathMatches returns the matched files from a glob path func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { - // to support globbing on old paths we use bruteforce to find matches with exit on first match + // to support globbing on old paths we use brute force to find matches with exit on first match // trim templateDir if any relPath := strings.TrimPrefix(absPath, c.templatesDirectory) // trim leading slash if any diff --git a/v2/pkg/catalog/disk/path.go b/v2/pkg/catalog/disk/path.go index 1fbef23ce..18673db29 100644 --- a/v2/pkg/catalog/disk/path.go +++ b/v2/pkg/catalog/disk/path.go @@ -89,7 +89,7 @@ func BackwardsCompatiblePaths(templateDir string, oldPath string) string { // trim the template directory from the path return newPathCallback(tmp) case strings.Contains(oldPath, urlutil.SchemeSeparator): - // scheme seperator is used to identify the path as url + // scheme separator is used to identify the path as url // TBD: add support for url directories ?? return oldPath case strings.Contains(oldPath, "*"): diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index f3b84f4c9..8535b8233 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -59,7 +59,7 @@ type Store struct { preprocessor templates.Preprocessor // NotFoundCallback is called for each not found template - // This overrides error handling for not found templatesss + // This overrides error handling for not found templates NotFoundCallback func(template string) bool } @@ -187,7 +187,7 @@ func (store *Store) ValidateTemplates() error { if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) { return nil } - return errors.New("errors occured during template validation") + return errors.New("errors occurred during template validation") } func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool { diff --git a/v2/pkg/core/execute_options.go b/v2/pkg/core/execute_options.go index 6055d9cb6..4654e7928 100644 --- a/v2/pkg/core/execute_options.go +++ b/v2/pkg/core/execute_options.go @@ -29,7 +29,7 @@ func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target return e.ExecuteScanWithOpts(templatesList, target, false) } -// ExecuteScanWithOpts executes scan with given scanStatergy +// ExecuteScanWithOpts executes scan with given scanStrategy func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { results := &atomic.Bool{} selfcontainedWg := &sync.WaitGroup{} diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 0d40b9768..1fb2d73ad 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -277,7 +277,7 @@ func (i *Input) setItem(metaInput *contextargs.MetaInput) { } } -// setHostMapStream sets iteam in stream mode +// setHostMapStream sets item in stream mode func (i *Input) setHostMapStream(data string) { if _, err := i.hostMapStream.Merge([][]byte{[]byte(data)}); err != nil { gologger.Warning().Msgf("%s\n", err) diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index 8a498b259..dca234383 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -17,9 +17,9 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) -var _ Provider = &customTemplateGithubRepo{} +var _ Provider = &customTemplateGitHubRepo{} -type customTemplateGithubRepo struct { +type customTemplateGitHubRepo struct { owner string reponame string gitCloneURL string @@ -27,8 +27,8 @@ type customTemplateGithubRepo struct { } // This function download the custom github template repository -func (customTemplate *customTemplateGithubRepo) Download(ctx context.Context) { - clonePath := customTemplate.getLocalRepoClonePath(config.DefaultConfig.CustomGithubTemplatesDirectory) +func (customTemplate *customTemplateGitHubRepo) Download(ctx context.Context) { + clonePath := customTemplate.getLocalRepoClonePath(config.DefaultConfig.CustomGitHubTemplatesDirectory) if !fileutil.FolderExists(clonePath) { err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken) @@ -41,8 +41,8 @@ func (customTemplate *customTemplateGithubRepo) Download(ctx context.Context) { } } -func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) { - downloadPath := config.DefaultConfig.CustomGithubTemplatesDirectory +func (customTemplate *customTemplateGitHubRepo) Update(ctx context.Context) { + downloadPath := config.DefaultConfig.CustomGitHubTemplatesDirectory clonePath := customTemplate.getLocalRepoClonePath(downloadPath) // If folder does not exits then clone/download the repo @@ -58,31 +58,31 @@ func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) { } } -// NewGithubProviders returns new instance of GitHub providers for downloading custom templates -func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, error) { - providers := []*customTemplateGithubRepo{} +// NewGitHubProviders returns new instance of GitHub providers for downloading custom templates +func NewGitHubProviders(options *types.Options) ([]*customTemplateGitHubRepo, error) { + providers := []*customTemplateGitHubRepo{} gitHubClient := getGHClientIncognito() if options.GitHubTemplateDisableDownload { return providers, nil } - for _, repoName := range options.GithubTemplateRepo { + for _, repoName := range options.GitHubTemplateRepo { owner, repo, err := getOwnerAndRepo(repoName) if err != nil { gologger.Error().Msgf("%s", err) continue } - githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken) + githubRepo, err := getGitHubRepo(gitHubClient, owner, repo, options.GitHubToken) if err != nil { gologger.Error().Msgf("%s", err) continue } - customTemplateRepo := &customTemplateGithubRepo{ + customTemplateRepo := &customTemplateGitHubRepo{ owner: owner, reponame: repo, gitCloneURL: githubRepo.GetCloneURL(), - githubToken: options.GithubToken, + githubToken: options.GitHubToken, } providers = append(providers, customTemplateRepo) } @@ -104,7 +104,7 @@ func getOwnerAndRepo(reponame string) (owner string, repo string, err error) { } // returns *github.Repository if passed github repo name -func getGithubRepo(gitHubClient *github.Client, repoOwner, repoName, githubToken string) (*github.Repository, error) { +func getGitHubRepo(gitHubClient *github.Client, repoOwner, repoName, githubToken string) (*github.Repository, error) { var retried bool getRepo: repo, _, err := gitHubClient.Repositories.Get(context.Background(), repoOwner, repoName) @@ -123,7 +123,7 @@ getRepo: } // download the git repo to a given path -func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) error { +func (ctr *customTemplateGitHubRepo) cloneRepo(clonePath, githubToken string) error { r, err := git.PlainClone(clonePath, false, &git.CloneOptions{ URL: ctr.gitCloneURL, Auth: getAuth(ctr.owner, githubToken), @@ -138,7 +138,7 @@ func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) er } // performs the git pull on given repo -func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) error { +func (ctr *customTemplateGitHubRepo) pullChanges(repoPath, githubToken string) error { r, err := git.PlainOpen(repoPath) if err != nil { return err @@ -155,7 +155,7 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e } // All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness -func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string { +func (ctr *customTemplateGitHubRepo) getLocalRepoClonePath(downloadPath string) string { return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner) } diff --git a/v2/pkg/external/customtemplates/github_test.go b/v2/pkg/external/customtemplates/github_test.go index e21a62a90..6c812f376 100644 --- a/v2/pkg/external/customtemplates/github_test.go +++ b/v2/pkg/external/customtemplates/github_test.go @@ -22,8 +22,8 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { config.DefaultConfig.SetTemplatesDir(templatesDirectory) options := testutils.DefaultOptions - options.GithubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"} - options.GithubToken = os.Getenv("GITHUB_TOKEN") + options.GitHubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"} + options.GitHubToken = os.Getenv("GITHUB_TOKEN") ctm, err := NewCustomTemplatesManager(options) require.Nil(t, err, "could not create custom templates manager") diff --git a/v2/pkg/external/customtemplates/gitlab.go b/v2/pkg/external/customtemplates/gitlab.go index db272c98c..61dbca0a3 100644 --- a/v2/pkg/external/customtemplates/gitlab.go +++ b/v2/pkg/external/customtemplates/gitlab.go @@ -21,8 +21,8 @@ type customTemplateGitLabRepo struct { projectIDs []int } -// NewGitlabProviders returns a new list of GitLab providers for downloading custom templates -func NewGitlabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) { +// NewGitLabProviders returns a new list of GitLab providers for downloading custom templates +func NewGitLabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) { providers := []*customTemplateGitLabRepo{} if options.GitLabToken != "" && !options.GitLabTemplateDisableDownload { // Establish a connection to GitLab and build a client object with which to download templates from GitLab diff --git a/v2/pkg/external/customtemplates/templates_provider.go b/v2/pkg/external/customtemplates/templates_provider.go index 02a9794f5..6ae01e3bd 100644 --- a/v2/pkg/external/customtemplates/templates_provider.go +++ b/v2/pkg/external/customtemplates/templates_provider.go @@ -41,7 +41,7 @@ func NewCustomTemplatesManager(options *types.Options) (*CustomTemplatesManager, } // Add GitHub providers - githubProviders, err := NewGithubProviders(options) + githubProviders, err := NewGitHubProviders(options) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create github providers for custom templates") } @@ -68,7 +68,7 @@ func NewCustomTemplatesManager(options *types.Options) (*CustomTemplatesManager, } // Add GitLab providers - gitlabProviders, err := NewGitlabProviders(options) + gitlabProviders, err := NewGitLabProviders(options) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create gitlab providers for custom templates") } diff --git a/v2/pkg/input/input.go b/v2/pkg/input/input.go index 5dadb2bf1..eb379a860 100644 --- a/v2/pkg/input/input.go +++ b/v2/pkg/input/input.go @@ -18,7 +18,7 @@ type Helper struct { InputsHTTP *hybrid.HybridMap } -// NewHelper returns a new inpt helper instance +// NewHelper returns a new input helper instance func NewHelper() *Helper { helper := &Helper{} return helper diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index f8d1246f2..507eb367a 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -70,7 +70,7 @@ func TestLoadTemplate(t *testing.T) { expectedErr: errors.New("field 'severity' is missing"), }, { - name: "template-without-serverity-with-correct-filter-id", + name: "template-without-severity-with-correct-filter-id", template: &templates.Template{ ID: "CVE-2021-27330", Info: model.Info{ @@ -84,7 +84,7 @@ func TestLoadTemplate(t *testing.T) { filter: filter.Config{IncludeIds: []string{"CVE-2021-27330"}}, }, { - name: "template-without-serverity-with-diff-filter-id", + name: "template-without-severity-with-diff-filter-id", template: &templates.Template{ ID: "CVE-2021-27330", Info: model.Info{ diff --git a/v2/pkg/protocols/common/automaticscan/doc.go b/v2/pkg/protocols/common/automaticscan/doc.go index b9756af76..02ff42f2e 100644 --- a/v2/pkg/protocols/common/automaticscan/doc.go +++ b/v2/pkg/protocols/common/automaticscan/doc.go @@ -14,5 +14,5 @@ // detection. // // The logic is very simple and can be further improved to increase the coverage of -// this mode of nuclei exection. +// this mode of nuclei execution. package automaticscan diff --git a/v2/pkg/protocols/common/helpers/deserialization/java.go b/v2/pkg/protocols/common/helpers/deserialization/java.go index 68dda623d..cde50d547 100644 --- a/v2/pkg/protocols/common/helpers/deserialization/java.go +++ b/v2/pkg/protocols/common/helpers/deserialization/java.go @@ -75,7 +75,7 @@ func urlsafeBase64Encode(data []byte) string { } // generateCommonsCollections40Payload generates org.apache.commons:commons-collections4:4.0 -// deserialization paylaod for a command. +// deserialization payload for a command. func generateCommonsCollections40Payload(cmd string) []byte { buffer := &bytes.Buffer{} @@ -90,7 +90,7 @@ func generateCommonsCollections40Payload(cmd string) []byte { } // generateCommonsCollections440PPayload generates commons-collections 3.1 -// deserialization paylaod for a command. +// deserialization payload for a command. func generateCommonsCollections31Payload(cmd string) []byte { buffer := &bytes.Buffer{} @@ -105,7 +105,7 @@ func generateCommonsCollections31Payload(cmd string) []byte { } // generateGroovy1Payload generates org.codehaus.groovy:groovy:2.3.9 -// deserialization paylaod for a command. +// deserialization payload for a command. func generateGroovy1Payload(cmd string) []byte { buffer := &bytes.Buffer{} @@ -119,7 +119,7 @@ func generateGroovy1Payload(cmd string) []byte { return buffer.Bytes() } -// generateDNSPayload generates DNS interaction deserialization paylaod for a DNS Name. +// generateDNSPayload generates DNS interaction deserialization payload for a DNS Name. // Taken from ysoserial DNS gadget. func generateDNSPayload(URL string) []byte { parsed, err := url.Parse(URL) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 3c006f661..6539915a7 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -128,7 +128,7 @@ func (c *Client) poll() error { return nil } -// requestShouldStopAtFirstmatch checks if furthur interactions should be stopped +// requestShouldStopAtFirstmatch checks if further interactions should be stopped // note: extra care should be taken while using this function since internalEvent is // synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that // we could use `TryLock()` but that may over complicate things and need to differentiate diff --git a/v2/pkg/protocols/common/interactsh/options.go b/v2/pkg/protocols/common/interactsh/options.go index 22a745d37..48d536803 100644 --- a/v2/pkg/protocols/common/interactsh/options.go +++ b/v2/pkg/protocols/common/interactsh/options.go @@ -43,7 +43,7 @@ type Options struct { DisableHttpFallback bool // NoInteractsh disables the engine NoInteractsh bool - // NoColor dissbles printing colors for matches + // NoColor disables printing colors for matches NoColor bool StopAtFirstMatch bool diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index 95293a519..829c2ce34 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -106,14 +106,14 @@ func Init(options *types.Options) error { } // isIpAssociatedWithInterface checks if the given IP is associated with the given interface. -func isIpAssociatedWithInterface(souceIP, interfaceName string) (bool, error) { +func isIpAssociatedWithInterface(sourceIP, interfaceName string) (bool, error) { addrs, err := interfaceAddresses(interfaceName) if err != nil { return false, err } for _, addr := range addrs { if ipnet, ok := addr.(*net.IPNet); ok { - if ipnet.IP.String() == souceIP { + if ipnet.IP.String() == sourceIP { return true, nil } } diff --git a/v2/pkg/protocols/common/randomip/randomip_test.go b/v2/pkg/protocols/common/randomip/randomip_test.go index 2fcda223d..9cc9eb7b3 100644 --- a/v2/pkg/protocols/common/randomip/randomip_test.go +++ b/v2/pkg/protocols/common/randomip/randomip_test.go @@ -45,7 +45,7 @@ func TestGetRandomIp(t *testing.T) { valid: true, }, { - name: "Negativ CIDR length", + name: "Negative CIDR length", cidr: []string{"10.11.12.13/-1"}, valid: false, errorMsg: "10.11.12.13/-1 is not a valid CIDR", diff --git a/v2/pkg/protocols/common/uncover/uncover.go b/v2/pkg/protocols/common/uncover/uncover.go index 7ba2fa905..9a30590b1 100644 --- a/v2/pkg/protocols/common/uncover/uncover.go +++ b/v2/pkg/protocols/common/uncover/uncover.go @@ -53,7 +53,7 @@ func GetTargetsFromUncover(ctx context.Context, outputFormat string, opts *uncov return outputChan, nil } -// processUncoverOutput returns output strign depending on uncover field +// processUncoverOutput returns output string depending on uncover field func processUncoverOutput(result sources.Result, outputFormat string) string { if (result.IP == "" || result.Port == 0) && stringsutil.ContainsAny(outputFormat, "ip", "port") { // if ip or port is not present, fallback to using host diff --git a/v2/pkg/protocols/common/variables/doc.go b/v2/pkg/protocols/common/variables/doc.go index 74e7d9c31..b0b54a87e 100644 --- a/v2/pkg/protocols/common/variables/doc.go +++ b/v2/pkg/protocols/common/variables/doc.go @@ -22,6 +22,6 @@ package variables // Non-Linear Sources (need to re-evaluate) // 1. VariablesMap // 2. PayloadsMap -// Everytime Linear Sources are updated , Non-Linear Sources need to be re-evaluated +// Every time Linear Sources are updated , Non-Linear Sources need to be re-evaluated // Constants (no need to re-evaluate, should contain only scalars) diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index f9ce8e6a9..500deb18c 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -227,7 +227,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads return nil } -// getLastNaviationURL returns last successfully navigated URL +// getLastNavigationURL returns last successfully navigated URL func (request *Request) getLastNavigationURLWithLog(reqLog map[string]string) string { for i := len(request.Steps) - 1; i >= 0; i-- { if request.Steps[i].ActionType.ActionType == engine.ActionNavigate { diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index af551d09c..921af0245 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -253,7 +253,7 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalVars, interactshURLs: r.interactshURLs}, nil } -// generateRawRequest generates Raw Request from from request data from template and variables +// generateRawRequest generates Raw Request from request data from template and variables // finalVars = contains all variables including generator and protocol specific variables // generatorValues = contains variables used in fuzzing or other generator specific values func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest string, baseURL *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) { diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 46a6b562a..58aae6cb8 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -53,7 +53,7 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b return nil, errorutil.NewWithTag("raw", "could not automergepath for template path %v", urlx.GetRelativePath()).Wrap(parseErr) } rawrequest.Path = cloned.GetRelativePath() - // If unsafe changes must be made in raw request string iteself + // If unsafe changes must be made in raw request string itself case unsafe: prevPath := rawrequest.Path cloned := inputURL.Clone() diff --git a/v2/pkg/protocols/http/request_annotations.go b/v2/pkg/protocols/http/request_annotations.go index 5f0e26777..6555957c8 100644 --- a/v2/pkg/protocols/http/request_annotations.go +++ b/v2/pkg/protocols/http/request_annotations.go @@ -19,9 +19,9 @@ var ( // @tls-sni:target overrides the input target with the annotated one // special values: // request.host: takes the value from the host header - // target: overiddes with the specific value + // target: overrides with the specific value reSniAnnotation = regexp.MustCompile(`(?m)^@tls-sni:\s*(.+)\s*$`) - // @timeout:duration overrides the input timout with a custom duration + // @timeout:duration overrides the input timeout with a custom duration reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`) // @once sets the request to be executed only once for a specific URL reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`) @@ -36,15 +36,15 @@ const ( // parseFlowAnnotations and override requests flow func parseFlowAnnotations(rawRequest string) (flowMark, bool) { var fm flowMark - // parse request for known ovverride annotations - var hasFlowOveride bool + // parse request for known override annotations + var hasFlowOverride bool // @once if reOnceAnnotation.MatchString(rawRequest) { fm = Once - hasFlowOveride = true + hasFlowOverride = true } - return fm, hasFlowOveride + return fm, hasFlowOverride } type annotationOverrides struct { @@ -55,7 +55,7 @@ type annotationOverrides struct { // parseAnnotations and override requests settings func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) { - // parse request for known ovverride annotations + // parse request for known override annotations // @Host:target if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 { diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index aecff19ea..09d408115 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -58,15 +58,15 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, variablesMap := request.options.Variables.Evaluate(variables) variables = generators.MergeMaps(variablesMap, variables, request.options.Constants) - visitedAddressess := make(mapsutil.Map[string, struct{}]) + visitedAddresses := make(mapsutil.Map[string, struct{}]) for _, kv := range request.addresses { actualAddress := replacer.Replace(kv.address, variables) - if visitedAddressess.Has(actualAddress) && !request.options.Options.DisableClustering { + if visitedAddresses.Has(actualAddress) && !request.options.Options.DisableClustering { continue } - visitedAddressess.Set(actualAddress, struct{}{}) + visitedAddresses.Set(actualAddress, struct{}{}) if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { outputEvent := request.responseToDSLMap("", "", "", address, "") diff --git a/v2/pkg/protocols/utils/http/requtils.go b/v2/pkg/protocols/utils/http/requtils.go index 97bb49f6e..4f2f860e7 100644 --- a/v2/pkg/protocols/utils/http/requtils.go +++ b/v2/pkg/protocols/utils/http/requtils.go @@ -11,7 +11,7 @@ import ( var ( // TODO: adapt regex for cases where port is updated urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`) - // regex to detect traling slash in path (not applicable to raw requests) + // regex to detect trailing slash in path (not applicable to raw requests) trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`) // ErrNoMoreRequests is internal error to ) diff --git a/v2/pkg/protocols/utils/variables_test.go b/v2/pkg/protocols/utils/variables_test.go index a603137f3..5c4e25efa 100644 --- a/v2/pkg/protocols/utils/variables_test.go +++ b/v2/pkg/protocols/utils/variables_test.go @@ -12,7 +12,7 @@ import ( func TestHTTPVariables(t *testing.T) { baseURL := "http://localhost:9001/test/123" parsed, _ := urlutil.Parse(baseURL) - // trailingslash is only true when both target/inputURL and payload {{BaseURL}}/xyz both have slash + // trailing slash is only true when both target/inputURL and payload {{BaseURL}}/xyz both have slash values := GenerateVariables(parsed, false, nil) require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl") diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 26468fd35..ab51478ad 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -64,7 +64,7 @@ type Request struct { // Payloads support both key-values combinations where a list // of payloads is provided, or optionally a single file can also // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the websocket request,description=Payloads contains any payloads for the current request"` generator *generators.PayloadGenerator diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 301d66a04..d1c8caf10 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -29,7 +29,7 @@ import ( // to the first individual request is compared for equality. // The equality check is performed as described below - // -// Cases where clustering is not perfomed (request is considered different) +// Cases where clustering is not performed (request is considered different) // - If request contains payloads,raw,body,unsafe,req-condition,name attributes // - If request methods,max-redirects,cookie-reuse,redirects are not equal // - If request paths aren't identical. diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index eec06055a..e8d342084 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -114,7 +114,7 @@ type Template struct { Variables variables.Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request"` // description: | - // Constants contains any scalar costant for the current template + // Constants contains any scalar constant for the current template Constants map[string]interface{} `yaml:"constants,omitempty" json:"constants,omitempty" jsonschema:"title=constant for the template,description=constants contains any constant for the template"` // TotalRequests is the total number of requests for the template. diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 992e5bed6..fb490ffb5 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -141,8 +141,8 @@ func init() { TemplateDoc.Fields[16].Name = "constants" TemplateDoc.Fields[16].Type = "map[string]interface{}" TemplateDoc.Fields[16].Note = "" - TemplateDoc.Fields[16].Description = "Constants contains any scalar costant for the current template" - TemplateDoc.Fields[16].Comments[encoder.LineComment] = "Constants contains any scalar costant for the current template" + TemplateDoc.Fields[16].Description = "Constants contains any scalar constant for the current template" + TemplateDoc.Fields[16].Comments[encoder.LineComment] = "Constants contains any scalar constant for the current template" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 68e9e78fb..26ffe7000 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -64,8 +64,8 @@ var DefaultOptions = &types.Options{ InteractionsEviction: 60, InteractionsCoolDownPeriod: 5, InteractionsPollDuration: 5, - GithubTemplateRepo: []string{}, - GithubToken: "", + GitHubTemplateRepo: []string{}, + GitHubToken: "", } // TemplateInfo contains info for a mock executed template. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c3842bbb6..fe22ad302 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -102,7 +102,7 @@ type Options struct { CloudURL string // CloudAPIKey is the api-key for the nuclei cloud endpoint CloudAPIKey string - // Scanlist feature to get all the scan ids for a user + // ScanList feature to get all the scan ids for a user ScanList bool // ListDatasources enables listing of datasources for user ListDatasources bool @@ -326,7 +326,7 @@ type Options struct { IncludeConditions goflags.StringSlice // Custom Config Directory CustomConfigDir string - // Enable uncover egine + // Enable uncover engine Uncover bool // Uncover search query UncoverQuery goflags.StringSlice @@ -345,9 +345,9 @@ type Options struct { // PublicTemplateDisableDownload disables downloading templates from the nuclei-templates public repository PublicTemplateDisableDownload bool // GitHub token used to clone/pull from private repos for custom templates - GithubToken string - // GithubTemplateRepo is the list of custom public/private templates GitHub repos - GithubTemplateRepo []string + GitHubToken string + // GitHubTemplateRepo is the list of custom public/private templates GitHub repos + GitHubTemplateRepo []string // GitHubTemplateDisableDownload disables downloading templates from custom GitHub repositories GitHubTemplateDisableDownload bool // GitLabServerURL is the gitlab server to use for custom templates From 0776b2e237f8e714378ec67c5a5f804867c7e661 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 1 Aug 2023 22:17:42 +0300 Subject: [PATCH 61/62] fix curl command (#4014) --- v2/pkg/protocols/http/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 04469edc1..aed626ff5 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -641,7 +641,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil && !request.Race { bodyBytes, _ := generatedRequest.request.BodyBytes() resp.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes)) - command, err := http2curl.GetCurlCommand(resp.Request) + command, err := http2curl.GetCurlCommand(generatedRequest.request.Request) if err == nil && command != nil { curlCommand = command.String() } From c8a7df98f377a3624ace24f71ca24cd687e1fc06 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Fri, 4 Aug 2023 00:56:32 +0530 Subject: [PATCH 62/62] fix removing double slash prefix in raw req path (#3960) * update utils lib * add integration test on unsafe:false * fix build error --------- Co-authored-by: mzack --- integration_tests/http/http-paths.yaml | 7 +++++++ integration_tests/http/raw-unsafe-path.yaml | 5 +++++ v2/go.mod | 2 +- v2/go.sum | 4 ++-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/integration_tests/http/http-paths.yaml b/integration_tests/http/http-paths.yaml index a43725dfc..ae389a379 100644 --- a/integration_tests/http/http-paths.yaml +++ b/integration_tests/http/http-paths.yaml @@ -16,6 +16,7 @@ info: - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "/test/..;/..;/" - "/xyz/%25u2s/%25invalid" + - "//CFIDE/wizards/common/utils.cfc" # duplicating here because same results are expected even if http request is written in different format - "/1337?with=param" - "/some%0A/%0D" @@ -24,6 +25,7 @@ info: - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "/test/..;/..;/" - "/xyz/%25u2s/%25invalid" + - "//CFIDE/wizards/common/utils.cfc" # Test all templates with FullURLs requests: @@ -62,6 +64,10 @@ requests: - |+ GET /xyz/%u2s/%invalid HTTP/1.1 Host: scanme.sh + # test relative path start with // + - |+ + GET //CFIDE/wizards/common/utils.cfc HTTP/1.1 + Host: scanme.sh matchers: - type: status @@ -77,6 +83,7 @@ requests: - "{{BaseURL}}/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "{{BaseURL}}/test/..;/..;/" - "{{BaseURL}}/xyz/%u2s/%invalid" + - "{{BaseURL}}//CFIDE/wizards/common/utils.cfc" matchers: - type: status diff --git a/integration_tests/http/raw-unsafe-path.yaml b/integration_tests/http/raw-unsafe-path.yaml index 4c86dcfd9..a10721eb3 100644 --- a/integration_tests/http/raw-unsafe-path.yaml +++ b/integration_tests/http/raw-unsafe-path.yaml @@ -16,6 +16,7 @@ info: - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d" - "/test/..;/..;/" - "/xyz/%u2s/%invalid" + - "//CFIDE/wizards/common/utils.cfc" # Test all unsafe URL Handling Edgecases @@ -50,6 +51,10 @@ requests: - |+ GET /xyz/%u2s/%invalid HTTP/1.1 Host: scanme.sh + # test relative path start with // (should not be removed) + - |+ + GET //CFIDE/wizards/common/utils.cfc HTTP/1.1 + Host: scanme.sh unsafe: true matchers: diff --git a/v2/go.mod b/v2/go.mod index ad060b754..d3e16a239 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -77,7 +77,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.1 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.45 + github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 diff --git a/v2/go.sum b/v2/go.sum index 3f6ebda3c..19c7a233d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -435,8 +435,8 @@ github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJK github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.45 h1:i0xwdpaGrTIw8hVtutjpVXvPCMcKKwb9VbqWxlwL08M= -github.com/projectdiscovery/utils v0.0.45/go.mod h1:kJu+OqAWsoilLto06ajSp4U3gWuz51GEVLL/PogJXPk= +github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 h1:bgTXdrA/yFhFGfjhMIsczVNhnsMEHFidgS/FD2Tq5Js= +github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=