From 067b091d87293351477763372a43a4053b159e15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 May 2022 05:29:18 +0000 Subject: [PATCH 01/32] chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.38 to 0.0.40. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.38...v0.0.40) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index d7183e2bc..9d5d8b8ec 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 - github.com/projectdiscovery/wappalyzergo v0.0.38 + github.com/projectdiscovery/wappalyzergo v0.0.40 github.com/stretchr/testify v1.7.1 github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b ) diff --git a/v2/go.sum b/v2/go.sum index 4f2ecee48..a37312420 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -498,8 +498,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.38 h1:0HGXJfMGMfveM5R0oGIWL0Sc513+gVPoHQkjZxmT2GA= -github.com/projectdiscovery/wappalyzergo v0.0.38/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.40 h1:E/rADqztV3+APjE5u/SEH046Ylc1R2wzG7TZVATf0is= +github.com/projectdiscovery/wappalyzergo v0.0.40/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= From ede9e46ae36faf32d238ecc2316b6b50a7958288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 11:59:59 +0000 Subject: [PATCH 02/32] chore(deps): bump github.com/aws/aws-sdk-go from 1.44.7 to 1.44.8 in /v2 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.7 to 1.44.8. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.7...v1.44.8) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 2bf8d12a0..fd6c43255 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -63,7 +63,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.7 +require github.com/aws/aws-sdk-go v1.44.8 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 diff --git a/v2/go.sum b/v2/go.sum index 591a791a6..180e8129c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,8 +86,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.7 h1:LpCM8Fpw/L58vgdve6A+UqJr8dzo6Xj7HX7DIIGHg2A= -github.com/aws/aws-sdk-go v1.44.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.8 h1:Fc8lFiP97MjB/BFaeZ7ixAjkqrNoic/qvlRqqJb5cxY= +github.com/aws/aws-sdk-go v1.44.8/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From f8e462e3ab63b7c1129fa5b6133cc3f42dd390c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 05:05:50 +0000 Subject: [PATCH 03/32] chore(deps): bump docker/login-action from 1 to 2 Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 2b04cbebd..f4050a872 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -26,7 +26,7 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} From f9269bcaf5a5b3eafe9b74b4cd096af9b7e57dbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 05:30:08 +0000 Subject: [PATCH 04/32] chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.40 to 0.0.41. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.40...v0.0.41) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 7755b37d3..658647067 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 - github.com/projectdiscovery/wappalyzergo v0.0.40 + github.com/projectdiscovery/wappalyzergo v0.0.41 github.com/stretchr/testify v1.7.1 github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b ) diff --git a/v2/go.sum b/v2/go.sum index 4af194e72..8fe2b040d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -103,7 +103,6 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= -github.com/caddyserver/certmagic v0.16.0/go.mod h1:jKQ5n+ViHAr6DbPwEGLTSM2vDwTO6EvCKBblBRUvvuQ= github.com/caddyserver/certmagic v0.16.1 h1:rdSnjcUVJojmL4M0efJ+yHXErrrijS4YYg3FuwRdJkI= github.com/caddyserver/certmagic v0.16.1/go.mod h1:jKQ5n+ViHAr6DbPwEGLTSM2vDwTO6EvCKBblBRUvvuQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -324,7 +323,6 @@ github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwS github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw= github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= @@ -459,8 +457,6 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1: github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= -github.com/projectdiscovery/interactsh v1.0.3 h1:QFL/fFTfvhLeIRFBSguWP8r6h8vf65Bhkf8wzKU2/qQ= -github.com/projectdiscovery/interactsh v1.0.3/go.mod h1:3943djLJ4SZBZfJ7A2XEy8xspfFmT0BDiN9FCW7PAXY= github.com/projectdiscovery/interactsh v1.0.4 h1:73HHOeqGduU1lkIqeYkuEfzdejXRqxQJFZCEbQ1uvkU= github.com/projectdiscovery/interactsh v1.0.4/go.mod h1:57Xl3Q59aTb8VaFNGkCPO8pWTyTYhlM33p+HDhL8Ygw= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= @@ -490,8 +486,6 @@ github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlS github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= -github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e h1:ohoSKR8w4GzGjmOaqdCa8pvHm3qbAyv489skpyrkCX0= -github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk= @@ -505,8 +499,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.40 h1:E/rADqztV3+APjE5u/SEH046Ylc1R2wzG7TZVATf0is= -github.com/projectdiscovery/wappalyzergo v0.0.40/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.41 h1:t6zsfLiMFyjsIv0BYYKSDSxDbsG+FEuO1/TjE/eD55o= +github.com/projectdiscovery/wappalyzergo v0.0.41/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= From d7d80e3447b0faeb9ad5a2f861c2d12a1257c984 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 9 May 2022 11:02:21 +0530 Subject: [PATCH 05/32] Added normalized wappalyzer versioned appName --- .../common/automaticscan/automaticscan.go | 11 ++++++++++- .../common/automaticscan/automaticscan_test.go | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/protocols/common/automaticscan/automaticscan_test.go diff --git a/v2/pkg/protocols/common/automaticscan/automaticscan.go b/v2/pkg/protocols/common/automaticscan/automaticscan.go index 3cf72db1f..465e14089 100644 --- a/v2/pkg/protocols/common/automaticscan/automaticscan.go +++ b/v2/pkg/protocols/common/automaticscan/automaticscan.go @@ -171,7 +171,7 @@ func (s *Service) processWappalyzerInputPair(input string) { fingerprints := s.wappalyzer.Fingerprint(resp.Header, data) normalized := make(map[string]struct{}) for k := range fingerprints { - normalized[strings.ToLower(k)] = struct{}{} + normalized[normalizeAppName(k)] = struct{}{} } if s.opts.Options.Verbose { @@ -215,6 +215,15 @@ func (s *Service) processWappalyzerInputPair(input string) { } } +func normalizeAppName(appName string) string { + if strings.Contains(appName, ":") { + if parts := strings.Split(appName, ":"); len(parts) == 2 { + appName = parts[0] + } + } + return strings.ToLower(appName) +} + func uniqueSlice(slice []string) []string { data := make(map[string]struct{}, len(slice)) for _, item := range slice { diff --git a/v2/pkg/protocols/common/automaticscan/automaticscan_test.go b/v2/pkg/protocols/common/automaticscan/automaticscan_test.go new file mode 100644 index 000000000..b60694338 --- /dev/null +++ b/v2/pkg/protocols/common/automaticscan/automaticscan_test.go @@ -0,0 +1,15 @@ +package automaticscan + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeAppName(t *testing.T) { + appName := normalizeAppName("JBoss") + require.Equal(t, "jboss", appName, "could not get normalized name") + + appName = normalizeAppName("JBoss:2.3.5") + require.Equal(t, "jboss", appName, "could not get normalized name") +} From 61fc65b6214c366229bce3f7058deea262f95661 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 05:47:38 +0000 Subject: [PATCH 06/32] chore(deps): bump github.com/aws/aws-sdk-go from 1.44.8 to 1.44.9 in /v2 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.8 to 1.44.9. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.8...v1.44.9) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 7755b37d3..18add8719 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -63,7 +63,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.7 +require github.com/aws/aws-sdk-go v1.44.9 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 diff --git a/v2/go.sum b/v2/go.sum index 4af194e72..9ef29a6fa 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,8 +86,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.7 h1:LpCM8Fpw/L58vgdve6A+UqJr8dzo6Xj7HX7DIIGHg2A= -github.com/aws/aws-sdk-go v1.44.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.9 h1:s3lsEFbc8i7ghQmcEpcdyvoO/WMwyCVa9pUq3Lq//Ok= +github.com/aws/aws-sdk-go v1.44.9/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -103,7 +103,6 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= -github.com/caddyserver/certmagic v0.16.0/go.mod h1:jKQ5n+ViHAr6DbPwEGLTSM2vDwTO6EvCKBblBRUvvuQ= github.com/caddyserver/certmagic v0.16.1 h1:rdSnjcUVJojmL4M0efJ+yHXErrrijS4YYg3FuwRdJkI= github.com/caddyserver/certmagic v0.16.1/go.mod h1:jKQ5n+ViHAr6DbPwEGLTSM2vDwTO6EvCKBblBRUvvuQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -324,7 +323,6 @@ github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwS github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw= github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= @@ -459,8 +457,6 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1: github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= -github.com/projectdiscovery/interactsh v1.0.3 h1:QFL/fFTfvhLeIRFBSguWP8r6h8vf65Bhkf8wzKU2/qQ= -github.com/projectdiscovery/interactsh v1.0.3/go.mod h1:3943djLJ4SZBZfJ7A2XEy8xspfFmT0BDiN9FCW7PAXY= github.com/projectdiscovery/interactsh v1.0.4 h1:73HHOeqGduU1lkIqeYkuEfzdejXRqxQJFZCEbQ1uvkU= github.com/projectdiscovery/interactsh v1.0.4/go.mod h1:57Xl3Q59aTb8VaFNGkCPO8pWTyTYhlM33p+HDhL8Ygw= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= @@ -490,8 +486,6 @@ github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlS github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= -github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e h1:ohoSKR8w4GzGjmOaqdCa8pvHm3qbAyv489skpyrkCX0= -github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220414143248-bb6eabffa43e/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk= From 27673746d1b5b670031daa36d198e4efe4531bef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 May 2022 05:23:54 +0000 Subject: [PATCH 07/32] chore(deps): bump github.com/go-rod/rod from 0.106.5 to 0.106.6 in /v2 Bumps [github.com/go-rod/rod](https://github.com/go-rod/rod) from 0.106.5 to 0.106.6. - [Release notes](https://github.com/go-rod/rod/releases) - [Commits](https://github.com/go-rod/rod/compare/v0.106.5...v0.106.6) --- updated-dependencies: - dependency-name: github.com/go-rod/rod dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3048b706b..db27079d6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.1.1 github.com/go-playground/validator/v10 v10.11.0 - github.com/go-rod/rod v0.106.5 + github.com/go-rod/rod v0.106.6 github.com/gobwas/ws v1.1.0 github.com/google/go-github v17.0.0+incompatible github.com/itchyny/gojq v0.12.7 diff --git a/v2/go.sum b/v2/go.sum index 607340d45..bb7c87211 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -173,8 +173,8 @@ github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2B github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= -github.com/go-rod/rod v0.106.5 h1:HQOxmOb/xjZ+QnnK4D8t+3+CWFreKSuqid/65zH4poA= -github.com/go-rod/rod v0.106.5/go.mod h1:N/ZJik0+EYXNpW/74q0V1H7K3/UTfwHV/tRxY0CY/Vw= +github.com/go-rod/rod v0.106.6 h1:zJorVPG7s8Xgbh7PkSySP4FNoo0OiougKaMb3j6zT6w= +github.com/go-rod/rod v0.106.6/go.mod h1:xkZOchuKqTOkMOBkrzb7uJpbKZRab1haPCWDvuZkS2U= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -605,8 +605,8 @@ github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= github.com/ysmood/got v0.14.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= -github.com/ysmood/got v0.28.2 h1:2dtVOneu4YGDk9skgzq8aqW9vZ1ssBla+ddqrE91bNs= -github.com/ysmood/got v0.28.2/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/got v0.29.1 h1:7TNTm3Bw5kdBGXFp04qnZ9DqlZ1XS1z1JdeobeJY6Mo= +github.com/ysmood/got v0.29.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= github.com/ysmood/gotrace v0.2.2/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= From 25d6cbd95b6aa015de71e8f59b449e2251700e52 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 10 May 2022 17:26:46 +0530 Subject: [PATCH 08/32] Use nuclei-templates instead of custom autoscan directory list (#1968) * Use nuclei-templates instead of custom autoscan directory list * Use templates directory from config file --- v2/pkg/protocols/common/automaticscan/automaticscan.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/common/automaticscan/automaticscan.go b/v2/pkg/protocols/common/automaticscan/automaticscan.go index 465e14089..7821784bc 100644 --- a/v2/pkg/protocols/common/automaticscan/automaticscan.go +++ b/v2/pkg/protocols/common/automaticscan/automaticscan.go @@ -67,6 +67,8 @@ func New(opts Options) (*Service, error) { if opts.ExecuterOpts.Options.Verbose { gologger.Verbose().Msgf("Normalized mapping (%d): %v\n", len(mappingData), mappingData) } + defaultTemplatesDirectories := []string{config.TemplatesDirectory} + // adding custom template path if available if len(opts.ExecuterOpts.Options.Templates) > 0 { defaultTemplatesDirectories = append(defaultTemplatesDirectories, opts.ExecuterOpts.Options.Templates...) @@ -118,10 +120,6 @@ func (s *Service) Execute() { } } -var ( - defaultTemplatesDirectories = []string{"cves/", "default-logins/", "dns/", "exposures/", "miscellaneous/", "misconfiguration/", "network/", "takeovers/", "vulnerabilities/"} -) - const maxDefaultBody = 2 * 1024 * 1024 // executeWappalyzerTechDetection implements the logic to run the wappalyzer From d5578200f047db4736c4bd9add2d2971565fe8e4 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 10 May 2022 17:33:27 +0530 Subject: [PATCH 09/32] Fixed query parameter addition to http raw requests (#1975) --- v2/pkg/protocols/http/raw/raw.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 6d8db6293..94fbc10ca 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -130,6 +130,9 @@ read_line: rawRequest.Path = strings.TrimSuffix(rawRequest.Path, "/") } rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) + if parsedURL.RawQuery != "" { + rawRequest.FullURL = fmt.Sprintf("%s?%s", rawRequest.FullURL, parsedURL.RawQuery) + } // If raw request doesn't have a Host header and isn't marked unsafe, // this will generate the Host header from the parsed baseURL From 59fc6963e0257ad1e0f8dc04e4bc5f2fb6f56b47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 May 2022 17:48:44 +0530 Subject: [PATCH 10/32] chore(deps): bump docker/setup-buildx-action from 1 to 2 (#1960) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mzack --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f4050a872..84e6b56e4 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -23,7 +23,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v2 From 1683851b46eaca605911968608f6f67c8f9f8709 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 May 2022 17:49:08 +0530 Subject: [PATCH 11/32] chore(deps): bump docker/build-push-action from 2 to 3 (#1959) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 2 to 3. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mzack --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 84e6b56e4..c66fc29d5 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -32,7 +32,7 @@ jobs: password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 From 2eb845262b0a0e4b9a2b01853c9578aa273b9440 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 May 2022 17:49:26 +0530 Subject: [PATCH 12/32] chore(deps): bump docker/setup-qemu-action from 1 to 2 (#1961) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mzack --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index c66fc29d5..43b29c737 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -20,7 +20,7 @@ jobs: echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name)" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 From a2947192e2e9195cba4686739ebef7de12a6bea2 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 10 May 2022 18:47:22 +0530 Subject: [PATCH 13/32] Added mutex to output writing (#1969) --- v2/pkg/output/output.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 3b52d5580..a46118d14 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -7,6 +7,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "github.com/pkg/errors" @@ -48,6 +49,7 @@ type StandardWriter struct { noTimestamp bool noMetadata bool matcherStatus bool + mutex *sync.Mutex aurora aurora.Aurora outputFile io.WriteCloser traceFile io.WriteCloser @@ -161,6 +163,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, Match matcherStatus: MatcherStatus, noTimestamp: noTimestamp, aurora: auroraColorizer, + mutex: &sync.Mutex{}, outputFile: outputFile, traceFile: traceOutput, errorFile: errorOutput, @@ -193,6 +196,9 @@ func (w *StandardWriter) Write(event *ResultEvent) error { if len(data) == 0 { return nil } + w.mutex.Lock() + defer w.mutex.Unlock() + _, _ = os.Stdout.Write(data) _, _ = os.Stdout.Write([]byte("\n")) From 4c79a53c3ba51faaf7156576617e16bcdc8f03f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 May 2022 05:23:07 +0000 Subject: [PATCH 14/32] chore(deps): bump github.com/miekg/dns from 1.1.48 to 1.1.49 in /v2 Bumps [github.com/miekg/dns](https://github.com/miekg/dns) from 1.1.48 to 1.1.49. - [Release notes](https://github.com/miekg/dns/releases) - [Changelog](https://github.com/miekg/dns/blob/master/Makefile.release) - [Commits](https://github.com/miekg/dns/compare/v1.1.48...v1.1.49) --- updated-dependencies: - dependency-name: github.com/miekg/dns dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3048b706b..4f7d5b320 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,7 +21,7 @@ require ( github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.17.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.48 + github.com/miekg/dns v1.1.49 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.1.1 github.com/pkg/errors v0.9.1 diff --git a/v2/go.sum b/v2/go.sum index 607340d45..f0297f7f6 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -373,8 +373,9 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8= +github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= From 0fea763dc843181c5e3841482b853b7e467fa26b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 May 2022 05:23:12 +0000 Subject: [PATCH 15/32] chore(deps): bump github.com/xanzy/go-gitlab in /v2 Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.64.0 to 0.65.0. - [Release notes](https://github.com/xanzy/go-gitlab/releases) - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.64.0...v0.65.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3048b706b..aec7b6979 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -50,7 +50,7 @@ require ( github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 - github.com/xanzy/go-gitlab v0.64.0 + github.com/xanzy/go-gitlab v0.65.0 github.com/ysmood/gson v0.7.1 // indirect github.com/ysmood/leakless v0.7.0 // indirect go.uber.org/atomic v1.9.0 diff --git a/v2/go.sum b/v2/go.sum index 607340d45..cf59e10be 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -593,8 +593,8 @@ github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= -github.com/xanzy/go-gitlab v0.64.0 h1:rMgQdW9S1w3qvNAH2LYpFd2xh7KNLk+JWJd7sorNuTc= -github.com/xanzy/go-gitlab v0.64.0/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= +github.com/xanzy/go-gitlab v0.65.0 h1:9xSA9cRVhz3Z54JacIHdvWnNmNAoSz/BDnyMGOf3yIg= +github.com/xanzy/go-gitlab v0.65.0/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= From c9723a5a3fef6867f9f31c2e4f17af23d98820f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 May 2022 08:58:57 +0000 Subject: [PATCH 16/32] chore(deps): bump github.com/aws/aws-sdk-go in /v2 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.9 to 1.44.11. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.9...v1.44.11) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5e58526bf..905bced72 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -63,7 +63,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.8 +require github.com/aws/aws-sdk-go v1.44.11 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 diff --git a/v2/go.sum b/v2/go.sum index 76207f620..c1fd982a0 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,8 +86,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.8 h1:Fc8lFiP97MjB/BFaeZ7ixAjkqrNoic/qvlRqqJb5cxY= -github.com/aws/aws-sdk-go v1.44.8/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.11 h1:eIC59RrNY7xXYmGy/kKkLj4PGB325Jca22lcxZwbpBE= +github.com/aws/aws-sdk-go v1.44.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 2f1330345fb14f1fd2b73038a6a8bcbd0636f761 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 11 May 2022 12:30:39 +0200 Subject: [PATCH 17/32] Adding global SNI support for HTTP protocol via CLI (#1964) * Adding global SNI support via CLI * adding integration test * adding cli option to docs * reverting deleted test --- README.md | 3 ++- integration_tests/http/get-sni.yaml | 15 ++++++++++++ v2/cmd/integration-test/http.go | 23 +++++++++++++++++++ v2/cmd/nuclei/main.go | 1 + .../protocols/headless/engine/http_client.go | 4 ++++ .../http/httpclientpool/clientpool.go | 4 ++++ v2/pkg/protocols/websocket/websocket.go | 6 ++++- v2/pkg/types/types.go | 2 ++ 8 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 integration_tests/http/get-sni.yaml diff --git a/README.md b/README.md index 3837ef716..20fc7631d 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ nuclei -h This will display help for the tool. Here are all the switches it supports. -```yaml +```console Nuclei is a fast, template based vulnerability scanner focusing on extensive configurability, massive extensibility and ease of use. @@ -150,6 +150,7 @@ CONFIGURATIONS: -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts -ztls Use ztls library with autofallback to standard one for tls13 + -sni string Global SNI hostname (default: input domain name) INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) diff --git a/integration_tests/http/get-sni.yaml b/integration_tests/http/get-sni.yaml new file mode 100644 index 000000000..a934245e9 --- /dev/null +++ b/integration_tests/http/get-sni.yaml @@ -0,0 +1,15 @@ +id: basic-get + +info: + name: Basic GET Request with CLI SNI + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "test-ok" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c13c497d0..5dd1e5e61 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -48,6 +48,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/stop-at-first-match.yaml": &httpStopAtFirstMatch{}, "http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{}, "http/variables.yaml": &httpVariables{}, + "http/get-sni.yaml": &customCLISNI{}, } type httpInteractshRequest struct{} @@ -810,3 +811,25 @@ func (h *httpVariables) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type customCLISNI struct{} + +// Execute executes a test case and returns an error if occurred +func (h *customCLISNI) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + if r.TLS.ServerName == "test" { + _, _ = w.Write([]byte("test-ok")) + } else { + _, _ = w.Write([]byte("test-ko")) + } + }) + ts := httptest.NewTLSServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-sni", "test") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index a46720542..9bc5d1680 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -147,6 +147,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "Use ztls library with autofallback to standard one for tls13"), + flagSet.StringVar(&options.SNI, "sni", "", "Global SNI hostname (default: input domain name)"), ) createGroup(flagSet, "interactsh", "interactsh", diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 9b1c5b0df..86970267b 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -28,6 +28,10 @@ func newHttpClient(options *types.Options) (*http.Client, error) { InsecureSkipVerify: true, } + if options.SNI != "" { + tlsConfig.ServerName = options.SNI + } + // Add the client certificate authentication to the request if it's configured var err error tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 81f25885f..7a8836567 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -179,6 +179,10 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl InsecureSkipVerify: true, } + if options.SNI != "" { + tlsConfig.ServerName = options.SNI + } + // Add the client certificate authentication to the request if it's configured tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options) if err != nil { diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index b60b05e35..bf4eb1904 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -185,11 +185,15 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam } header.Set(key, string(finalData)) } + tlsConfig := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} + if requestOptions.Options.SNI != "" { + tlsConfig.ServerName = requestOptions.Options.SNI + } websocketDialer := ws.Dialer{ Header: ws.HandshakeHeaderHTTP(header), Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second, NetDial: request.dialer.Dial, - TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, + TLSConfig: tlsConfig, } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 3b0280e51..475129516 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -214,6 +214,8 @@ type Options struct { StoreResponseDir string // DisableRedirects disables following redirects for http request module DisableRedirects bool + // SNI custom hostname + SNI string } func (options *Options) AddVarPayload(key string, value interface{}) { From 0e9292d9fb6ee03dde2e3b44554bfb4a27e04621 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 May 2022 05:18:18 +0000 Subject: [PATCH 18/32] chore(deps): bump github.com/aws/aws-sdk-go in /v2 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.11 to 1.44.12. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.11...v1.44.12) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e1113f8ab..94d6ddc07 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -63,7 +63,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.11 +require github.com/aws/aws-sdk-go v1.44.12 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 diff --git a/v2/go.sum b/v2/go.sum index f53982aff..b290007ae 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,8 +86,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.11 h1:eIC59RrNY7xXYmGy/kKkLj4PGB325Jca22lcxZwbpBE= -github.com/aws/aws-sdk-go v1.44.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.12 h1:5f7ESFKQv5WHX8m37H2T8G+tc/rggy7sfdZ8ioqXFY8= +github.com/aws/aws-sdk-go v1.44.12/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From e67b306c2acf537cd18fd30f5147fbccb570a767 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 May 2022 05:18:24 +0000 Subject: [PATCH 19/32] chore(deps): bump github.com/spf13/cast from 1.4.1 to 1.5.0 in /v2 Bumps [github.com/spf13/cast](https://github.com/spf13/cast) from 1.4.1 to 1.5.0. - [Release notes](https://github.com/spf13/cast/releases) - [Commits](https://github.com/spf13/cast/compare/v1.4.1...v1.5.0) --- updated-dependencies: - dependency-name: github.com/spf13/cast dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- v2/go.mod | 3 +-- v2/go.sum | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e1113f8ab..4bca06630 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -45,7 +45,7 @@ require ( github.com/segmentio/ksuid v1.0.4 github.com/shirou/gopsutil/v3 v3.22.4 github.com/spaolacci/murmur3 v1.1.0 - github.com/spf13/cast v1.4.1 + github.com/spf13/cast v1.5.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 @@ -103,7 +103,6 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/frankban/quicktest v1.14.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index f53982aff..09d68921d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -151,8 +151,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= -github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -541,8 +541,9 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= From 6ca4374f917c5cdd8adfd4cacc46ad1ac6beb4c0 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 12 May 2022 05:10:14 -0500 Subject: [PATCH 20/32] sonar category: String literals should not be duplicated (#1944) * sonar category: String literals should not be duplicated * lint error fix * better naming conventions for constants * improved naming conventions and methods --- v2/pkg/core/workflow_execute.go | 12 +++-- .../protocols/common/interactsh/interactsh.go | 17 ++++--- .../protocols/headless/engine/page_actions.go | 49 +++++++++++-------- v2/pkg/protocols/headless/request.go | 8 +-- v2/pkg/protocols/http/build_request.go | 12 +++-- v2/pkg/protocols/http/signer/aws.go | 8 +-- v2/pkg/protocols/websocket/websocket.go | 17 ++++--- v2/pkg/reporting/format/format.go | 31 ++++++------ v2/pkg/reporting/reporting.go | 17 ++++--- v2/pkg/reporting/trackers/jira/jira.go | 21 ++++---- v2/pkg/types/resume.go | 4 +- 11 files changed, 117 insertions(+), 79 deletions(-) diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 9ee87df4a..aae9955ad 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -9,6 +9,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) +const workflowExecutionErrorMessageTemplate = "[%s] Could not execute workflow step: %s\n" + // executeWorkflow runs a workflow on an input and returns true or false func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { results := &atomic.Bool{} @@ -18,7 +20,7 @@ func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { swg.Add() func(template *workflows.WorkflowTemplate) { if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) + gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) } swg.Done() }(template) @@ -64,7 +66,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str if len(template.Executers) == 1 { mainErr = err } else { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) + gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) } continue } @@ -94,7 +96,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str go func(subtemplate *workflows.WorkflowTemplate) { if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", subtemplate.Template, err) + gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, subtemplate.Template, err) } swg.Done() }(subtemplate) @@ -105,7 +107,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str if len(template.Executers) == 1 { mainErr = err } else { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) + gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) } continue } @@ -118,7 +120,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str go func(template *workflows.WorkflowTemplate) { if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) + gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) } swg.Done() }(subtemplate) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 0daaebcf6..e6f9d2f69 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -56,6 +56,11 @@ var ( interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`) ) +const ( + stopAtFirstMatchAttribute = "stop-at-first-match" + templateIdAttribute = "template-id" +) + // Options contains configuration options for interactsh nuclei integration. type Options struct { // ServerURL is the URL of the interactsh server. @@ -178,8 +183,8 @@ func (c *Client) firstTimeInitializeClient() error { return } - if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok || c.options.StopAtFirstMatch { - gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent["template-id"].(string), request.Event.InternalEvent["host"].(string))) + if _, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { + gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent[templateIdAttribute].(string), request.Event.InternalEvent["host"].(string))) if gotItem != nil { return } @@ -219,8 +224,8 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched = true - if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok || c.options.StopAtFirstMatch { - c.matchedTemplates.Set(hash(data.Event.InternalEvent["template-id"].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) + if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { + c.matchedTemplates.Set(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) } } return true @@ -317,8 +322,8 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { for _, interactshURL := range interactshURLs { id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.hostname), ".") - if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok || c.options.StopAtFirstMatch { - gotItem := c.matchedTemplates.Get(hash(data.Event.InternalEvent["template-id"].(string), data.Event.InternalEvent["host"].(string))) + if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { + gotItem := c.matchedTemplates.Get(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string))) if gotItem != nil { break } diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 45388c682..cb5359091 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -18,6 +18,15 @@ import ( "github.com/segmentio/ksuid" ) +var ( + invalidArgumentsError = errors.New("invalid arguments provided") +) + +const ( + couldNotGetElementErrorMessage = "could not get element" + couldNotScrollErrorMessage = "could not scroll into view" +) + // ExecuteActions executes a list of actions on a page. func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]string, error) { var err error @@ -243,7 +252,7 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string /*TODO review func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL /*TODO review unused parameter*/) error { URL := p.getActionArgWithDefaultValues(action, "url") if URL == "" { - return errors.New("invalid arguments provided") + return invalidArgumentsError } // Handle the dynamic value substitution here. @@ -267,7 +276,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.UR func (p *Page) RunScript(action *Action, out map[string]string) error { code := p.getActionArgWithDefaultValues(action, "code") if code == "" { - return errors.New("invalid arguments provided") + return invalidArgumentsError } if p.getActionArgWithDefaultValues(action, "hook") == "true" { if _, err := p.page.EvalOnNewDocument(code); err != nil { @@ -288,10 +297,10 @@ func (p *Page) RunScript(action *Action, out map[string]string) error { func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } if err = element.Click(proto.InputMouseButtonLeft); err != nil { return errors.Wrap(err, "could not click element") @@ -308,10 +317,10 @@ func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review u func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } if err = element.Click(proto.InputMouseButtonRight); err != nil { return errors.Wrap(err, "could not right click element") @@ -349,14 +358,14 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error { func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { - return errors.New("invalid arguments provided") + return invalidArgumentsError } element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } if err = element.Input(value); err != nil { return errors.Wrap(err, "could not input element") @@ -368,14 +377,14 @@ func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unu func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { - return errors.New("invalid arguments provided") + return invalidArgumentsError } element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } t, err := time.Parse(time.RFC3339, value) if err != nil { @@ -391,14 +400,14 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { - return errors.New("invalid arguments provided") + return invalidArgumentsError } element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } selectedBool := false @@ -430,7 +439,7 @@ func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused func (p *Page) GetResource(act *Action, out map[string]string) error { element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } resource, err := element.Resource() if err != nil { @@ -446,10 +455,10 @@ func (p *Page) GetResource(act *Action, out map[string]string) error { func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error { element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } value := p.getActionArgWithDefaultValues(act, "value") filesPaths := strings.Split(value, ",") @@ -463,10 +472,10 @@ func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unuse func (p *Page) ExtractElement(act *Action, out map[string]string) error { element, err := p.pageElementBy(act.Data) if err != nil { - return errors.Wrap(err, "could not get element") + return errors.Wrap(err, couldNotGetElementErrorMessage) } if err = element.ScrollIntoView(); err != nil { - return errors.Wrap(err, "could not scroll into view") + return errors.Wrap(err, couldNotScrollErrorMessage) } switch p.getActionArgWithDefaultValues(act, "target") { case "attribute": diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index fbc46c65a..393acc5e3 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -19,6 +19,8 @@ import ( var _ protocols.Request = &Request{} +const couldGetHtmlElementErrorMessage = "could get html element" + // Type returns the type of the protocol request func (request *Request) Type() templateTypes.ProtocolType { return templateTypes.HeadlessProtocol @@ -60,7 +62,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map if err != nil { request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could get html element") + return errors.Wrap(err, couldGetHtmlElementErrorMessage) } defer instance.Close() @@ -70,14 +72,14 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map if err != nil { request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could get html element") + return errors.Wrap(err, couldGetHtmlElementErrorMessage) } timeout := time.Duration(request.options.Options.PageTimeout) * time.Second out, page, err := instance.Run(parsedURL, request.Steps, payloads, timeout) if err != nil { request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could get html element") + return errors.Wrap(err, couldGetHtmlElementErrorMessage) } defer page.Close() diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 2fcc062cb..efcebd467 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -32,6 +32,8 @@ var ( urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) ) +const evaluateHelperExpressionErrorMessage = "could not evaluate helper expressions" + // generatedRequest is a single generated request wrapped for a template request type generatedRequest struct { original *Request @@ -206,12 +208,12 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st var err error data, err = expressions.Evaluate(data, finalValues) if err != nil { - return nil, errors.Wrap(err, "could not evaluate helper expressions") + return nil, errors.Wrap(err, evaluateHelperExpressionErrorMessage) } method, err := expressions.Evaluate(r.request.Method.String(), finalValues) if err != nil { - return nil, errors.Wrap(err, "could not evaluate helper expressions") + return nil, errors.Wrap(err, evaluateHelperExpressionErrorMessage) } // Build a request on the specified URL @@ -245,7 +247,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest var err error rawRequest, err = expressions.Evaluate(rawRequest, finalValues) if err != nil { - return nil, errors.Wrap(err, "could not evaluate helper expressions") + return nil, errors.Wrap(err, evaluateHelperExpressionErrorMessage) } rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe) if err != nil { @@ -302,7 +304,7 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte } value, err := expressions.Evaluate(value, values) if err != nil { - return nil, errors.Wrap(err, "could not evaluate helper expressions") + return nil, errors.Wrap(err, evaluateHelperExpressionErrorMessage) } req.Header[header] = []string{value} if header == "Host" { @@ -323,7 +325,7 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte } body, err := expressions.Evaluate(body, values) if err != nil { - return nil, errors.Wrap(err, "could not evaluate helper expressions") + return nil, errors.Wrap(err, evaluateHelperExpressionErrorMessage) } req.Body = ioutil.NopCloser(strings.NewReader(body)) } diff --git a/v2/pkg/protocols/http/signer/aws.go b/v2/pkg/protocols/http/signer/aws.go index 967bf8c42..0e8587546 100644 --- a/v2/pkg/protocols/http/signer/aws.go +++ b/v2/pkg/protocols/http/signer/aws.go @@ -22,6 +22,8 @@ type AwsSignerArgs struct { AwsSecretToken string } +var credentialCreationError = errors.New("couldn't create the credentials structure") + func (awsSignerArgs AwsSignerArgs) Validate() error { if awsSignerArgs.AwsId == "" { return errors.New("empty id") @@ -56,7 +58,7 @@ func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) { } creds := credentials.NewStaticCredentials(args.AwsId, args.AwsSecretToken, "") if creds == nil { - return nil, errors.New("couldn't create the credentials structure") + return nil, credentialCreationError } signer := v4.NewSigner(creds) return &AwsSigner{creds: creds, signer: signer}, nil @@ -65,7 +67,7 @@ func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) { func NewAwsSignerFromEnv() (*AwsSigner, error) { creds := credentials.NewEnvCredentials() if creds == nil { - return nil, errors.New("couldn't create the credentials structure") + return nil, credentialCreationError } signer := v4.NewSigner(creds) return &AwsSigner{creds: creds, signer: signer}, nil @@ -74,7 +76,7 @@ func NewAwsSignerFromEnv() (*AwsSigner, error) { func NewAwsSignerFromFile() (*AwsSigner, error) { creds := credentials.NewSharedCredentials("", "") if creds == nil { - return nil, errors.New("couldn't create the credentials structure") + return nil, credentialCreationError } signer := v4.NewSigner(creds) return &AwsSigner{creds: creds, signer: signer}, nil diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index bf4eb1904..c321dc14e 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -86,6 +86,11 @@ type Input struct { Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` } +const ( + parseUrlErrorMessage = "could not parse input url" + evaluateTemplateExpressionErrorMessage = "could not evaluate template expressions" +) + // Compile compiles the request generators preparing any requests possible. func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.options = options @@ -164,7 +169,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam } parsed, err := url.Parse(input) if err != nil { - return errors.Wrap(err, "could not parse input url") + return errors.Wrap(err, parseUrlErrorMessage) } payloadValues["Hostname"] = parsed.Host payloadValues["Host"] = parsed.Hostname() @@ -181,7 +186,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam if dataErr != nil { requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) requestOptions.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") + return errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage) } header.Set(key, string(finalData)) } @@ -200,7 +205,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam if dataErr != nil { requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) requestOptions.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") + return errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage) } addressToDial := string(finalAddress) @@ -208,7 +213,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam if err != nil { requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not parse input url") + return errors.Wrap(err, parseUrlErrorMessage) } parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) addressToDial = parsedAddress.String() @@ -283,7 +288,7 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map if dataErr != nil { requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) requestOptions.Progress.IncrementFailedRequestsBy(1) - return nil, "", errors.Wrap(dataErr, "could not evaluate template expressions") + return nil, "", errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage) } reqBuilder.WriteString(string(finalData)) @@ -327,7 +332,7 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map func getAddress(toTest string) (string, error) { parsed, err := url.Parse(toTest) if err != nil { - return "", errors.Wrap(err, "could not parse input url") + return "", errors.Wrap(err, parseUrlErrorMessage) } scheme := strings.ToLower(parsed.Scheme) diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index 0e18f1ac1..04432ef93 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) + // Summary returns a formatted built one line summary of the event func Summary(event *output.ResultEvent) string { template := GetMatchedTemplate(event) @@ -53,20 +54,18 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the builder.WriteString(ToMarkdownTableString(&event.Info)) if event.Request != "" { - builder.WriteString("\n**Request**\n\n```http\n") - builder.WriteString(types.ToHexOrString(event.Request)) - builder.WriteString("\n```\n") + builder.WriteString(createMarkdownCodeBlock("Request", types.ToHexOrString(event.Request), "http")) } if event.Response != "" { - builder.WriteString("\n**Response**\n\n```http\n") + var responseString string // If the response is larger than 5 kb, truncate it before writing. if len(event.Response) > 5*1024 { - builder.WriteString(event.Response[:5*1024]) - builder.WriteString(".... Truncated ....") + responseString = (event.Response[:5*1024]) + responseString += ".... Truncated ...." } else { - builder.WriteString(event.Response) + responseString = event.Response } - builder.WriteString("\n```\n") + builder.WriteString(createMarkdownCodeBlock("Response", responseString, "http")) } if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { @@ -107,14 +106,10 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the builder.WriteString(event.Interaction.UniqueID) if event.Interaction.RawRequest != "" { - builder.WriteString("\n\n**Interaction Request**\n\n```\n") - builder.WriteString(event.Interaction.RawRequest) - builder.WriteString("\n```\n") + builder.WriteString(createMarkdownCodeBlock("Interaction Request", event.Interaction.RawRequest, "")) } if event.Interaction.RawResponse != "" { - builder.WriteString("\n**Interaction Response**\n\n```\n") - builder.WriteString(event.Interaction.RawResponse) - builder.WriteString("\n```\n") + builder.WriteString(createMarkdownCodeBlock("Interaction Response", event.Interaction.RawResponse, "")) } } @@ -237,3 +232,11 @@ func generateCVECWEIDLinksFromClassification(classification *model.Classificatio fields.Set("CVE-ID", strings.Join(cveIDs, ",")) } } + +func createMarkdownCodeBlock(title string, content string, language string) string { + return "\n" + createBoldMarkdown(title) + "\n```" + language + "\n" + content + "\n```\n" +} + +func createBoldMarkdown(value string) string { + return "**" + value + "**" +} diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index d30448e86..075315951 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -47,6 +47,11 @@ type Filter struct { Tags stringslice.StringSlice `yaml:"tags"` } +const ( + reportingClientCreationErrorMessage = "could not create reporting client" + exportClientCreationErrorMessage = "could not create exporting client" +) + // GetMatch returns true if a filter matches result event func (filter *Filter) GetMatch(event *output.ResultEvent) bool { return isSeverityMatch(event, filter) && isTagMatch(event, filter) // TODO revisit this @@ -113,7 +118,7 @@ func New(options *Options, db string) (*Client, error) { options.GitHub.HttpClient = options.HttpClient tracker, err := github.New(options.GitHub) if err != nil { - return nil, errors.Wrap(err, "could not create reporting client") + return nil, errors.Wrap(err, reportingClientCreationErrorMessage) } client.trackers = append(client.trackers, tracker) } @@ -121,7 +126,7 @@ func New(options *Options, db string) (*Client, error) { options.GitLab.HttpClient = options.HttpClient tracker, err := gitlab.New(options.GitLab) if err != nil { - return nil, errors.Wrap(err, "could not create reporting client") + return nil, errors.Wrap(err, reportingClientCreationErrorMessage) } client.trackers = append(client.trackers, tracker) } @@ -129,21 +134,21 @@ func New(options *Options, db string) (*Client, error) { options.Jira.HttpClient = options.HttpClient tracker, err := jira.New(options.Jira) if err != nil { - return nil, errors.Wrap(err, "could not create reporting client") + return nil, errors.Wrap(err, reportingClientCreationErrorMessage) } client.trackers = append(client.trackers, tracker) } if options.MarkdownExporter != nil { exporter, err := markdown.New(options.MarkdownExporter) if err != nil { - return nil, errors.Wrap(err, "could not create exporting client") + return nil, errors.Wrap(err, exportClientCreationErrorMessage) } client.exporters = append(client.exporters, exporter) } if options.SarifExporter != nil { exporter, err := sarif.New(options.SarifExporter) if err != nil { - return nil, errors.Wrap(err, "could not create exporting client") + return nil, errors.Wrap(err, exportClientCreationErrorMessage) } client.exporters = append(client.exporters, exporter) } @@ -151,7 +156,7 @@ func New(options *Options, db string) (*Client, error) { options.ElasticsearchExporter.HttpClient = options.HttpClient exporter, err := es.New(options.ElasticsearchExporter) if err != nil { - return nil, errors.Wrap(err, "could not create exporting client") + return nil, errors.Wrap(err, exportClientCreationErrorMessage) } client.exporters = append(client.exporters, exporter) } diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index fb2203278..715803d7d 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -46,6 +46,7 @@ type Options struct { HttpClient *retryablehttp.Client } + // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { username := options.Email @@ -185,9 +186,7 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n") builder.WriteString(format.ToMarkdownTableString(&event.Info)) - builder.WriteString("\n*Request*\n\n{code}\n") - builder.WriteString(event.Request) - builder.WriteString("\n{code}\n") + builder.WriteString(createMarkdownCodeBlock("Request", event.Request)) builder.WriteString("\n*Response*\n\n{code}\n") // If the response is larger than 5 kb, truncate it before writing. @@ -236,14 +235,10 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th builder.WriteString(event.Interaction.UniqueID) if event.Interaction.RawRequest != "" { - builder.WriteString("\n\n*Interaction Request*\n\n{code}\n") - builder.WriteString(event.Interaction.RawRequest) - builder.WriteString("\n{code}\n") + builder.WriteString(createMarkdownCodeBlock("Interaction Request", event.Interaction.RawRequest)) } if event.Interaction.RawResponse != "" { - builder.WriteString("\n*Interaction Response*\n\n{code}\n") - builder.WriteString(event.Interaction.RawResponse) - builder.WriteString("\n{code}\n") + builder.WriteString(createMarkdownCodeBlock("Interaction Response", event.Interaction.RawResponse)) } } @@ -271,3 +266,11 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th data := builder.String() return data } + +func createMarkdownCodeBlock(title string, content string) string { + return "\n" + createBoldMarkdown(title) + "\n" + content + "*\n\n{code}" +} + +func createBoldMarkdown(value string) string { + return "*" + value + "*\n\n{code}" +} diff --git a/v2/pkg/types/resume.go b/v2/pkg/types/resume.go index 06ce21e02..f319271fa 100644 --- a/v2/pkg/types/resume.go +++ b/v2/pkg/types/resume.go @@ -16,9 +16,9 @@ const DefaultResumeFileName = "resume-%s.cfg" func DefaultResumeFilePath() string { configDir, err := config.GetConfigDir() if err != nil { - return fmt.Sprintf("resume-%s.cfg", xid.New().String()) + return fmt.Sprintf(DefaultResumeFileName, xid.New().String()) } - resumeFile := filepath.Join(configDir, fmt.Sprintf("resume-%s.cfg", xid.New().String())) + resumeFile := filepath.Join(configDir, fmt.Sprintf(DefaultResumeFileName, xid.New().String())) return resumeFile } From 39c7317ec374038dc8ccfb33ef94a85cb81e81ea Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 12 May 2022 13:13:56 +0200 Subject: [PATCH 21/32] Adding SNI override via request annotations (#1970) * Adding SNI override via request annotations * adding cli flag priority --- integration_tests/http/get-override-sni.yaml | 18 ++++++++++ v2/cmd/integration-test/http.go | 23 ++++++++++++ v2/go.mod | 2 +- v2/go.sum | 2 ++ .../protocols/common/protocolstate/state.go | 1 + .../protocols/headless/engine/http_client.go | 1 + v2/pkg/protocols/http/build_request.go | 5 ++- .../http/httpclientpool/clientpool.go | 1 + v2/pkg/protocols/http/request_annotations.go | 35 +++++++++++++++++-- .../reporting/exporters/es/elasticsearch.go | 1 + 10 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 integration_tests/http/get-override-sni.yaml diff --git a/integration_tests/http/get-override-sni.yaml b/integration_tests/http/get-override-sni.yaml new file mode 100644 index 000000000..61239ee5c --- /dev/null +++ b/integration_tests/http/get-override-sni.yaml @@ -0,0 +1,18 @@ +id: basic-raw-http-example + +info: + name: Test RAW GET Template + author: pdteam + severity: info + +requests: + - raw: + - | + @tls-sni:request.host + GET / HTTP/1.1 + Host: test + + matchers: + - type: word + words: + - "test-ok" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 5dd1e5e61..205dcaf20 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -48,6 +48,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/stop-at-first-match.yaml": &httpStopAtFirstMatch{}, "http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{}, "http/variables.yaml": &httpVariables{}, + "http/get-override-sni.yaml": &httpSniAnnotation{}, "http/get-sni.yaml": &customCLISNI{}, } @@ -833,3 +834,25 @@ func (h *customCLISNI) Execute(filePath string) error { } return expectResultsCount(results, 1) } + +type httpSniAnnotation struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpSniAnnotation) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + if r.TLS.ServerName == "test" { + _, _ = w.Write([]byte("test-ok")) + } else { + _, _ = w.Write([]byte("test-ko")) + } + }) + ts := httptest.NewTLSServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/v2/go.mod b/v2/go.mod index dbdaeecde..4df1c0744 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -27,7 +27,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v1.0.0 - github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20220427234316-40b2541a84b8 github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 diff --git a/v2/go.sum b/v2/go.sum index bb355c305..cbe06a7bc 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -437,6 +437,8 @@ github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 h1:TUsZiwez9uFmph1hlTsiH7rdB+wi4524+lMuV2z6FaM= github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47/go.mod h1:GbQvP1ezGlQn0af3lVcl08b5eRQu960T7A9pwazybSo= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 h1:1hzvl0lsWpvQ8nn1s9YMyBjO13/Z+f/T4W2jroOohfo= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index a2fc4db8c..604ae732f 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -21,6 +21,7 @@ func Init(options *types.Options) error { } opts.WithDialerHistory = true opts.WithZTLS = options.ZTLS + opts.SNIName = options.SNI dialer, err := fastdialer.NewDialer(opts) if err != nil { return errors.Wrap(err, "could not create dialer") diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 86970267b..2ea0c25fc 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -41,6 +41,7 @@ func newHttpClient(options *types.Options) (*http.Client, error) { transport := &http.Transport{ DialContext: dialer.Dial, + DialTLSContext: dialer.DialTLS, MaxIdleConns: 500, MaxIdleConnsPerHost: 500, MaxConnsPerHost: 500, diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index efcebd467..284e47fbc 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -290,7 +290,10 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest return nil, err } - parseAnnotations(rawRequest, req) + if reqWithAnnotations, hasAnnotations := parseAnnotations(rawRequest, req); hasAnnotations { + req = reqWithAnnotations + request = request.WithContext(req.Context()) + } return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 7a8836567..c53c21b61 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -191,6 +191,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl transport := &http.Transport{ DialContext: Dialer.Dial, + DialTLSContext: Dialer.DialTLS, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, MaxConnsPerHost: maxConnsPerHost, diff --git a/v2/pkg/protocols/http/request_annotations.go b/v2/pkg/protocols/http/request_annotations.go index 34464fd90..8228fec44 100644 --- a/v2/pkg/protocols/http/request_annotations.go +++ b/v2/pkg/protocols/http/request_annotations.go @@ -1,22 +1,33 @@ package http import ( + "context" "net" "net/http" "regexp" "strings" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" ) -// @Host:target overrides the input target with the annotated one (similar to self-contained requests) -var reHostAnnotation = regexp.MustCompile(`(?m)^@Host:\s*(.+)\s*$`) +var ( + // @Host:target overrides the input target with the annotated one (similar to self-contained requests) + reHostAnnotation = regexp.MustCompile(`(?m)^@Host:\s*(.+)\s*$`) + // @tls-sni:target overrides the input target with the annotated one + // special values: + // request.host: takes the value from the host header + // target: overiddes with the specific value + reSniAnnotation = regexp.MustCompile(`(?m)^@tls-sni:\s*(.+)\s*$`) +) // parseAnnotations and override requests settings -func parseAnnotations(rawRequest string, request *http.Request) { +func parseAnnotations(rawRequest string, request *http.Request) (*http.Request, bool) { // parse request for known ovverride annotations + var modified bool + // @Host:target if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 { value := strings.TrimSpace(hosts[1]) // handle scheme @@ -39,7 +50,25 @@ func parseAnnotations(rawRequest string, request *http.Request) { } request.URL.Host = hostPort } + modified = true } + + // @tls-sni:target + if hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 { + value := strings.TrimSpace(hosts[1]) + value = stringsutil.TrimPrefixAny(value, "http://", "https://") + if idxForwardSlash := strings.Index(value, "/"); idxForwardSlash >= 0 { + value = value[:idxForwardSlash] + } + + if stringsutil.EqualFoldAny(value, "request.host") { + value = request.Host + } + ctx := context.WithValue(request.Context(), fastdialer.SniName, value) + request = request.Clone(ctx) + modified = true + } + return request, modified } func isHostPort(value string) bool { diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 96adac1b2..545ab9487 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -63,6 +63,7 @@ func New(option *Options) (*Exporter, error) { MaxIdleConns: 10, MaxIdleConnsPerHost: 10, DialContext: protocolstate.Dialer.Dial, + DialTLSContext: protocolstate.Dialer.DialTLS, TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, }, } From 4a5039cc75e6840927e35e11ffa9016d3d2f6b66 Mon Sep 17 00:00:00 2001 From: Owen Rumney Date: Thu, 12 May 2022 23:18:14 +0100 Subject: [PATCH 22/32] chore(deps): Bump sarif to v2 (#1930) Signed-off-by: Owen Rumney Co-authored-by: mzack --- v2/go.mod | 3 +-- v2/go.sum | 3 ++- v2/pkg/reporting/exporters/sarif/sarif.go | 21 ++++++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 4df1c0744..7273e3ac6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -23,7 +23,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/miekg/dns v1.1.49 github.com/olekukonko/tablewriter v0.0.5 - github.com/owenrumney/go-sarif v1.1.1 + github.com/owenrumney/go-sarif/v2 v2.1.1 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v1.0.0 @@ -153,7 +153,6 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - github.com/zclconf/go-cty v1.10.0 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/zap v1.21.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index cbe06a7bc..122776d5e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -416,6 +416,8 @@ github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7/go.mod h1:inRbqVxN github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8= github.com/owenrumney/go-sarif v1.1.1 h1:QNObu6YX1igyFKhdzd7vgzmw7XsWN3/6NMGuDzBgXmE= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= +github.com/owenrumney/go-sarif/v2 v2.1.1 h1:JVUO0cEhG8bvEWIxsRmURY4u7wBZUTgdh4zikkkiPM8= +github.com/owenrumney/go-sarif/v2 v2.1.1/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -630,7 +632,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= -github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 66c8740df..066a0b202 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/owenrumney/go-sarif/sarif" + "github.com/owenrumney/go-sarif/v2/sarif" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -44,7 +44,7 @@ func New(options *Options) (*Exporter, error) { return nil, errors.Wrap(err, "could not template path") } - run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei") + run := sarif.NewRunWithInformationURI("nuclei", "https://github.com/projectdiscovery/nuclei") return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil } @@ -56,9 +56,6 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { _, _ = h.Write([]byte(event.Host)) templateID := event.TemplateID + "-" + hex.EncodeToString(h.Sum(nil)) - fullDescription := format.MarkdownDescription(event) - sarifSeverity := getSarifSeverity(event) - var ruleName string if utils.IsNotBlank(event.Info.Name) { ruleName = event.Info.Name @@ -81,25 +78,27 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { _ = exporter.run.AddRule(templateID). WithDescription(ruleName). - WithHelp(fullDescription). + WithHelp(sarif.NewMarkdownMultiformatMessageString(format.MarkdownDescription(event))). WithHelpURI(templateURL). WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)) - result := exporter.run.AddResult(templateID). - WithMessage(sarif.NewMessage().WithText(event.Host)). - WithLevel(sarifSeverity) + result := sarif.NewRuleResult(templateID). + WithMessage(sarif.NewTextMessage(event.Host)). + WithLevel(getSarifSeverity(event)) + + exporter.run.AddResult(result) // Also write file match metadata to file if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) { for file, line := range event.FileToIndexPosition { - result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation( + result.AddLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation( sarif.NewPhysicalLocation(). WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(line).WithEndLine(line).WithEndColumn(32)), )) } } else { - result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( + result.AddLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( sarif.NewPhysicalLocation(). WithArtifactLocation(sarif.NewArtifactLocation().WithUri("README.md")). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), From 3d03be8183447061d28f25fe0fda61083b11996a Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 14 May 2022 14:06:48 +0200 Subject: [PATCH 23/32] Appending new references to existing ones (#1881) * Adding support to append new references * go modules * improving reference detection * replacing raw string manipulation with yaml lib * standardizing description appearance * omitting empty fields * adding missing remediation and metadata * misc update * Limit max references to 5 + fixed variables section in templates Co-authored-by: sandeep Co-authored-by: Ice3man --- v2/cmd/cve-annotate/main.go | 105 ++++++++++++++++++++++++++++++------ v2/go.mod | 4 +- v2/go.sum | 2 + 3 files changed, 92 insertions(+), 19 deletions(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index ccbdb9a25..4117fa51c 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "flag" "fmt" "io/ioutil" @@ -10,10 +11,17 @@ import ( "strings" "github.com/projectdiscovery/nvd" + "github.com/projectdiscovery/sliceutil" + "github.com/projectdiscovery/stringsutil" + "gopkg.in/yaml.v3" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) +const ( + yamlIndentSpaces = 2 +) + var ( input = flag.String("i", "", "Templates to annotate") templateDir = flag.String("d", "", "Custom template directory for update") @@ -63,6 +71,8 @@ var ( severityRegex = regexp.MustCompile(`severity: ([a-z]+)`) ) +const maxReferenceCount = 5 + func getCVEData(client *nvd.Client, filePath, data string) { matches := idRegex.FindAllStringSubmatch(data, 1) if len(matches) == 0 { @@ -76,10 +86,6 @@ func getCVEData(client *nvd.Client, filePath, data string) { } severityValue := severityMatches[0][1] - // Skip if there's classification data already - if strings.Contains(data, "classification:") { - return - } cveItem, err := client.FetchCVE(cveName) if err != nil { log.Printf("Could not fetch cve %s: %s\n", cveName, err) @@ -98,43 +104,84 @@ func getCVEData(client *nvd.Client, filePath, data string) { infoBlockIndexData := data[strings.Index(data, "info:"):] requestsIndex := strings.Index(infoBlockIndexData, "requests:") networkIndex := strings.Index(infoBlockIndexData, "network:") - if requestsIndex == -1 && networkIndex == -1 { + variablesIndex := strings.Index(infoBlockIndexData, "variables:") + if requestsIndex == -1 && networkIndex == -1 && variablesIndex == -1 { return } if networkIndex != -1 { requestsIndex = networkIndex } + if variablesIndex != -1 { + requestsIndex = variablesIndex + } infoBlockData := infoBlockIndexData[:requestsIndex] infoBlockClean := strings.TrimRight(infoBlockData, "\n") - newInfoBlock := infoBlockClean - var changed bool + infoBlock := InfoBlock{} + err = yaml.Unmarshal([]byte(data), &infoBlock) + if err != nil { + log.Printf("Could not unmarshal info block: %s\n", err) + } + var changed bool if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" { changed = true - newInfoBlock = strings.ReplaceAll(newInfoBlock, severityMatches[0][0], "severity: "+newSeverity) + infoBlock.Info.Severity = newSeverity fmt.Printf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore) } - if !strings.Contains(infoBlockClean, "classification") && (cvssScore != 0 && cvssMetrics != "") { + isCvssEmpty := cvssScore == 0 || cvssMetrics == "" + hasCvssChanged := infoBlock.Info.Classification.CvssScore != cvssScore || cvssMetrics != infoBlock.Info.Classification.CvssMetrics + if !isCvssEmpty && hasCvssChanged { changed = true - newInfoBlock += fmt.Sprintf("\n classification:\n cvss-metrics: %s\n cvss-score: %.2f\n cve-id: %s", cvssMetrics, cvssScore, cveName) + infoBlock.Info.Classification.CvssMetrics = cvssMetrics + infoBlock.Info.Classification.CvssScore = cvssScore + infoBlock.Info.Classification.CveId = cveName if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") { - newInfoBlock += fmt.Sprintf("\n cwe-id: %s", strings.Join(cweID, ",")) + infoBlock.Info.Classification.CweId = strings.Join(cweID, ",") } } // If there is no description field, fill the description from CVE information - if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 { + hasDescriptionData := len(cveItem.CVE.Description.DescriptionData) > 0 + isDescriptionEmpty := infoBlock.Info.Description == "" + if isDescriptionEmpty && hasDescriptionData { changed = true - newInfoBlock += fmt.Sprintf("\n description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value)) + // removes all new lines + description := stringsutil.ReplaceAny(cveItem.CVE.Description.DescriptionData[0].Value, "", "\n", "\\", "'", "\t") + description += "\n" + infoBlock.Info.Description = description } - if !strings.Contains(infoBlockClean, "reference:") && len(cveItem.CVE.References.ReferenceData) > 0 { + + // we are unmarshaling info block to have valid data + var referenceDataURLs []string + for _, reference := range cveItem.CVE.References.ReferenceData { + referenceDataURLs = append(referenceDataURLs, reference.URL) + } + hasReferenceData := len(cveItem.CVE.References.ReferenceData) > 0 + areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs) + referencesCount := len(infoBlock.Info.Reference) + if hasReferenceData && !areCveReferencesContained { changed = true - newInfoBlock += "\n reference:" for _, reference := range cveItem.CVE.References.ReferenceData { - newInfoBlock += fmt.Sprintf("\n - %s", reference.URL) + referencesCount++ + if referencesCount >= maxReferenceCount { + break + } + infoBlock.Info.Reference = append(infoBlock.Info.Reference, reference.URL) } + infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference)) } - newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock) + + var newInfoBlock bytes.Buffer + yamlEncoder := yaml.NewEncoder(&newInfoBlock) + yamlEncoder.SetIndent(yamlIndentSpaces) + err = yamlEncoder.Encode(infoBlock) + if err != nil { + log.Printf("Could not marshal info block: %s\n", err) + return + } + newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") + + newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlockData) if changed { _ = ioutil.WriteFile(filePath, []byte(newTemplate), 0644) fmt.Printf("Wrote updated template to %s\n", filePath) @@ -161,3 +208,27 @@ func isSeverityMatchingCvssScore(severity string, score float64) string { } return "" } + +// Cloning struct from nuclei as we don't want any validation +type InfoBlock struct { + Info TemplateInfo `yaml:"info"` +} + +type TemplateClassification struct { + CvssMetrics string `yaml:"cvss-metrics,omitempty"` + CvssScore float64 `yaml:"cvss-score,omitempty"` + CveId string `yaml:"cve-id,omitempty"` + CweId string `yaml:"cwe-id,omitempty"` +} + +type TemplateInfo struct { + Name string `yaml:"name"` + Author string `yaml:"author"` + Severity string `yaml:"severity"` + Description string `yaml:"description,omitempty"` + Reference []string `yaml:"reference,omitempty"` + Remediation string `yaml:"remediation,omitempty"` + Classification TemplateClassification `yaml:"classification,omitempty"` + Metadata map[string]string `yaml:"metadata,omitempty"` + Tags string `yaml:"tags,omitempty"` +} diff --git a/v2/go.mod b/v2/go.mod index 7273e3ac6..8f61025fa 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -76,11 +76,12 @@ require ( github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d - github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d + github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9 github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 github.com/projectdiscovery/wappalyzergo v0.0.41 github.com/stretchr/testify v1.7.1 github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) require ( @@ -167,5 +168,4 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 122776d5e..a6084d621 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -495,6 +495,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26 github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk= github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= +github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9 h1:dQO4FNxOhkZqZHW5nUh6f1NXIRUyoZ57bXETXjsyfNk= +github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9/go.mod h1:crCchiT/KYrAlEY8JbuMGE6XKym61mhAd1E7REq2VwQ= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= From e58ec777536c255e9cc037bc3837b210b14a1397 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 12:30:12 +0530 Subject: [PATCH 24/32] chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 (#2002) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.41 to 0.0.42. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.41...v0.0.42) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 8f61025fa..ecc8d7359 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9 github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 - github.com/projectdiscovery/wappalyzergo v0.0.41 + github.com/projectdiscovery/wappalyzergo v0.0.42 github.com/stretchr/testify v1.7.1 github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b diff --git a/v2/go.sum b/v2/go.sum index a6084d621..01765a7f7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -437,8 +437,6 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod github.com/projectdiscovery/cryptoutil v1.0.0 h1:5rQfnWDthJ5ZFcqze+rmT1N7l1HJQ6EB26MrjaYB7I0= github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= -github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 h1:TUsZiwez9uFmph1hlTsiH7rdB+wi4524+lMuV2z6FaM= -github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47/go.mod h1:GbQvP1ezGlQn0af3lVcl08b5eRQu960T7A9pwazybSo= github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 h1:1hzvl0lsWpvQ8nn1s9YMyBjO13/Z+f/T4W2jroOohfo= github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= @@ -493,8 +491,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= -github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk= -github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9 h1:dQO4FNxOhkZqZHW5nUh6f1NXIRUyoZ57bXETXjsyfNk= github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9/go.mod h1:crCchiT/KYrAlEY8JbuMGE6XKym61mhAd1E7REq2VwQ= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= @@ -506,8 +502,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.41 h1:t6zsfLiMFyjsIv0BYYKSDSxDbsG+FEuO1/TjE/eD55o= -github.com/projectdiscovery/wappalyzergo v0.0.41/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.42 h1:4DuJVpgesHSnUNVG+l62QIYEmcMXu3PLyhS1Wp75c30= +github.com/projectdiscovery/wappalyzergo v0.0.42/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= From 564b810af5cba00e1ce37408a0ccdf7d94f9e413 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 12:35:38 +0530 Subject: [PATCH 25/32] chore(deps): bump golang from 1.18.1-alpine to 1.18.2-alpine (#1999) Bumps golang from 1.18.1-alpine to 1.18.2-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 920ae644d..5bee5f5cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.1-alpine as build-env +FROM golang:1.18.2-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.15.4 From d67e75d3cd7c435a840daf6d82cfe13f9520a1c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 12:35:49 +0530 Subject: [PATCH 26/32] chore(deps): bump golangci/golangci-lint-action from 3.1.0 to 3.2.0 (#2000) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 735583eb3..3b6126e92 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -17,7 +17,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.1.0 + uses: golangci/golangci-lint-action@v3.2.0 with: version: latest args: --timeout 5m From 27bdd8c642752abf6222178109d249aa910c54cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 12:36:02 +0530 Subject: [PATCH 27/32] chore(deps): bump github.com/aws/aws-sdk-go in /v2 (#2001) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.12 to 1.44.14. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.12...v1.44.14) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index ecc8d7359..798fb142f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -63,7 +63,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.12 +require github.com/aws/aws-sdk-go v1.44.14 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 diff --git a/v2/go.sum b/v2/go.sum index 01765a7f7..0281b593b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,8 +86,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.12 h1:5f7ESFKQv5WHX8m37H2T8G+tc/rggy7sfdZ8ioqXFY8= -github.com/aws/aws-sdk-go v1.44.12/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.14 h1:qd7/muV1rElsbvkK9D1nHUzBoDlEw2etfeo4IE82eSQ= +github.com/aws/aws-sdk-go v1.44.14/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 062fe6d01ee4b16d28716618b91c5c699bd5b794 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 17 May 2022 14:33:17 +0530 Subject: [PATCH 28/32] Added urldns gadget from ysoserial (#1985) --- .../common/helpers/deserialization/java.go | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/common/helpers/deserialization/java.go b/v2/pkg/protocols/common/helpers/deserialization/java.go index 0ea107c37..3a5149463 100644 --- a/v2/pkg/protocols/common/helpers/deserialization/java.go +++ b/v2/pkg/protocols/common/helpers/deserialization/java.go @@ -5,6 +5,7 @@ import ( "compress/gzip" "encoding/base64" "encoding/hex" + "net/url" "strings" ) @@ -119,17 +120,32 @@ func generateGroovy1Payload(cmd string) []byte { } // generateDNSPayload generates DNS interaction deserialization paylaod for a DNS Name. -// Based on Gabriel Lawrence gadget -func generateDNSPayload(url string) []byte { +// Taken from ysoserial DNS gadget. +func generateDNSPayload(URL string) []byte { + parsed, err := url.Parse(URL) + if err != nil { + return nil + } buffer := &bytes.Buffer{} + hostname := parsed.Hostname() prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C770800000010000000017372000C6A6176612E6E65742E55524C962537361AFCE47203000749000868617368436F6465490004706F72744C0009617574686F726974797400124C6A6176612F6C616E672F537472696E673B4C000466696C6571007E00034C0004686F737471007E00034C000870726F746F636F6C71007E00034C000372656671007E00037870FFFFFFFFFFFFFFFF7400") buffer.Write(prefix) - buffer.WriteString(string(rune(len(url)))) - buffer.WriteString(url) - suffix, _ := hex.DecodeString("74000071007E00057400056874747073707874001968747470733A2F2F746573742E6A6578626F73732E696E666F78") - buffer.Write(suffix) + buffer.WriteString(string(rune(len(hostname)))) + buffer.WriteString(hostname) + + middle, _ := hex.DecodeString("74000071007E0005740005") + buffer.Write(middle) + buffer.WriteString(parsed.Scheme) + + middle, _ = hex.DecodeString("70787400") + buffer.Write(middle) + buffer.WriteString(string(rune(len(URL)))) + buffer.WriteString(URL) + + suffix, _ := hex.DecodeString("78") + buffer.Write(suffix) return buffer.Bytes() } From aef57175b2acc54836220ccc0ceb62af03e369d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 14:34:01 +0530 Subject: [PATCH 29/32] chore(deps): bump github.com/aws/aws-sdk-go in /v2 (#2003) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.14 to 1.44.15. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.14...v1.44.15) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 798fb142f..fc6908e36 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -63,7 +63,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/aws/aws-sdk-go v1.44.14 +require github.com/aws/aws-sdk-go v1.44.15 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 diff --git a/v2/go.sum b/v2/go.sum index 0281b593b..5632f9a1c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -86,8 +86,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.14 h1:qd7/muV1rElsbvkK9D1nHUzBoDlEw2etfeo4IE82eSQ= -github.com/aws/aws-sdk-go v1.44.14/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.15 h1:z02BVeV6k7hZMfWEQmKh3X23s3F9PBHFCcIVfNlut7A= +github.com/aws/aws-sdk-go v1.44.15/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 184f3fdc30a78bfa47c3b2e2254efba0114291f4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 17 May 2022 11:08:48 +0200 Subject: [PATCH 30/32] Fixing concurrent read/write map (#1989) --- v2/internal/runner/runner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 88b272062..5cb31ac40 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -570,8 +570,10 @@ func isTemplate(filename string) bool { // SaveResumeConfig to file func (r *Runner) SaveResumeConfig(path string) error { resumeCfg := types.NewResumeCfg() + r.resumeCfg.Lock() resumeCfg.ResumeFrom = r.resumeCfg.Current data, _ := json.MarshalIndent(resumeCfg, "", "\t") + r.resumeCfg.Unlock() return os.WriteFile(path, data, os.ModePerm) } From d5e451682988d8a51e24cd65a04711ae40b9679f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 17 May 2022 11:52:00 +0200 Subject: [PATCH 31/32] Iterating payloads over HTTP path/raw sequence (#1981) * Iterating payloads over path/raw sequence * fixing logic check --- v2/pkg/protocols/http/request_generator.go | 92 +++++++++------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/v2/pkg/protocols/http/request_generator.go b/v2/pkg/protocols/http/request_generator.go index 83abd3df0..58e31fbfb 100644 --- a/v2/pkg/protocols/http/request_generator.go +++ b/v2/pkg/protocols/http/request_generator.go @@ -12,11 +12,13 @@ import ( // values. Paths and Raw requests are supported as base input, so // it will automatically select between them based on the template. type requestGenerator struct { - currentIndex int - request *Request - options *protocols.ExecuterOptions - payloadIterator *generators.Iterator - interactshURLs []string + currentIndex int + currentPayloads map[string]interface{} + okCurrentPayload bool + request *Request + options *protocols.ExecuterOptions + payloadIterator *generators.Iterator + interactshURLs []string } // LeaveDefaultPorts skips normalization of default standard ports @@ -35,61 +37,45 @@ func (request *Request) newGenerator() *requestGenerator { // nextValue returns the next path or the next raw request depending on user input // It returns false if all the inputs have been exhausted by the generator instance. func (r *requestGenerator) nextValue() (value string, payloads map[string]interface{}, result bool) { - // For both raw/path requests, start with the request at current index. - // If we are not at the start, then check if the iterator for payloads - // has finished if there are any. + // Iterate each payload sequentially for each request path/raw // - // If the iterator has finished for the current request - // then reset it and move on to the next value, otherwise use the last request. + // If the sequence has finished for the current payload values + // then restart the sequence from the beginning and move on to the next payloads values + // otherwise use the last request. + var sequence []string + switch { + case len(r.request.Path) > 0: + sequence = r.request.Path + case len(r.request.Raw) > 0: + sequence = r.request.Raw + default: + return "", nil, false + } - if len(r.request.Path) > 0 && r.currentIndex < len(r.request.Path) { - if r.payloadIterator != nil { - payload, ok := r.payloadIterator.Value() - if !ok { - r.currentIndex++ - r.payloadIterator.Reset() + hasPayloadIterator := r.payloadIterator != nil + hasInitializedPayloads := r.currentPayloads != nil - // No more payloads request for us now. - if len(r.request.Path) == r.currentIndex { - return "", nil, false + if r.currentIndex == 0 && hasPayloadIterator && !hasInitializedPayloads { + r.currentPayloads, r.okCurrentPayload = r.payloadIterator.Value() + } + if r.currentIndex < len(sequence) { + currentRequest := sequence[r.currentIndex] + r.currentIndex++ + return currentRequest, r.currentPayloads, true + } + if r.currentIndex == len(sequence) { + if r.okCurrentPayload { + r.currentIndex = 0 + currentRequest := sequence[r.currentIndex] + if hasPayloadIterator { + r.currentPayloads, r.okCurrentPayload = r.payloadIterator.Value() + if r.okCurrentPayload { + r.currentIndex++ + return currentRequest, r.currentPayloads, true } - if item := r.request.Path[r.currentIndex]; item != "" { - newPayload, ok := r.payloadIterator.Value() - return item, newPayload, ok - } - return "", nil, false } - return r.request.Path[r.currentIndex], payload, true - } - if value := r.request.Path[r.currentIndex]; value != "" { - r.currentIndex++ - return value, nil, true } } - if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) { - if r.payloadIterator != nil { - payload, ok := r.payloadIterator.Value() - if !ok { - r.currentIndex++ - r.payloadIterator.Reset() - - // No more payloads request for us now. - if len(r.request.Raw) == r.currentIndex { - return "", nil, false - } - if item := r.request.Raw[r.currentIndex]; item != "" { - newPayload, ok := r.payloadIterator.Value() - return item, newPayload, ok - } - return "", nil, false - } - return r.request.Raw[r.currentIndex], payload, true - } - if item := r.request.Raw[r.currentIndex]; item != "" { - r.currentIndex++ - return item, nil, true - } - } return "", nil, false } From 91c35df9110d48d3c1b037528429b4b2d052c758 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 17 May 2022 16:36:33 +0530 Subject: [PATCH 32/32] version + readme update --- README.md | 12 ++++++------ v2/cmd/nuclei/main.go | 8 ++++---- v2/pkg/catalog/config/config.go | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 20fc7631d..8b0821dce 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,8 @@ FILTERING: OUTPUT: -o, -output string output file to write found issues/vulnerabilities - -silent display findings only - -nc, -no-color disable output content coloring (ANSI escape codes) + -sresp, -store-resp store all request/response passed through nuclei to output directory + -srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output") -json write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) -nm, -no-meta disable printing result metadata in cli output @@ -133,6 +133,8 @@ OUTPUT: -ms, -matcher-status display match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format + -silent display findings only + -nc, -no-color disable output content coloring (ANSI escape codes) CONFIGURATIONS: -config string path to the nuclei configuration file @@ -149,8 +151,8 @@ CONFIGURATIONS: -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts - -ztls Use ztls library with autofallback to standard one for tls13 - -sni string Global SNI hostname (default: input domain name) + -ztls use ztls library with autofallback to standard one for tls13 + -sni string tls sni hostname to use (default: input domain name) INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) @@ -189,8 +191,6 @@ DEBUG: -debug display all requests and responses -dreq, -debug-req display all sent requests -dresp, -debug-resp display all received responses - -sresp, -store-resp store all request/response passed through nuclei to output directory - -srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output") -p, -proxy string[] list of http/socks5 proxy to use (comma separated or file input) -pi, -proxy-internal proxy all internal requests -tlog, -trace-log string file to write sent requests trace log diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 9bc5d1680..c49a2bbeb 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -119,6 +119,8 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"), + flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"), + flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"), flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"), @@ -146,8 +148,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), - flagSet.BoolVar(&options.ZTLS, "ztls", false, "Use ztls library with autofallback to standard one for tls13"), - flagSet.StringVar(&options.SNI, "sni", "", "Global SNI hostname (default: input domain name)"), + flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), + flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"), ) createGroup(flagSet, "interactsh", "interactsh", @@ -191,8 +193,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"), flagSet.BoolVarP(&options.DebugRequests, "debug-req", "dreq", false, "show all sent requests"), flagSet.BoolVarP(&options.DebugResponse, "debug-resp", "dresp", false, "show all received responses"), - flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"), - flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"), flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "list of http/socks5 proxy to use (comma separated or file input)"), flagSet.BoolVarP(&options.ProxyInternal, "proxy-internal", "pi", false, "proxy all internal requests"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 8450b8419..127b281c3 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -27,7 +27,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.7.0` +const Version = `2.7.1` func getConfigDetails() (string, error) { configDir, err := GetConfigDir()