External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 16:25:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding support to scan all v4/v6 IPs (#2709)

Browse Source 
* Adding support to scan all v4/v6 IPs

* adding tests

* metainput prototype

* using new signature

* fixing nil pointer

* adding request context with metadata

* removing log instruction

* fixing merge conflicts

* adding clone helpers

* attempting to fix ipv6 square parenthesis wrap

* fixing dialed ip info

* fixing syntax

* fixing output ip selection

* adding integration tests

* disabling test due to gh ipv6 issue

* using ipv4 only due to GH limited networking

* extending metainput marshaling

* fixing hmap key

* adding test for httpx integration

* fixing lint error

* reworking marshaling/id-calculation

* adding ip version validation

* improving handling non url targets

* fixing condition check
This commit is contained in:
Mzack9999 2022-11-09 14:18:56 +01:00 committed by GitHub
parent 9e56451d2e
commit 1fbbca66f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 543 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
integration_tests/http/get-all-ips.yaml Normal file
View File

@ -0,0 +1,15 @@
id: get-all-ips
info:
name: Basic GET Request on all IPS
author: pdteam
severity: info
requests:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "ok"

15
integration_tests/http/get-without-scheme.yaml Normal file
View File

@ -0,0 +1,15 @@
id: get-without-scheme
info:
name: Basic GET Request without scheme
author: pdteam
severity: info
requests:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "ok"

3
v2/cmd/integration-test/code.go
View File

@ -23,6 +23,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
@ -123,7 +124,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
} }
store.Load() store.Load()
input := &inputs.SimpleInputProvider{Inputs: []string{templateURL}} input := &inputs.SimpleInputProvider{Inputs: []*contextargs.MetaInput{{Input: templateURL}}}
_ = engine.Execute(store.Templates(), input) _ = engine.Execute(store.Templates(), input)
engine.WorkPool().Wait() // Wait for the scan to finish engine.WorkPool().Wait() // Wait for the scan to finish

27
v2/cmd/integration-test/http.go
View File

@ -57,6 +57,8 @@ var httpTestcases = map[string]testutils.TestCase{
"http/get-sni-unsafe.yaml": &customCLISNIUnsafe{}, "http/get-sni-unsafe.yaml": &customCLISNIUnsafe{},
"http/annotation-timeout.yaml": &annotationTimeout{}, "http/annotation-timeout.yaml": &annotationTimeout{},
"http/custom-attack-type.yaml": &customAttackType{}, "http/custom-attack-type.yaml": &customAttackType{},
"http/get-all-ips.yaml": &scanAllIPS{},
"http/get-without-scheme.yaml": &httpGetWithoutScheme{},
} }
type httpInteractshRequest struct{} type httpInteractshRequest struct{}
@ -1001,3 +1003,28 @@ func (h *customAttackType) Execute(filePath string) error {
} }
return expectResultsCount(got, 4) return expectResultsCount(got, 4)
} }
// Disabled as GH doesn't support ipv6
type scanAllIPS struct{}
// Execute executes a test case and returns an error if occurred
func (h *scanAllIPS) Execute(filePath string) error {
got, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-scan-all-ips", "-iv", "4")
if err != nil {
return err
}
// limiting test to ipv4 (GH doesn't support ipv6)
return expectResultsCount(got, 1)
}
// ensure that ip|host are handled without http|https scheme
type httpGetWithoutScheme struct{}
// Execute executes a test case and returns an error if occurred
func (h *httpGetWithoutScheme) Execute(filePath string) error {
got, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(got, 1)
}

2
v2/cmd/nuclei/main.go
View File

@ -125,6 +125,8 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan", goflags.StringSliceOptions), flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan", goflags.StringSliceOptions),
flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"), flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
flagSet.StringVar(&options.Resume, "resume", "", "Resume scan using resume.cfg (clustering will be disabled)"), flagSet.StringVar(&options.Resume, "resume", "", "Resume scan using resume.cfg (clustering will be disabled)"),
flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "Scan all the ip's associated with dns record"),
flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", []string{"4"}, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
) )
flagSet.CreateGroup("templates", "Templates", flagSet.CreateGroup("templates", "Templates",

4
v2/cmd/tools/fuzzplayground/main.go
View File

@ -2,7 +2,7 @@ package main
import ( import (
"fmt" "fmt"
"io/ioutil" "io"
"net/http" "net/http"
"os/exec" "os/exec"
"strconv" "strconv"
@ -66,7 +66,7 @@ func requestHandler(ctx echo.Context) error {
} }
defer data.Body.Close() defer data.Body.Close()
body, _ := ioutil.ReadAll(data.Body) body, _ := io.ReadAll(data.Body)
return ctx.HTML(200, fmt.Sprintf(bodyTemplate, string(body))) return ctx.HTML(200, fmt.Sprintf(bodyTemplate, string(body)))
} }

4
v2/go.mod
View File

@ -25,8 +25,7 @@ require (
github.com/owenrumney/go-sarif/v2 v2.1.2 github.com/owenrumney/go-sarif/v2 v2.1.2
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/cryptoutil v1.0.0 // indirect github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0
github.com/projectdiscovery/fastdialer v0.0.17
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/hmap v0.0.2
@ -208,6 +207,7 @@ require (
github.com/nwaples/rardecode v1.1.2 // indirect github.com/nwaples/rardecode v1.1.2 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/projectdiscovery/asnmap v0.0.1 // indirect github.com/projectdiscovery/asnmap v0.0.1 // indirect
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect; indirectdev
github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/fileutil v0.0.3 // indirect
github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/iputil v0.0.2 // indirect
github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/sliceutil v0.0.1 // indirect

27
v2/go.sum
View File

@ -518,8 +518,6 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA
github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU=
github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE=
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
@ -596,26 +594,21 @@ github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK
github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8=
github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U=
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc=
github.com/projectdiscovery/cryptoutil v1.0.0 h1:5rQfnWDthJ5ZFcqze+rmT1N7l1HJQ6EB26MrjaYB7I0=
github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4=
github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80= github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80=
github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0 h1:1hcFBedqq8772PxN3Lbq7Itr3N59C8ro0xaTxYkmg9s=
github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0/go.mod h1:KSHL57MbR0PbdJpagiqqB0jPqO1GUcnYZT5ngAvsmqQ=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew=
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ=
github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY=
github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g=
github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg=
github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80=
github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M=
github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
@ -633,14 +626,12 @@ github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU=
github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U=
github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI=
github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ=
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw=
github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY=
github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU=
github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
@ -661,24 +652,18 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk=
github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w= github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w=
github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk= github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ=
github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE=
github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA=
github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI=
github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg=
github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI= github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc=
github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU=
@ -762,8 +747,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@ -993,17 +976,14 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
@ -1102,7 +1082,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

5
v2/internal/runner/enumerate.go
View File

@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"go.uber.org/atomic" "go.uber.org/atomic"
) )
@ -84,8 +85,8 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic
// TODO: Add payload file and workflow support for private templates // TODO: Add payload file and workflow support for private templates
catalogChecksums := nucleicloud.ReadCatalogChecksum() catalogChecksums := nucleicloud.ReadCatalogChecksum()
targets := make([]string, 0, r.hmapInputProvider.Count()) targets := make([]*contextargs.MetaInput, 0, r.hmapInputProvider.Count())
r.hmapInputProvider.Scan(func(value string) bool { r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
targets = append(targets, value) targets = append(targets, value)
return true return true
}) })

11
v2/internal/runner/inputs.go
View File

@ -9,6 +9,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/retryablehttp-go"
"github.com/remeh/sizedwaitgroup" "github.com/remeh/sizedwaitgroup"
@ -37,18 +38,18 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
// Probe the non-standard URLs and store them in cache // Probe the non-standard URLs and store them in cache
swg := sizedwaitgroup.New(bulkSize) swg := sizedwaitgroup.New(bulkSize)
count := int32(0) count := int32(0)
r.hmapInputProvider.Scan(func(value string) bool { r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") { if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") {
return true return true
} }
swg.Add() swg.Add()
go func(input string) { go func(input *contextargs.MetaInput) {
defer swg.Done() defer swg.Done()
if result := probeURL(input, httpclient); result != "" { if result := probeURL(input.Input, httpclient); result != "" {
atomic.AddInt32(&count, 1) atomic.AddInt32(&count, 1)
_ = hm.Set(input, []byte(result)) _ = hm.Set(input.Input, []byte(result))
} }
}(value) }(value)
return true return true

8
v2/internal/runner/nucleicloud/types.go
View File

@ -1,11 +1,15 @@
package nucleicloud package nucleicloud
import "time" import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
)
// AddScanRequest is a nuclei scan input item. // AddScanRequest is a nuclei scan input item.
type AddScanRequest struct { type AddScanRequest struct {
// RawTargets is a list of raw target URLs for the scan. // RawTargets is a list of raw target URLs for the scan.
RawTargets []string `json:"raw_targets,omitempty"` RawTargets []*contextargs.MetaInput `json:"raw_targets,omitempty"`
// PublicTemplates is a list of public templates for the scan // PublicTemplates is a list of public templates for the scan
PublicTemplates []string `json:"public_templates,omitempty"` PublicTemplates []string `json:"public_templates,omitempty"`
// PrivateTemplates is a map of template-name->contents that // PrivateTemplates is a map of template-name->contents that

18
v2/internal/runner/options.go
View File

@ -2,6 +2,7 @@ package runner
import ( import (
"bufio" "bufio"
"fmt"
"io" "io"
"log" "log"
"os" "os"
@ -135,6 +136,23 @@ func validateOptions(options *types.Options) error {
} }
validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile}) validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile})
} }
// verify that a valid ip version type was selected (4, 6)
var useIPV4, useIPV6 bool
for _, ipv := range options.IPVersion {
switch ipv {
case "4":
useIPV4 = true
case "6":
useIPV6 = true
default:
return fmt.Errorf("unsupported ip version: %s", ipv)
}
}
if !useIPV4 && !useIPV6 {
return errors.New("ipv4 and/or ipv6 must be selected")
}
return nil return nil
} }

5
v2/internal/runner/runner.go
View File

@ -37,6 +37,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/automaticscan" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/automaticscan"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
@ -478,8 +479,8 @@ func (r *Runner) RunEnumeration() error {
func (r *Runner) isInputNonHTTP() bool { func (r *Runner) isInputNonHTTP() bool {
var nonURLInput bool var nonURLInput bool
r.hmapInputProvider.Scan(func(value string) bool { r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
if !strings.Contains(value, "://") { if !strings.Contains(value.Input, "://") {
nonURLInput = true nonURLInput = true
return false return false
} }

3
v2/pkg/core/engine.go
View File

@ -2,6 +2,7 @@ package core
import ( import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
) )
@ -29,7 +30,7 @@ type InputProvider interface {
Count() int64 Count() int64
// Scan iterates the input and each found item is passed to the // Scan iterates the input and each found item is passed to the
// callback consumer. // callback consumer.
Scan(callback func(value string) bool) Scan(callback func(value *contextargs.MetaInput) bool)
} }
// New returns a new Engine instance // New returns a new Engine instance

26
v2/pkg/core/execute.go
View File

@ -43,6 +43,8 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp
wg.Add() wg.Add()
go func(tpl *templates.Template) { go func(tpl *templates.Template) {
defer wg.Done()
switch { switch {
case tpl.SelfContained: case tpl.SelfContained:
// Self Contained requests are executed here separately // Self Contained requests are executed here separately
@ -51,7 +53,6 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp
// All other request types are executed here // All other request types are executed here
e.executeModelWithInput(templateType, tpl, target, results) e.executeModelWithInput(templateType, tpl, target, results)
} }
wg.Done()
}(template) }(template)
} }
e.workPool.Wait() e.workPool.Wait()
@ -98,7 +99,7 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
currentInfo.Unlock() currentInfo.Unlock()
} }
target.Scan(func(scannedValue string) bool { target.Scan(func(scannedValue *contextargs.MetaInput) bool {
// Best effort to track the host progression // Best effort to track the host progression
// skips indexes lower than the minimum in-flight at interruption time // skips indexes lower than the minimum in-flight at interruption time
var skip bool var skip bool
@ -122,12 +123,12 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
currentInfo.Unlock() currentInfo.Unlock()
// Skip if the host has had errors // Skip if the host has had errors
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
return true return true
} }
wg.WaitGroup.Add() wg.WaitGroup.Add()
go func(index uint32, skip bool, value string) { go func(index uint32, skip bool, value *contextargs.MetaInput) {
defer wg.WaitGroup.Done() defer wg.WaitGroup.Done()
defer cleanupInFlight(index) defer cleanupInFlight(index)
if skip { if skip {
@ -141,7 +142,7 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
match = e.executeWorkflow(value, template.CompiledWorkflow) match = e.executeWorkflow(value, template.CompiledWorkflow)
default: default:
ctxArgs := contextargs.New() ctxArgs := contextargs.New()
ctxArgs.Input = value ctxArgs.MetaInput = value
match, err = template.Executer.Execute(ctxArgs) match, err = template.Executer.Execute(ctxArgs)
} }
if err != nil { if err != nil {
@ -188,14 +189,14 @@ func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target
func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool, callback func(*output.ResultEvent)) { func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool, callback func(*output.ResultEvent)) {
wg := e.workPool.InputPool(templateType) wg := e.workPool.InputPool(templateType)
target.Scan(func(scannedValue string) bool { target.Scan(func(scannedValue *contextargs.MetaInput) bool {
// Skip if the host has had errors // Skip if the host has had errors
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
return true return true
} }
wg.WaitGroup.Add() wg.WaitGroup.Add()
go func(value string) { go func(value *contextargs.MetaInput) {
defer wg.WaitGroup.Done() defer wg.WaitGroup.Done()
var match bool var match bool
@ -205,7 +206,7 @@ func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType,
match = e.executeWorkflow(value, template.CompiledWorkflow) match = e.executeWorkflow(value, template.CompiledWorkflow)
default: default:
ctxArgs := contextargs.New() ctxArgs := contextargs.New()
ctxArgs.Input = value ctxArgs.MetaInput = value
err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) { err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) {
for _, result := range event.Results { for _, result := range event.Results {
callback(result) callback(result)
@ -235,7 +236,7 @@ func (e *ChildExecuter) Close() *atomic.Bool {
} }
// Execute executes a template and URLs // Execute executes a template and URLs
func (e *ChildExecuter) Execute(template *templates.Template, URL string) { func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
templateType := template.Type() templateType := template.Type()
var wg *sizedwaitgroup.SizedWaitGroup var wg *sizedwaitgroup.SizedWaitGroup
@ -247,14 +248,15 @@ func (e *ChildExecuter) Execute(template *templates.Template, URL string) {
wg.Add() wg.Add()
go func(tpl *templates.Template) { go func(tpl *templates.Template) {
defer wg.Done()
ctxArgs := contextargs.New() ctxArgs := contextargs.New()
ctxArgs.Input = URL ctxArgs.MetaInput = value
match, err := template.Executer.Execute(ctxArgs) match, err := template.Executer.Execute(ctxArgs)
if err != nil { if err != nil {
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err) gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
} }
e.results.CompareAndSwap(false, match) e.results.CompareAndSwap(false, match)
wg.Done()
}(template) }(template)
} }

131
v2/pkg/core/inputs/hybrid/hmap.go
View File

@ -5,6 +5,7 @@ package hybrid
import ( import (
"bufio" "bufio"
"io" "io"
"net/url"
"os" "os"
"strings" "strings"
"time" "time"
@ -16,13 +17,17 @@ import (
"github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/mapcidr"
asn "github.com/projectdiscovery/mapcidr/asn" asn "github.com/projectdiscovery/mapcidr/asn"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
fileutil "github.com/projectdiscovery/utils/file" fileutil "github.com/projectdiscovery/utils/file"
iputil "github.com/projectdiscovery/utils/ip" iputil "github.com/projectdiscovery/utils/ip"
sliceutil "github.com/projectdiscovery/utils/slice"
) )
// Input is a hmap/filekv backed nuclei Input provider // Input is a hmap/filekv backed nuclei Input provider
type Input struct { type Input struct {
ipOptions *ipOptions
inputCount int64 inputCount int64
dupeCount int64 dupeCount int64
hostMap *hybrid.HybridMap hostMap *hybrid.HybridMap
@ -37,7 +42,14 @@ func New(options *types.Options) (*Input, error) {
return nil, errors.Wrap(err, "could not create temporary input file") return nil, errors.Wrap(err, "could not create temporary input file")
} }
input := &Input{hostMap: hm} input := &Input{
hostMap: hm,
ipOptions: &ipOptions{
ScanAllIPs: options.ScanAllIPs,
IPV4: sliceutil.Contains(options.IPVersion, "4"),
IPV6: sliceutil.Contains(options.IPVersion, "6"),
},
}
if options.Stream { if options.Stream {
fkvOptions := filekv.DefaultOptions fkvOptions := filekv.DefaultOptions
if tmpFileName, err := fileutil.GetTempFileName(); err != nil { if tmpFileName, err := fileutil.GetTempFileName(); err != nil {
@ -72,16 +84,15 @@ func (i *Input) Close() {
func (i *Input) initializeInputSources(options *types.Options) error { func (i *Input) initializeInputSources(options *types.Options) error {
// Handle targets flags // Handle targets flags
for _, target := range options.Targets { for _, target := range options.Targets {
if iputil.IsCIDR(target) { switch {
case iputil.IsCIDR(target):
i.expandCIDRInputValue(target) i.expandCIDRInputValue(target)
continue case asn.IsASN(target):
}
if asn.IsASN(target) {
i.expandASNInputValue(target) i.expandASNInputValue(target)
continue default:
}
i.normalizeStoreInputValue(target) i.normalizeStoreInputValue(target)
} }
}
// Handle stdin // Handle stdin
if options.Stdin { if options.Stdin {
@ -94,8 +105,9 @@ func (i *Input) initializeInputSources(options *types.Options) error {
if inputErr != nil { if inputErr != nil {
return errors.Wrap(inputErr, "could not open targets file") return errors.Wrap(inputErr, "could not open targets file")
} }
defer input.Close()
i.scanInputFromReader(input) i.scanInputFromReader(input)
input.Close()
} }
return nil return nil
} }
@ -104,34 +116,89 @@ func (i *Input) initializeInputSources(options *types.Options) error {
func (i *Input) scanInputFromReader(reader io.Reader) { func (i *Input) scanInputFromReader(reader io.Reader) {
scanner := bufio.NewScanner(reader) scanner := bufio.NewScanner(reader)
for scanner.Scan() { for scanner.Scan() {
if iputil.IsCIDR(scanner.Text()) { item := scanner.Text()
i.expandCIDRInputValue(scanner.Text()) switch {
continue case iputil.IsCIDR(item):
i.expandCIDRInputValue(item)
case asn.IsASN(item):
i.expandASNInputValue(item)
default:
i.normalizeStoreInputValue(item)
} }
if asn.IsASN(scanner.Text()) {
i.expandASNInputValue(scanner.Text())
continue
}
i.normalizeStoreInputValue(scanner.Text())
} }
} }
// normalizeStoreInputValue normalizes and stores passed input values // normalizeStoreInputValue normalizes and stores passed input values
func (i *Input) normalizeStoreInputValue(value string) { func (i *Input) normalizeStoreInputValue(value string) {
url := strings.TrimSpace(value) URL := strings.TrimSpace(value)
if url == "" { if URL == "" {
return return
} }
if _, ok := i.hostMap.Get(url); ok { metaInput := &contextargs.MetaInput{Input: URL}
keyURL, err := metaInput.MarshalString()
if err != nil {
gologger.Warning().Msgf("%s\n", err)
return
}
if _, ok := i.hostMap.Get(keyURL); ok {
i.dupeCount++ i.dupeCount++
return return
} }
i.inputCount++ switch {
_ = i.hostMap.Set(url, nil) case i.ipOptions.ScanAllIPs:
// we need to resolve the hostname
// check if it's an url
var host string
parsedURL, err := url.Parse(value)
if err == nil && parsedURL.Host != "" {
host = parsedURL.Host
} else {
parsedURL = nil
host = value
}
dnsData, err := protocolstate.Dialer.GetDNSData(host)
if err == nil && (len(dnsData.A)+len(dnsData.AAAA)) > 0 {
var ips []string
if i.ipOptions.IPV4 {
ips = append(ips, dnsData.A...)
}
if i.ipOptions.IPV6 {
ips = append(ips, dnsData.AAAA...)
}
for _, ip := range ips {
if ip == "" {
continue
}
metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
key, err := metaInput.MarshalString()
if err != nil {
gologger.Warning().Msgf("%s\n", err)
continue
}
_ = i.hostMap.Set(key, nil)
if i.hostMapStream != nil { if i.hostMapStream != nil {
_ = i.hostMapStream.Set([]byte(url), nil) _ = i.hostMapStream.Set([]byte(key), nil)
}
}
break
}
fallthrough
default:
i.setItem(keyURL)
}
}
// setItem in the kv store
func (i *Input) setItem(k string) {
i.inputCount++
_ = i.hostMap.Set(k, nil)
if i.hostMapStream != nil {
_ = i.hostMapStream.Set([]byte(k), nil)
} }
} }
@ -142,9 +209,13 @@ func (i *Input) Count() int64 {
// Scan iterates the input and each found item is passed to the // Scan iterates the input and each found item is passed to the
// callback consumer. // callback consumer.
func (i *Input) Scan(callback func(value string) bool) { func (i *Input) Scan(callback func(value *contextargs.MetaInput) bool) {
callbackFunc := func(k, _ []byte) error { callbackFunc := func(k, _ []byte) error {
if !callback(string(k)) { metaInput := &contextargs.MetaInput{}
if err := metaInput.Unmarshal(string(k)); err != nil {
return err
}
if !callback(metaInput) {
return io.EOF return io.EOF
} }
return nil return nil
@ -160,14 +231,20 @@ func (i *Input) Scan(callback func(value string) bool) {
func (i *Input) expandCIDRInputValue(value string) { func (i *Input) expandCIDRInputValue(value string) {
ips, _ := mapcidr.IPAddressesAsStream(value) ips, _ := mapcidr.IPAddressesAsStream(value)
for ip := range ips { for ip := range ips {
if _, ok := i.hostMap.Get(ip); ok { metaInput := &contextargs.MetaInput{Input: ip}
key, err := metaInput.MarshalString()
if err != nil {
gologger.Warning().Msgf("%s\n", err)
return
}
if _, ok := i.hostMap.Get(key); ok {
i.dupeCount++ i.dupeCount++
continue continue
} }
i.inputCount++ i.inputCount++
_ = i.hostMap.Set(ip, nil) _ = i.hostMap.Set(key, nil)
if i.hostMapStream != nil { if i.hostMapStream != nil {
_ = i.hostMapStream.Set([]byte(ip), nil) _ = i.hostMapStream.Set([]byte(key), nil)
} }
} }
} }

75
v2/pkg/core/inputs/hybrid/hmap_test.go
View File

@ -6,6 +6,9 @@ import (
"testing" "testing"
"github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -30,11 +33,73 @@ func Test_expandCIDRInputValue(t *testing.T) {
input.expandCIDRInputValue(tt.cidr) input.expandCIDRInputValue(tt.cidr)
// scan // scan
got := []string{} got := []string{}
input.hostMap.Scan(func(k, v []byte) error { input.hostMap.Scan(func(k, _ []byte) error {
got = append(got, string(k)) var metainput contextargs.MetaInput
if err := metainput.Unmarshal(string(k)); err != nil {
return err
}
got = append(got, metainput.Input)
return nil return nil
}) })
require.ElementsMatch(t, tt.expected, got, "could not get correct cidrs") require.ElementsMatch(t, tt.expected, got, "could not get correct cidrs")
input.Close()
}
}
func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
defaultOpts := types.DefaultOptions()
_ = protocolstate.Init(defaultOpts)
tests := []struct {
hostname string
ipv4 bool
ipv6 bool
expected []string
}{
{
hostname: "scanme.sh",
ipv4: true,
ipv6: true,
expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
}, {
hostname: "scanme.sh",
ipv4: true,
expected: []string{"128.199.158.128"},
}, {
hostname: "scanme.sh",
ipv6: true,
expected: []string{"2400:6180:0:d0::91:1001"},
}, {
hostname: "http://scanme.sh",
ipv4: true,
ipv6: true,
expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
},
}
for _, tt := range tests {
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
require.Nil(t, err, "could not create temporary input file")
input := &Input{
hostMap: hm,
ipOptions: &ipOptions{
ScanAllIPs: true,
IPV4: tt.ipv4,
IPV6: tt.ipv6,
},
}
input.normalizeStoreInputValue(tt.hostname)
// scan
got := []string{}
input.hostMap.Scan(func(k, v []byte) error {
var metainput contextargs.MetaInput
if err := metainput.Unmarshal(string(k)); err != nil {
return err
}
got = append(got, metainput.CustomIP)
return nil
})
require.ElementsMatch(t, tt.expected, got, "could not get correct ips")
input.Close()
} }
} }
@ -61,7 +126,11 @@ func Test_expandASNInputValue(t *testing.T) {
// scan the hmap // scan the hmap
got := []string{} got := []string{}
input.hostMap.Scan(func(k, v []byte) error { input.hostMap.Scan(func(k, v []byte) error {
got = append(got, string(k)) var metainput contextargs.MetaInput
if err := metainput.Unmarshal(string(k)); err != nil {
return err
}
got = append(got, metainput.Input)
return nil return nil
}) })
// read the expected IPs from the file // read the expected IPs from the file

7
v2/pkg/core/inputs/hybrid/options.go Normal file
View File

@ -0,0 +1,7 @@
package hybrid
type ipOptions struct {
ScanAllIPs bool
IPV4 bool
IPV6 bool
}

6
v2/pkg/core/inputs/inputs.go
View File

@ -1,7 +1,9 @@
package inputs package inputs
import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
type SimpleInputProvider struct { type SimpleInputProvider struct {
Inputs []string Inputs []*contextargs.MetaInput
} }
// Count returns the number of items for input provider // Count returns the number of items for input provider
@ -10,7 +12,7 @@ func (s *SimpleInputProvider) Count() int64 {
} }
// Scan calls a callback function till the input provider is exhausted // Scan calls a callback function till the input provider is exhausted
func (s *SimpleInputProvider) Scan(callback func(value string) bool) { func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) bool) {
for _, v := range s.Inputs { for _, v := range s.Inputs {
if !callback(v) { if !callback(v) {
return return

6
v2/pkg/core/workflow_execute.go
View File

@ -16,13 +16,13 @@ import (
const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
// executeWorkflow runs a workflow on an input and returns true or false // executeWorkflow runs a workflow on an input and returns true or false
func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Workflow) bool {
results := &atomic.Bool{} results := &atomic.Bool{}
// at this point we should be at the start root execution of a workflow tree, hence we create global shared instances // at this point we should be at the start root execution of a workflow tree, hence we create global shared instances
workflowCookieJar, _ := cookiejar.New(nil) workflowCookieJar, _ := cookiejar.New(nil)
ctxArgs := contextargs.New() ctxArgs := contextargs.New()
ctxArgs.Input = input ctxArgs.MetaInput = input
ctxArgs.CookieJar = workflowCookieJar ctxArgs.CookieJar = workflowCookieJar
swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads)
@ -88,7 +88,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co
} }
if err != nil { if err != nil {
if w.Options.HostErrorsCache != nil { if w.Options.HostErrorsCache != nil {
w.Options.HostErrorsCache.MarkFailed(input.Input, err) w.Options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
} }
if len(template.Executers) == 1 { if len(template.Executers) == 1 {
mainErr = err mainErr = err

58
v2/pkg/core/workflow_execute_test.go
View File

@ -25,7 +25,7 @@ func TestWorkflowsSimple(t *testing.T) {
}} }}
engine := &Engine{} engine := &Engine{}
matched := engine.executeWorkflow("https://test.com", workflow) matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
require.True(t, matched, "could not get correct match value") require.True(t, matched, "could not get correct match value")
} }
@ -35,19 +35,19 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
{Executers: []*workflows.ProtocolExecuterPair{{ {Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
firstInput = input firstInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}}, }},
{Executers: []*workflows.ProtocolExecuterPair{{ {Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
secondInput = input secondInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}}, }},
}} }}
engine := &Engine{} engine := &Engine{}
matched := engine.executeWorkflow("https://test.com", workflow) matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
require.True(t, matched, "could not get correct match value") require.True(t, matched, "could not get correct match value")
require.Equal(t, "https://test.com", firstInput, "could not get correct first input") require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@ -60,20 +60,20 @@ func TestWorkflowsSubtemplates(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
{Executers: []*workflows.ProtocolExecuterPair{{ {Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
firstInput = input firstInput = input.Input
}, outputs: []*output.InternalWrappedEvent{ }, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}}, {OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}},
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
secondInput = input secondInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}}}}, }}}},
}} }}
engine := &Engine{} engine := &Engine{}
matched := engine.executeWorkflow("https://test.com", workflow) matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
require.True(t, matched, "could not get correct match value") require.True(t, matched, "could not get correct match value")
require.Equal(t, "https://test.com", firstInput, "could not get correct first input") require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@ -86,18 +86,18 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
{Executers: []*workflows.ProtocolExecuterPair{{ {Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: false, executeHook: func(input string) { Executer: &mockExecuter{result: false, executeHook: func(input *contextargs.MetaInput) {
firstInput = input firstInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
secondInput = input secondInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}}}}, }}}},
}} }}
engine := &Engine{} engine := &Engine{}
matched := engine.executeWorkflow("https://test.com", workflow) matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
require.False(t, matched, "could not get correct match value") require.False(t, matched, "could not get correct match value")
require.Equal(t, "https://test.com", firstInput, "could not get correct first input") require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@ -110,8 +110,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
{Executers: []*workflows.ProtocolExecuterPair{{ {Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
firstInput = input firstInput = input.Input
}, outputs: []*output.InternalWrappedEvent{ }, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{ {OperatorsResult: &operators.Result{
Matches: map[string][]string{"tomcat": {}}, Matches: map[string][]string{"tomcat": {}},
@ -119,14 +119,14 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
}}, }},
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "tomcat"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ }, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "tomcat"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
secondInput = input secondInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}}}}}}, }}}}}},
}} }}
engine := &Engine{} engine := &Engine{}
matched := engine.executeWorkflow("https://test.com", workflow) matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
require.True(t, matched, "could not get correct match value") require.True(t, matched, "could not get correct match value")
require.Equal(t, "https://test.com", firstInput, "could not get correct first input") require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@ -139,8 +139,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
{Executers: []*workflows.ProtocolExecuterPair{{ {Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
firstInput = input firstInput = input.Input
}, outputs: []*output.InternalWrappedEvent{ }, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{ {OperatorsResult: &operators.Result{
Matches: map[string][]string{"tomcat": {}}, Matches: map[string][]string{"tomcat": {}},
@ -148,14 +148,14 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
}}, }},
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "apache"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ }, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "apache"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
secondInput = input secondInput = input.Input
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
}}}}}}, }}}}}},
}} }}
engine := &Engine{} engine := &Engine{}
matched := engine.executeWorkflow("https://test.com", workflow) matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
require.False(t, matched, "could not get correct match value") require.False(t, matched, "could not get correct match value")
require.Equal(t, "https://test.com", firstInput, "could not get correct first input") require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
@ -164,7 +164,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
type mockExecuter struct { type mockExecuter struct {
result bool result bool
executeHook func(input string) executeHook func(input *contextargs.MetaInput)
outputs []*output.InternalWrappedEvent outputs []*output.InternalWrappedEvent
} }
@ -181,7 +181,7 @@ func (m *mockExecuter) Requests() int {
// Execute executes the protocol group and returns true or false if results were found. // Execute executes the protocol group and returns true or false if results were found.
func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) { func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) {
if m.executeHook != nil { if m.executeHook != nil {
m.executeHook(input.Input) m.executeHook(input.MetaInput)
} }
return m.result, nil return m.result, nil
} }
@ -189,7 +189,7 @@ func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
if m.executeHook != nil { if m.executeHook != nil {
m.executeHook(input.Input) m.executeHook(input.MetaInput)
} }
for _, output := range m.outputs { for _, output := range m.outputs {
callback(output) callback(output)

10
v2/pkg/protocols/common/automaticscan/automaticscan.go
View File

@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
@ -132,11 +133,12 @@ func (s *Service) executeWappalyzerTechDetection() error {
// Iterate through each target making http request and identifying fingerprints // Iterate through each target making http request and identifying fingerprints
inputPool := s.engine.WorkPool().InputPool(types.HTTPProtocol) inputPool := s.engine.WorkPool().InputPool(types.HTTPProtocol)
s.target.Scan(func(value string) bool { s.target.Scan(func(value *contextargs.MetaInput) bool {
inputPool.WaitGroup.Add() inputPool.WaitGroup.Add()
go func(input string) { go func(input *contextargs.MetaInput) {
defer inputPool.WaitGroup.Done() defer inputPool.WaitGroup.Done()
s.processWappalyzerInputPair(input) s.processWappalyzerInputPair(input)
}(value) }(value)
return true return true
@ -145,8 +147,8 @@ func (s *Service) executeWappalyzerTechDetection() error {
return nil return nil
} }
func (s *Service) processWappalyzerInputPair(input string) { func (s *Service) processWappalyzerInputPair(input *contextargs.MetaInput) {
req, err := retryablehttp.NewRequest(http.MethodGet, input, nil) req, err := retryablehttp.NewRequest(http.MethodGet, input.Input, nil)
if err != nil { if err != nil {
return return
} }

19
v2/pkg/protocols/common/contextargs/contextargs.go
View File

@ -9,8 +9,9 @@ import (
// Context implements a shared context struct to share information across multiple templates within a workflow // Context implements a shared context struct to share information across multiple templates within a workflow
type Context struct { type Context struct {
// Input target for the executor // Meta is the target for the executor
Input string MetaInput *MetaInput
// CookieJar shared within workflow's http templates // CookieJar shared within workflow's http templates
CookieJar *cookiejar.Jar CookieJar *cookiejar.Jar
@ -22,12 +23,12 @@ type Context struct {
// Create a new contextargs instance // Create a new contextargs instance
func New() *Context { func New() *Context {
return &Context{} return &Context{MetaInput: &MetaInput{}}
} }
// Create a new contextargs instance with input string // Create a new contextargs instance with input string
func NewWithInput(input string) *Context { func NewWithInput(input string) *Context {
return &Context{Input: input} return &Context{MetaInput: &MetaInput{Input: input}}
} }
func (ctx *Context) initialize() { func (ctx *Context) initialize() {
@ -107,3 +108,13 @@ func (ctx *Context) Has(key string) bool {
func (ctx *Context) HasArgs() bool { func (ctx *Context) HasArgs() bool {
return ctx.hasArgs() return ctx.hasArgs()
} }
func (ctx *Context) Clone() *Context {
newCtx := &Context{
MetaInput: ctx.MetaInput.Clone(),
RWMutex: ctx.RWMutex,
args: ctx.args,
CookieJar: ctx.CookieJar,
}
return newCtx
}

69
v2/pkg/protocols/common/contextargs/metainput.go Normal file
View File

@ -0,0 +1,69 @@
package contextargs
import (
"bytes"
"fmt"
"strings"
jsoniter "github.com/json-iterator/go"
)
// MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput)
type MetaInput struct {
// Input represent the target
Input string
// CustomIP to use for connection
CustomIP string
}
func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) {
var b bytes.Buffer
err := jsoniter.NewEncoder(&b).Encode(metaInput)
return b, err
}
// ID returns a unique id/hash for metainput
func (metaInput *MetaInput) ID() string {
if metaInput.CustomIP != "" {
return fmt.Sprintf("%s-%s", metaInput.Input, metaInput.CustomIP)
}
return metaInput.Input
}
func (metaInput *MetaInput) MarshalString() (string, error) {
b, err := metaInput.marshalToBuffer()
return b.String(), err
}
func (metaInput *MetaInput) MustMarshalString() string {
marshaled, _ := metaInput.MarshalString()
return marshaled
}
func (metaInput *MetaInput) MarshalBytes() ([]byte, error) {
b, err := metaInput.marshalToBuffer()
return b.Bytes(), err
}
func (metaInput *MetaInput) MustMarshalBytes() []byte {
marshaled, _ := metaInput.MarshalBytes()
return marshaled
}
func (metaInput *MetaInput) Unmarshal(data string) error {
return jsoniter.NewDecoder(strings.NewReader(data)).Decode(metaInput)
}
func (metaInput *MetaInput) Clone() *MetaInput {
return &MetaInput{
Input: metaInput.Input,
CustomIP: metaInput.CustomIP,
}
}
func (metaInput *MetaInput) PrettyPrint() string {
if metaInput.CustomIP != "" {
return fmt.Sprintf("%s [%s]", metaInput.Input, metaInput.CustomIP)
}
return metaInput.Input
}

24
v2/pkg/protocols/common/executer/executer.go
View File

@ -70,14 +70,14 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
} }
previous := make(map[string]interface{}) previous := make(map[string]interface{})
for _, req := range e.requests { for _, req := range e.requests {
inputItem := *input inputItem := input.Clone()
if e.options.InputHelper != nil && input.Input != "" { if e.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.Input = e.options.InputHelper.Transform(input.Input, req.Type()); inputItem.Input == "" { if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" {
return false, nil return false, nil
} }
} }
err := req.ExecuteWithResults(&inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
ID := req.GetID() ID := req.GetID()
if ID != "" { if ID != "" {
builder := &strings.Builder{} builder := &strings.Builder{}
@ -108,9 +108,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
}) })
if err != nil { if err != nil {
if e.options.HostErrorsCache != nil { if e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.Input, err) e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
} }
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err) gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err)
} }
// If a match was found and stop at first match is set, break out of the loop and return // If a match was found and stop at first match is set, break out of the loop and return
if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {
@ -134,14 +134,14 @@ func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback proto
for _, req := range e.requests { for _, req := range e.requests {
req := req req := req
inputItem := *input inputItem := input.Clone()
if e.options.InputHelper != nil && input.Input != "" { if e.options.InputHelper != nil && input.MetaInput.Input != "" {
if inputItem.Input = e.options.InputHelper.Transform(input.Input, req.Type()); inputItem.Input == "" { if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" {
return nil return nil
} }
} }
err := req.ExecuteWithResults(&inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
ID := req.GetID() ID := req.GetID()
if ID != "" { if ID != "" {
builder := &strings.Builder{} builder := &strings.Builder{}
@ -161,9 +161,9 @@ func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback proto
}) })
if err != nil { if err != nil {
if e.options.HostErrorsCache != nil { if e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.Input, err) e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
} }
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err) gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err)
} }
// If a match was found and stop at first match is set, break out of the loop and return // If a match was found and stop at first match is set, break out of the loop and return
if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {

8
v2/pkg/protocols/dns/request.go
View File

@ -34,10 +34,10 @@ func (request *Request) Type() templateTypes.ProtocolType {
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Parse the URL and return domain if URL. // Parse the URL and return domain if URL.
var domain string var domain string
if utils.IsURL(input.Input) { if utils.IsURL(input.MetaInput.Input) {
domain = extractDomain(input.Input) domain = extractDomain(input.MetaInput.Input)
} else { } else {
domain = input.Input domain = input.MetaInput.Input
} }
var err error var err error
@ -112,7 +112,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
} }
} }
outputEvent := request.responseToDSLMap(compiledRequest, response, input.Input, input.Input, traceData) outputEvent := request.responseToDSLMap(compiledRequest, response, input.MetaInput.Input, input.MetaInput.Input, traceData)
for k, v := range previous { for k, v := range previous {
outputEvent[k] = v outputEvent[k] = v
} }

3
v2/pkg/protocols/dns/request_test.go
View File

@ -54,8 +54,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
t.Run("domain-valid", func(t *testing.T) { t.Run("domain-valid", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput("example.com")
ctxArgs.Input = "example.com"
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })

10
v2/pkg/protocols/file/request.go
View File

@ -47,7 +47,7 @@ var emptyResultErr = errors.New("Empty result")
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize) wg := sizedwaitgroup.New(request.options.Options.BulkSize)
err := request.getInputPaths(input.Input, func(filePath string) { err := request.getInputPaths(input.MetaInput.Input, func(filePath string) {
wg.Add() wg.Add()
func(filePath string) { func(filePath string) {
defer wg.Done() defer wg.Done()
@ -63,7 +63,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
// every new file in the compressed multi-file archive counts 1 // every new file in the compressed multi-file archive counts 1
request.options.Progress.AddToTotal(1) request.options.Progress.AddToTotal(1)
archiveFileName := filepath.Join(filePath, file.Name()) archiveFileName := filepath.Join(filePath, file.Name())
event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.Input, file.Size(), previous) event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous)
if err != nil { if err != nil {
if errors.Is(err, emptyResultErr) { if errors.Is(err, emptyResultErr) {
// no matches but one file elaborated // no matches but one file elaborated
@ -116,7 +116,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
_ = tmpFileOut.Sync() _ = tmpFileOut.Sync()
// rewind the file // rewind the file
_, _ = tmpFileOut.Seek(0, 0) _, _ = tmpFileOut.Seek(0, 0)
event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.Input, fileStat.Size(), previous) event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous)
if err != nil { if err != nil {
if errors.Is(err, emptyResultErr) { if errors.Is(err, emptyResultErr) {
// no matches but one file elaborated // no matches but one file elaborated
@ -136,7 +136,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
default: default:
// normal file - increments the counter by 1 // normal file - increments the counter by 1
request.options.Progress.AddToTotal(1) request.options.Progress.AddToTotal(1)
event, fileMatches, err := request.processFile(filePath, input.Input, previous) event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous)
if err != nil { if err != nil {
if errors.Is(err, emptyResultErr) { if errors.Is(err, emptyResultErr) {
// no matches but one file elaborated // no matches but one file elaborated
@ -158,7 +158,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
wg.Wait() wg.Wait()
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err) request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request") return errors.Wrap(err, "could not send file request")
} }

3
v2/pkg/protocols/file/request_test.go
View File

@ -66,8 +66,7 @@ func TestFileExecuteWithResults(t *testing.T) {
t.Run("valid", func(t *testing.T) { t.Run("valid", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput(tempDir)
ctxArgs.Input = tempDir
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })

2
v2/pkg/protocols/headless/request.go
View File

@ -31,7 +31,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
inputURL := input.Input inputURL := input.MetaInput.Input
if request.options.Browser.UserAgent() == "" { if request.options.Browser.UserAgent() == "" {
request.options.Browser.SetUserAgent(request.compiledUserAgent) request.options.Browser.SetUserAgent(request.compiledUserAgent)
} }

87
v2/pkg/protocols/http/request.go
View File

@ -48,7 +48,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
// executeRaceRequest executes race condition request for a URL // executeRaceRequest executes race condition request for a URL
func (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
reqURL := input.Input reqURL := input.MetaInput.Input
var generatedRequests []*generatedRequest var generatedRequests []*generatedRequest
// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
@ -59,7 +59,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
if !ok { if !ok {
return nil return nil
} }
requestForDump, err := generator.Make(context.Background(), reqURL, inputData, payloads, nil) ctx := request.newContext(input)
requestForDump, err := generator.Make(ctx, reqURL, inputData, payloads, nil)
if err != nil { if err != nil {
return err return err
} }
@ -87,7 +88,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
if !ok { if !ok {
break break
} }
generatedRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, nil) ctx := request.newContext(input)
generatedRequest, err := generator.Make(ctx, reqURL, inputData, payloads, nil)
if err != nil { if err != nil {
return err return err
} }
@ -130,7 +132,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
if !ok { if !ok {
break break
} }
generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues) ctx := request.newContext(input)
generatedHttpRequest, err := generator.Make(ctx, input.MetaInput.Input, inputData, payloads, dynamicValues)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
break break
@ -138,8 +141,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err return err
} }
if input.Input == "" { if input.MetaInput.Input == "" {
input.Input = generatedHttpRequest.URL() input.MetaInput.Input = generatedHttpRequest.URL()
} }
swg.Add() swg.Add()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
@ -166,7 +169,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
generator := request.newGenerator(false) generator := request.newGenerator(false)
// need to extract the target from the url // need to extract the target from the url
URL, err := url.Parse(input.Input) URL, err := url.Parse(input.MetaInput.Input)
if err != nil { if err != nil {
return err return err
} }
@ -197,13 +200,14 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
if !ok { if !ok {
break break
} }
generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues) ctx := request.newContext(input)
generatedHttpRequest, err := generator.Make(ctx, input.MetaInput.Input, inputData, payloads, dynamicValues)
if err != nil { if err != nil {
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err return err
} }
if input.Input == "" { if input.MetaInput.Input == "" {
input.Input = generatedHttpRequest.URL() input.MetaInput.Input = generatedHttpRequest.URL()
} }
generatedHttpRequest.pipelinedClient = pipeClient generatedHttpRequest.pipelinedClient = pipeClient
swg.Add() swg.Add()
@ -225,14 +229,14 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
// executeFuzzingRule executes fuzzing request for a URL // executeFuzzingRule executes fuzzing request for a URL
func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
parsed, err := url.Parse(input.Input) parsed, err := url.Parse(input.MetaInput.Input)
if err != nil { if err != nil {
return errors.Wrap(err, "could not parse url") return errors.Wrap(err, "could not parse url")
} }
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool { fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
hasInteractMarkers := len(gr.InteractURLs) > 0 hasInteractMarkers := len(gr.InteractURLs) > 0
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) { if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
return false return false
} }
@ -266,7 +270,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
} }
if requestErr != nil { if requestErr != nil {
if request.options.HostErrorsCache != nil { if request.options.HostErrorsCache != nil {
request.options.HostErrorsCache.MarkFailed(input.Input, requestErr) request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)
} }
} }
request.options.Progress.IncrementRequests() request.options.Progress.IncrementRequests()
@ -285,7 +289,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
if !result { if !result {
break break
} }
generated, err := generator.Make(context.Background(), input.Input, value, payloads, nil) generated, err := generator.Make(context.Background(), input.MetaInput.Input, value, payloads, nil)
if err != nil { if err != nil {
continue continue
} }
@ -346,10 +350,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
request.options.RateLimiter.Take() request.options.RateLimiter.Take()
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(request.options.Options.Timeout)*time.Second) ctx := request.newContext(input)
ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
defer cancel() defer cancel()
generatedHttpRequest, err := generator.Make(ctx, input.Input, data, payloads, dynamicValue) generatedHttpRequest, err := generator.Make(ctxWithTimeout, input.MetaInput.Input, data, payloads, dynamicValue)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
return true, nil return true, nil
@ -367,11 +372,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...) generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...)
} }
hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0 hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
if input.Input == "" { if input.MetaInput.Input == "" {
input.Input = generatedHttpRequest.URL() input.MetaInput.Input = generatedHttpRequest.URL()
} }
// Check if hosts keep erroring // Check if hosts keep erroring
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) { if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.ID()) {
return true, nil return true, nil
} }
var gotMatches bool var gotMatches bool
@ -400,7 +405,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
} }
if err != nil { if err != nil {
if request.options.HostErrorsCache != nil { if request.options.HostErrorsCache != nil {
request.options.HostErrorsCache.MarkFailed(input.Input, err) request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
} }
requestErr = err requestErr = err
} }
@ -472,7 +477,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if !generatedRequest.original.Race { if !generatedRequest.original.Race {
var dumpError error var dumpError error
// TODO: dump is currently not working with post-processors - somehow it alters the signature // TODO: dump is currently not working with post-processors - somehow it alters the signature
dumpedRequest, dumpError = dump(generatedRequest, input.Input) dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input)
if dumpError != nil { if dumpError != nil {
return dumpError return dumpError
} }
@ -480,12 +485,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil { if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr) gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
return errStopExecution return errStopExecution
} }
} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr) gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
return errStopExecution return errStopExecution
} }
} }
@ -499,7 +504,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if parsed, parseErr := url.Parse(formedURL); parseErr == nil { if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host hostname = parsed.Host
} }
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
} else if generatedRequest.request != nil { } else if generatedRequest.request != nil {
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
} }
@ -507,7 +512,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
formedURL = generatedRequest.rawRequest.FullURL formedURL = generatedRequest.rawRequest.FullURL
// use request url as matched url if empty // use request url as matched url if empty
if formedURL == "" { if formedURL == "" {
formedURL = input.Input formedURL = input.MetaInput.Input
if generatedRequest.rawRequest.Path != "" { if generatedRequest.rawRequest.Path != "" {
formedURL = fmt.Sprintf("%s%s", formedURL, generatedRequest.rawRequest.Path) formedURL = fmt.Sprintf("%s%s", formedURL, generatedRequest.rawRequest.Path)
} }
@ -520,7 +525,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes
options.ForceReadAllBody = request.ForceReadAllBody options.ForceReadAllBody = request.ForceReadAllBody
options.SNI = request.options.Options.SNI options.SNI = request.options.Options.SNI
resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
} else { } else {
hostname = generatedRequest.request.URL.Host hostname = generatedRequest.request.URL.Host
formedURL = generatedRequest.request.URL.String() formedURL = generatedRequest.request.URL.String()
@ -553,13 +558,13 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
} }
// use request url as matched url if empty // use request url as matched url if empty
if formedURL == "" { if formedURL == "" {
formedURL = input.Input formedURL = input.MetaInput.Input
} }
// Dump the requests containing all headers // Dump the requests containing all headers
if !generatedRequest.original.Race { if !generatedRequest.original.Race {
var dumpError error var dumpError error
dumpedRequest, dumpError = dump(generatedRequest, input.Input) dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input)
if dumpError != nil { if dumpError != nil {
return dumpError return dumpError
} }
@ -572,7 +577,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
gologger.Print().Msgf("%s", dumpedRequestString) gologger.Print().Msgf("%s", dumpedRequestString)
} }
if request.options.Options.StoreResponse { if request.options.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString)) request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString))
} }
} }
} }
@ -588,11 +593,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// If we have interactsh markers and request times out, still send // If we have interactsh markers and request times out, still send
// a callback event so in case we receive an interaction, correlation is possible. // a callback event so in case we receive an interaction, correlation is possible.
if hasInteractMatchers { if hasInteractMatchers {
outputEvent := request.responseToDSLMap(&http.Response{}, input.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 { if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i] hostname = hostname[:i]
} }
if input.MetaInput.CustomIP != "" {
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent} event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil { if request.CompiledOperators != nil {
@ -672,7 +682,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if response.resp == nil { if response.resp == nil {
continue // Skip nil responses continue // Skip nil responses
} }
matchedURL := input.Input matchedURL := input.MetaInput.Input
if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
matchedURL = generatedRequest.rawRequest.FullURL matchedURL = generatedRequest.rawRequest.FullURL
} }
@ -687,12 +697,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
} }
finalEvent := make(output.InternalEvent) finalEvent := make(output.InternalEvent)
outputEvent := request.responseToDSLMap(response.resp, input.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 { if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i] hostname = hostname[:i]
} }
outputEvent["curl-command"] = curlCommand outputEvent["curl-command"] = curlCommand
if input.MetaInput.CustomIP != "" {
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
}
if request.options.Interactsh != nil { if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent) request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent)
} }
@ -723,7 +737,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
responseContentType := resp.Header.Get("Content-Type") responseContentType := resp.Header.Get("Content-Type")
isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize
dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.Input) dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
callback(event) callback(event)
@ -842,3 +856,10 @@ func (request *Request) pruneSignatureInternalValues(maps ...map[string]interfac
} }
} }
} }
func (request *Request) newContext(input *contextargs.Context) context.Context {
if input.MetaInput.CustomIP != "" {
return context.WithValue(context.Background(), "ip", input.MetaInput.CustomIP) //nolint
}
return context.Background()
}

3
v2/pkg/protocols/http/request_test.go
View File

@ -82,8 +82,7 @@ Disallow: /c`))
t.Run("test", func(t *testing.T) { t.Run("test", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput(ts.URL)
ctxArgs.Input = ts.URL
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched { if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matchCount++ matchCount++

6
v2/pkg/protocols/network/request.go
View File

@ -43,10 +43,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
if request.SelfContained { if request.SelfContained {
address = "" address = ""
} else { } else {
address, err = getAddress(input.Input) address, err = getAddress(input.MetaInput.Input)
} }
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err) request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not get address from url") return errors.Wrap(err, "could not get address from url")
} }
@ -57,7 +57,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
variables = generators.MergeMaps(variablesMap, variables) variables = generators.MergeMaps(variablesMap, variables)
actualAddress := replacer.Replace(kv.address, variables) actualAddress := replacer.Replace(kv.address, variables)
if err := request.executeAddress(variables, actualAddress, address, input.Input, kv.tls, previous, callback); err != nil { if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil {
gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err) gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err)
continue continue
} }

9
v2/pkg/protocols/network/request_test.go
View File

@ -65,8 +65,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
t.Run("domain-valid", func(t *testing.T) { t.Run("domain-valid", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput(parsed.Host)
ctxArgs.Input = parsed.Host
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
@ -82,8 +81,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
t.Run("invalid-port-override", func(t *testing.T) { t.Run("invalid-port-override", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput("127.0.0.1:11211")
ctxArgs.Input = "127.0.0.1:11211"
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
@ -97,8 +95,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
t.Run("hex-to-string", func(t *testing.T) { t.Run("hex-to-string", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput(parsed.Host)
ctxArgs.Input = parsed.Host
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })

4
v2/pkg/protocols/offlinehttp/request.go
View File

@ -32,7 +32,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize) wg := sizedwaitgroup.New(request.options.Options.BulkSize)
err := request.getInputPaths(input.Input, func(data string) { err := request.getInputPaths(input.MetaInput.Input, func(data string) {
wg.Add() wg.Add()
go func(data string) { go func(data string) {
@ -98,7 +98,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
}) })
wg.Wait() wg.Wait()
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, input.Input, "file", err) request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, "file", err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request") return errors.Wrap(err, "could not send file request")
} }

20
v2/pkg/protocols/ssl/ssl.go
View File

@ -127,7 +127,7 @@ func (request *Request) GetID() string {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
address, err := getAddress(input.Input) address, err := getAddress(input.MetaInput.Input)
if err != nil { if err != nil {
return nil return nil
} }
@ -153,7 +153,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
if dataErr != nil { if dataErr != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), dataErr) requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1) requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions") return errors.Wrap(dataErr, "could not evaluate template expressions")
} }
@ -165,7 +165,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
response, err := request.tlsx.Connect(host, host, port) response, err := request.tlsx.Connect(host, host, port)
if err != nil { if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), err) requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1) requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server") return errors.Wrap(err, "could not connect to server")
} }
@ -174,12 +174,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
gologger.Verbose().Msgf("Sent SSL request to %s", address) gologger.Verbose().Msgf("Sent SSL request to %s", address)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.Input) msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.MetaInput.Input)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
gologger.Debug().Str("address", input.Input).Msg(msg) gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg)
} }
if requestOptions.Options.StoreResponse { if requestOptions.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), msg) request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg)
} }
} }
@ -192,7 +192,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
data["response"] = jsonDataString data["response"] = jsonDataString
data["host"] = input data["host"] = input
data["matched"] = addressToDial data["matched"] = addressToDial
if input.MetaInput.CustomIP != "" {
data["ip"] = input.MetaInput.CustomIP
} else {
data["ip"] = request.dialer.GetDialedIP(hostname) data["ip"] = request.dialer.GetDialedIP(hostname)
}
data["template-path"] = requestOptions.TemplatePath data["template-path"] = requestOptions.TemplatePath
data["template-id"] = requestOptions.TemplateID data["template-id"] = requestOptions.TemplateID
data["template-info"] = requestOptions.TemplateInfo data["template-info"] = requestOptions.TemplateInfo
@ -220,13 +224,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.Input) msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
gologger.Debug().Msg(msg) gologger.Debug().Msg(msg)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false)) gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
} }
if requestOptions.Options.StoreResponse { if requestOptions.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString)) request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
} }
} }
callback(event) callback(event)

3
v2/pkg/protocols/ssl/ssl_test.go
View File

@ -28,8 +28,7 @@ func TestSSLProtocol(t *testing.T) {
require.Nil(t, err, "could not compile ssl request") require.Nil(t, err, "could not compile ssl request")
var gotEvent output.InternalEvent var gotEvent output.InternalEvent
ctxArgs := contextargs.New() ctxArgs := contextargs.NewWithInput("google.com:443")
ctxArgs.Input = "google.com:443"
err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) { err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
gotEvent = event.InternalEvent gotEvent = event.InternalEvent
}) })

6
v2/pkg/protocols/websocket/websocket.go
View File

@ -137,7 +137,7 @@ func (request *Request) GetID() string {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
hostname, err := getAddress(input.Input) hostname, err := getAddress(input.MetaInput.Input)
if err != nil { if err != nil {
return err return err
} }
@ -150,13 +150,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if !ok { if !ok {
break break
} }
if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil { if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil {
return err return err
} }
} }
} else { } else {
value := make(map[string]interface{}) value := make(map[string]interface{})
if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil { if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil {
return err return err
} }
} }

2
v2/pkg/protocols/whois/whois.go
View File

@ -85,7 +85,7 @@ func (request *Request) GetID() string {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// generate variables // generate variables
variables := generateVariables(input.Input) variables := generateVariables(input.MetaInput.Input)
if vardump.EnableVarDump { if vardump.EnableVarDump {
gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables))

4
v2/pkg/templates/cluster.go
View File

@ -208,7 +208,7 @@ func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) {
} }
}) })
if err != nil && e.options.HostErrorsCache != nil { if err != nil && e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.Input, err) e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err)
} }
return results, err return results, err
} }
@ -230,7 +230,7 @@ func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callbac
} }
}) })
if err != nil && e.options.HostErrorsCache != nil { if err != nil && e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input.Input, err) e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err)
} }
return err return err
} }

8
v2/pkg/types/types.go
View File

@ -272,12 +272,16 @@ type Options struct {
IncludeConditions goflags.StringSlice IncludeConditions goflags.StringSlice
// Custom Config Directory // Custom Config Directory
CustomConfigDir string CustomConfigDir string
// ConfigPath contains the config path (used by healthcheck)
ConfigPath string
// ScanAllIPs associated to a dns record
ScanAllIPs bool
// IPVersion to scan (4,6)
IPVersion goflags.StringSlice
// Github token used to clone/pull from private repos for custom templates // Github token used to clone/pull from private repos for custom templates
GithubToken string GithubToken string
// GithubTemplateRepo is the list of custom public/private templates github repos // GithubTemplateRepo is the list of custom public/private templates github repos
GithubTemplateRepo goflags.StringSlice GithubTemplateRepo goflags.StringSlice
ConfigPath string // Used by healthcheck
} }
func (options *Options) AddVarPayload(key string, value interface{}) { func (options *Options) AddVarPayload(key string, value interface{}) {