From a0bd3b854ef8f3246b31384aa0d600be73320622 Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Thu, 12 Jun 2025 15:03:33 +0200 Subject: [PATCH 01/12] feat(templating): add vars templating into yaml inputs --- cmd/nuclei/main.go | 1 + go.mod | 8 ++- go.sum | 7 ++ pkg/input/formats/formats.go | 4 ++ .../formats/testdata/ginandjuice.ytt.yaml | 26 +++++++ pkg/input/formats/yaml/multidoc.go | 44 +++++++++--- pkg/input/formats/yaml/multidoc_test.go | 43 ++++++++++++ pkg/input/formats/yaml/ytt.go | 69 +++++++++++++++++++ pkg/input/provider/interface.go | 1 + pkg/types/types.go | 2 + 10 files changed, 192 insertions(+), 13 deletions(-) create mode 100644 pkg/input/formats/testdata/ginandjuice.ytt.yaml create mode 100644 pkg/input/formats/yaml/ytt.go diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 8cc2b53f1..e1a3bfb1c 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -260,6 +260,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.InputFileMode, "input-mode", "im", "list", fmt.Sprintf("mode of input file (%v)", provider.SupportedInputFormats())), flagSet.BoolVarP(&options.FormatUseRequiredOnly, "required-only", "ro", false, "use only required fields in input format when generating requests"), flagSet.BoolVarP(&options.SkipFormatValidation, "skip-format-validation", "sfv", false, "skip format validation (like missing vars) when parsing input file"), + flagSet.BoolVarP(&options.VarsTextTemplating, "vars-text-templating", "vtt", false, "enable text templating for vars in input file (only for yaml input mode)"), ) flagSet.CreateGroup("templates", "Templates", diff --git a/go.mod b/go.mod index 23ff55738..80b35383f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/projectdiscovery/nuclei/v3 -go 1.23.0 +go 1.24.2 -toolchain go1.24.1 +toolchain go1.24.3 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible @@ -119,6 +119,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect + carvel.dev/ytt v0.52.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect @@ -184,7 +185,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -194,6 +195,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 // indirect github.com/kataras/jwt v0.1.10 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/pgzip v1.2.6 // indirect diff --git a/go.sum b/go.sum index 3c80d1e5d..222127cdf 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +carvel.dev/ytt v0.52.0 h1:tkJPL8Gun5snVfypNXbmMKwnbwMyspcTi3Ypyso3nRY= +carvel.dev/ytt v0.52.0/go.mod h1:QgmuU7E15EXW1r2wxTt7zExVz14IHwEG4WNMmaFBkJo= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -577,6 +579,8 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -652,6 +656,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 h1:4bcRTTSx+LKSxMWibIwzHnDNmaN1x52oEpvnjCy+8vk= +github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368/go.mod h1:lKGj1op99m4GtQISxoD2t+K+WO/q2NzEPKvfXFQfbCA= github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ= github.com/kataras/jwt v0.1.10/go.mod h1:xkimAtDhU/aGlQqjwvgtg+VyuPwMiyZHaY8LJRh0mYo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -1386,6 +1392,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index 03c65d3fe..a23ff5d06 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -28,6 +28,10 @@ type InputFormatOptions struct { // RequiredOnly only uses required fields when generating requests // instead of all fields RequiredOnly bool + // VarsTextTemplating uses Variables and inject it into the input + // this is used for text templating of variables based on carvel ytt + // Only available for Yaml formats + VarsTextTemplating bool } // Format is an interface implemented by all input formats diff --git a/pkg/input/formats/testdata/ginandjuice.ytt.yaml b/pkg/input/formats/testdata/ginandjuice.ytt.yaml new file mode 100644 index 000000000..f1889a67f --- /dev/null +++ b/pkg/input/formats/testdata/ginandjuice.ytt.yaml @@ -0,0 +1,26 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") + +#@ def get_value(key, default=""): +#@ if hasattr(data.values, key): +#@ value = getattr(data.values, key) +#@ return str(value) +#@ else: +#@ return default +#@ end +#@ end + +timestamp: 2024-02-20T19:24:13+05:32 +url: https://ginandjuice.shop/users/3 +request: + #@yaml/text-templated-strings + raw: |+ + POST /users/3 HTTP/1.1 + Host: ginandjuice.shop + Authorization: Bearer 3x4mpl3t0k3n + Accept-Encoding: gzip + Content-Type: application/x-www-form-urlencoded + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 + + foo=(@= get_value("foo") @)&bar=(@= get_value("bar") @) \ No newline at end of file diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index 6d75e0334..a6ae8067a 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -46,23 +46,47 @@ func (j *YamlMultiDocFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { - decoder := YamlUtil.NewDecoder(input) + finalInput := input + + // Apply text templating if enabled + if j.opts.VarsTextTemplating { + data, err := io.ReadAll(input) + if err != nil { + return errors.Wrap(err, "could not read input") + } + tpl := []string{string(data)} + dvs := mapToKeyValueSlice(j.opts.Variables) + finalInput, err = ytt(tpl, dvs) + if err != nil { + return errors.Wrap(err, "could not apply ytt templating") + } + finalData, err := io.ReadAll(finalInput) + if err != nil { + return errors.Wrap(err, "could not read templated input") + } + gologger.Debug().Msgf("Templated YAML content: %s", string(finalData)) + finalInput = strings.NewReader(string(finalData)) + + } + + decoder := YamlUtil.NewDecoder(finalInput) for { var request proxifyRequest - err := decoder.Decode(&request) - if err == io.EOF { - break + if err := decoder.Decode(&request); err != nil { + if err == io.EOF { + break + } + return errors.Wrap(err, "could not decode yaml file") } - if err != nil { - return errors.Wrap(err, "could not decode json file") - } - if strings.TrimSpace(request.Request.Raw) == "" { + + raw := strings.TrimSpace(request.Request.Raw) + if raw == "" { continue } - rawRequest, err := types.ParseRawRequestWithURL(request.Request.Raw, request.URL) + rawRequest, err := types.ParseRawRequestWithURL(raw, request.URL) if err != nil { - gologger.Warning().Msgf("multidoc-yaml: Could not parse raw request %s: %s\n", request.URL, err) + gologger.Warning().Msgf("multidoc-yaml: Could not parse raw request %s: %s", request.URL, err) continue } resultsCb(rawRequest) diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index 0b91e774a..063d0f13c 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" "github.com/stretchr/testify/require" ) @@ -31,3 +32,45 @@ func TestYamlFormatterParse(t *testing.T) { require.Len(t, urls, len(expectedUrls), "invalid number of urls") require.ElementsMatch(t, urls, expectedUrls, "invalid urls") } + +func TestYamlFormatterParseWithVariables(t *testing.T) { + format := New() + proxifyYttFile := "../testdata/ginandjuice.ytt.yaml" + + expectedUrls := []string{ + "https://ginandjuice.shop/users/3", + } + + format.SetOptions(formats.InputFormatOptions{ + VarsTextTemplating: true, + Variables: map[string]interface{}{ + "foo": "catalog", + "bar": "product", + }, + }) + file, err := os.Open(proxifyYttFile) + require.Nilf(t, err, "error opening proxify ytt input file: %v", err) + defer file.Close() + + var urls []string + err = format.Parse(file, func(request *types.RequestResponse) bool { + urls = append(urls, request.URL.String()) + expectedRaw := `POST /users/3 HTTP/1.1 +Host: ginandjuice.shop +Authorization: Bearer 3x4mpl3t0k3n +Accept-Encoding: gzip +Content-Type: application/x-www-form-urlencoded +Connection: close +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 + +foo=catalog&bar=product` + require.Equal(t, expectedRaw, request.Request.Raw, "request raw does not match expected value") + + return false + }, proxifyYttFile) + + require.Nilf(t, err, "error parsing yaml file: %v", err) + require.Len(t, urls, len(expectedUrls), "invalid number of urls") + require.ElementsMatch(t, urls, expectedUrls, "invalid urls") + +} diff --git a/pkg/input/formats/yaml/ytt.go b/pkg/input/formats/yaml/ytt.go new file mode 100644 index 000000000..8bbaea752 --- /dev/null +++ b/pkg/input/formats/yaml/ytt.go @@ -0,0 +1,69 @@ +package yaml + +import ( + "bytes" + "fmt" + "io" + + yttcmd "carvel.dev/ytt/pkg/cmd/template" + yttui "carvel.dev/ytt/pkg/cmd/ui" + yttfiles "carvel.dev/ytt/pkg/files" +) + +func ytt(tpl, dvs []string) (io.Reader, error) { + // create and invoke ytt "template" command + templatingOptions := yttcmd.NewOptions() + + input, err := templatesAsInput(tpl...) + if err != nil { + return nil, err + } + + // equivalent to `--data-value-yaml` + templatingOptions.DataValuesFlags.KVsFromYAML = dvs + + // for in-memory use, pipe output to "/dev/null" + noopUI := yttui.NewCustomWriterTTY(false, noopWriter{}, noopWriter{}) + + // Evaluate the template given the configured data values... + output := templatingOptions.RunWithFiles(input, noopUI) + if output.Err != nil { + return nil, output.Err + } + + // output.DocSet contains the full set of resulting YAML documents, in order. + bs, err := output.DocSet.AsBytes() + if err != nil { + return nil, err + } + return bytes.NewReader(bs), nil +} + +// templatesAsInput conveniently wraps one or more strings, each in a files.File, into a template.Input. +func templatesAsInput(tpl ...string) (yttcmd.Input, error) { + var files []*yttfiles.File + for i, t := range tpl { + // to make this less brittle, you'll probably want to use well-defined names for `path`, here, for each input. + // this matters when you're processing errors which report based on these paths. + file, err := yttfiles.NewFileFromSource(yttfiles.NewBytesSource(fmt.Sprintf("tpl%d.yml", i), []byte(t))) + if err != nil { + return yttcmd.Input{}, err + } + + files = append(files, file) + } + + return yttcmd.Input{Files: files}, nil +} + +func mapToKeyValueSlice(m map[string]interface{}) []string { + var result []string + for k, v := range m { + result = append(result, fmt.Sprintf("%s=%v", k, v)) + } + return result +} + +type noopWriter struct{} + +func (w noopWriter) Write(data []byte) (int, error) { return len(data), nil } diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index e6d5da14a..769af60fb 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -116,6 +116,7 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { Variables: generators.MergeMaps(extraVars, opts.Options.Vars.AsMap()), SkipFormatValidation: opts.Options.SkipFormatValidation, RequiredOnly: opts.Options.FormatUseRequiredOnly, + VarsTextTemplating: opts.Options.VarsTextTemplating, }, }) } diff --git a/pkg/types/types.go b/pkg/types/types.go index 41c95ef68..5b71225cb 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -419,6 +419,8 @@ type Options struct { FormatUseRequiredOnly bool // SkipFormatValidation is used to skip format validation SkipFormatValidation bool + // VarsTextTemplating is used to inject variables into yaml input files + VarsTextTemplating bool // PayloadConcurrency is the number of concurrent payloads to run per template PayloadConcurrency int // ProbeConcurrency is the number of concurrent http probes to run with httpx From 5f501da06365934023e1cba5d9fd2db1d9162aef Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Thu, 12 Jun 2025 15:44:11 +0200 Subject: [PATCH 02/12] fix: enhance code rabbit --- go.mod | 2 +- go.sum | 10 ++++++---- pkg/input/formats/yaml/multidoc.go | 1 - pkg/input/formats/yaml/multidoc_test.go | 4 +++- pkg/input/formats/yaml/ytt.go | 5 ++++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 80b35383f..3d60b4efe 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( ) require ( + carvel.dev/ytt v0.52.0 code.gitea.io/sdk/gitea v0.17.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 @@ -119,7 +120,6 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect - carvel.dev/ytt v0.52.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect diff --git a/go.sum b/go.sum index 222127cdf..4dcdb0f46 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzS github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1vBI4o= github.com/DataDog/gostackparse v0.6.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= @@ -515,6 +517,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -577,8 +581,6 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -656,6 +658,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57 h1:CwBRArr+BWBopnUJhDjJw86rPL/jGbEjfHWKzTasSqE= +github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57/go.mod h1:B0xN2MiNBGWOWi9CcfAo9LBI8IU4J1utlbOIJCsmKr4= github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 h1:4bcRTTSx+LKSxMWibIwzHnDNmaN1x52oEpvnjCy+8vk= github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368/go.mod h1:lKGj1op99m4GtQISxoD2t+K+WO/q2NzEPKvfXFQfbCA= github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ= @@ -864,8 +868,6 @@ github.com/projectdiscovery/cdncheck v1.1.17 h1:YSqKk05+UGSxmPIp7tlCvRegF63FUqO+ github.com/projectdiscovery/cdncheck v1.1.17/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.4.2 h1:9PnD6EyDAZFvpQmJ0700gkQ96Fqlzl+lnTdcVHAagXI= -github.com/projectdiscovery/dsl v0.4.2/go.mod h1:J1RizRF6O3lvk2v8p/tLAYqaxWg6N52OWc+uS5ZmO2U= github.com/projectdiscovery/dsl v0.4.3 h1:ZrbRkyK38hRiYMX7s6ohaTorDpq321ErqJuBUDmh49g= github.com/projectdiscovery/dsl v0.4.3/go.mod h1:cyt2IaYhS5SlyZ1D2BdK0QwIBXQW/u9zaBmRAKYKAmk= github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w= diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index a6ae8067a..679707672 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -64,7 +64,6 @@ func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRe if err != nil { return errors.Wrap(err, "could not read templated input") } - gologger.Debug().Msgf("Templated YAML content: %s", string(finalData)) finalInput = strings.NewReader(string(finalData)) } diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index 063d0f13c..3064e8a7d 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -2,6 +2,7 @@ package yaml import ( "os" + "strings" "testing" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" @@ -64,7 +65,8 @@ Connection: close User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 foo=catalog&bar=product` - require.Equal(t, expectedRaw, request.Request.Raw, "request raw does not match expected value") + normalised := strings.ReplaceAll(request.Request.Raw, "\r\n", "\n") + require.Equal(t, expectedRaw, strings.TrimSuffix(normalised, "\n"), "request raw does not match expected value") return false }, proxifyYttFile) diff --git a/pkg/input/formats/yaml/ytt.go b/pkg/input/formats/yaml/ytt.go index 8bbaea752..c075119ca 100644 --- a/pkg/input/formats/yaml/ytt.go +++ b/pkg/input/formats/yaml/ytt.go @@ -4,10 +4,12 @@ import ( "bytes" "fmt" "io" + "strings" yttcmd "carvel.dev/ytt/pkg/cmd/template" yttui "carvel.dev/ytt/pkg/cmd/ui" yttfiles "carvel.dev/ytt/pkg/files" + "gopkg.in/yaml.v2" ) func ytt(tpl, dvs []string) (io.Reader, error) { @@ -59,7 +61,8 @@ func templatesAsInput(tpl ...string) (yttcmd.Input, error) { func mapToKeyValueSlice(m map[string]interface{}) []string { var result []string for k, v := range m { - result = append(result, fmt.Sprintf("%s=%v", k, v)) + y, _ := yaml.Marshal(v) + result = append(result, fmt.Sprintf("%s=%s", k, strings.TrimSpace(string(y)))) } return result } From 3eb3f668971b2732f0714befe12abb2ba6690840 Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Fri, 13 Jun 2025 13:59:24 +0200 Subject: [PATCH 03/12] fix: change gologger runner version --- internal/runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 424d27116..365e708db 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -863,8 +863,8 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { return fmt.Sprintf("Current %s version: %v %v", versionType, version, updateutils.GetVersionDescription(version, latestVersion)) } - gologger.Info().Msgf(versionInfo(config.Version, cfg.LatestNucleiVersion, "nuclei")) - gologger.Info().Msgf(versionInfo(cfg.TemplateVersion, cfg.LatestNucleiTemplatesVersion, "nuclei-templates")) + gologger.Info().Msg(versionInfo(config.Version, cfg.LatestNucleiVersion, "nuclei")) + gologger.Info().Msg(versionInfo(cfg.TemplateVersion, cfg.LatestNucleiTemplatesVersion, "nuclei-templates")) if !HideAutoSaveMsg { if r.pdcpUploadErrMsg != "" { gologger.Print().Msgf("%s", r.pdcpUploadErrMsg) From 248548e075db5822710c12a5336d8503e4c8e040 Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Tue, 24 Jun 2025 18:32:45 +0200 Subject: [PATCH 04/12] feat(ytt): add ytt files var + add vars from cli and config --- cmd/nuclei/main.go | 31 +++++++++++++++ integration_tests/fuzz/fuzz-body.yaml | 38 +++++++++++++++++++ pkg/input/formats/formats.go | 2 + .../testdata/{ => ytt}/ginandjuice.ytt.yaml | 9 ++--- .../formats/testdata/ytt/ytt-profile.yaml | 10 +++++ pkg/input/formats/testdata/ytt/ytt-vars.yaml | 3 ++ pkg/input/formats/yaml/multidoc.go | 2 +- pkg/input/formats/yaml/ytt.go | 7 +++- pkg/input/provider/interface.go | 1 + pkg/types/types.go | 2 + 10 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 integration_tests/fuzz/fuzz-body.yaml rename pkg/input/formats/testdata/{ => ytt}/ginandjuice.ytt.yaml (71%) create mode 100644 pkg/input/formats/testdata/ytt/ytt-profile.yaml create mode 100644 pkg/input/formats/testdata/ytt/ytt-vars.yaml diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index e1a3bfb1c..13fb0d69d 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/utils/env" _ "github.com/projectdiscovery/utils/pprof" stringsutil "github.com/projectdiscovery/utils/strings" + "gopkg.in/yaml.v2" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -261,6 +262,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.FormatUseRequiredOnly, "required-only", "ro", false, "use only required fields in input format when generating requests"), flagSet.BoolVarP(&options.SkipFormatValidation, "skip-format-validation", "sfv", false, "skip format validation (like missing vars) when parsing input file"), flagSet.BoolVarP(&options.VarsTextTemplating, "vars-text-templating", "vtt", false, "enable text templating for vars in input file (only for yaml input mode)"), + flagSet.StringSliceVarP(&options.VarsFilePaths, "var-file-paths", "vfp", nil, "list of yaml file contained vars to inject into yaml input", goflags.CommaSeparatedStringSliceOptions), ) flagSet.CreateGroup("templates", "Templates", @@ -569,6 +571,7 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started config.DefaultConfig.SetConfigDir(customConfigDir) readFlagsConfig(flagSet) } + if cfgFile != "" { if !fileutil.FileExists(cfgFile) { gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile) @@ -577,6 +580,34 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started if err := flagSet.MergeConfigFile(cfgFile); err != nil { gologger.Fatal().Msgf("Could not read config: %s\n", err) } + + if !options.Vars.IsEmpty() { + // Maybe we should add vars to the config file as well? + file, err := os.Open(cfgFile) + if err != nil { + gologger.Fatal().Msgf("Could not open config file: %s\n", err) + } + defer file.Close() + data := make(map[string]interface{}) + err = yaml.NewDecoder(file).Decode(&data) + if err != nil { + gologger.Fatal().Msgf("Could not decode config file: %s\n", err) + } + + variables := data["var"] + if variables != nil { + for _, value := range variables.([]interface{}) { + if strVal, ok := value.(string); ok { + options.Vars.Set(strVal) + } else { + gologger.Warning().Msgf("Skipping non-string variable in config: %#v", value) + } + } + } else { + gologger.Warning().Msgf("No 'var' section found in config file: %s", cfgFile) + } + + } } if options.NewTemplatesDirectory != "" { config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory) diff --git a/integration_tests/fuzz/fuzz-body.yaml b/integration_tests/fuzz/fuzz-body.yaml new file mode 100644 index 000000000..01a3006f6 --- /dev/null +++ b/integration_tests/fuzz/fuzz-body.yaml @@ -0,0 +1,38 @@ +id: fuzz-body + +info: + name: fuzzing error sqli payloads in http req body + author: pdteam + severity: info + description: | + This template attempts to find SQL injection vulnerabilities by fuzzing http body + It automatically handles and parses json,xml,multipart form and x-www-form-urlencoded data + and performs fuzzing on the value of every key + +http: + - pre-condition: + - type: dsl + dsl: + - method != "GET" + - method != "HEAD" + condition: and + + payloads: + injection: + - "'" + - "\"" + - ";" + + fuzzing: + - part: body + type: postfix + mode: single + fuzz: + - '{{injection}}' + + stop-at-first-match: true + matchers: + - type: word + words: + - "unrecognized token:" + - "null" diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index a23ff5d06..acc6c31d4 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -32,6 +32,8 @@ type InputFormatOptions struct { // this is used for text templating of variables based on carvel ytt // Only available for Yaml formats VarsTextTemplating bool + // VarsFilePaths is the path to the file containing variables + VarsFilePaths []string } // Format is an interface implemented by all input formats diff --git a/pkg/input/formats/testdata/ginandjuice.ytt.yaml b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml similarity index 71% rename from pkg/input/formats/testdata/ginandjuice.ytt.yaml rename to pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml index f1889a67f..a43bca070 100644 --- a/pkg/input/formats/testdata/ginandjuice.ytt.yaml +++ b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml @@ -1,10 +1,9 @@ #@ load("@ytt:data", "data") -#@ load("@ytt:assert", "assert") +#@ load("@ytt:json", "json") #@ def get_value(key, default=""): #@ if hasattr(data.values, key): -#@ value = getattr(data.values, key) -#@ return str(value) +#@ return str(getattr(data.values, key)) #@ else: #@ return default #@ end @@ -17,10 +16,10 @@ request: raw: |+ POST /users/3 HTTP/1.1 Host: ginandjuice.shop - Authorization: Bearer 3x4mpl3t0k3n + Authorization: Bearer (@= get_value("token") @) Accept-Encoding: gzip Content-Type: application/x-www-form-urlencoded Connection: close User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 - foo=(@= get_value("foo") @)&bar=(@= get_value("bar") @) \ No newline at end of file + foo=(@= get_value("foo") @)&bar=(@= get_value("bar") @)&debug=(@= get_value("debug", "false") @) \ No newline at end of file diff --git a/pkg/input/formats/testdata/ytt/ytt-profile.yaml b/pkg/input/formats/testdata/ytt/ytt-profile.yaml new file mode 100644 index 000000000..2cad498bc --- /dev/null +++ b/pkg/input/formats/testdata/ytt/ytt-profile.yaml @@ -0,0 +1,10 @@ +list: pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml +input-mode: yaml +templates: + - integration_tests/fuzz/fuzz-body.yaml +var: + - debug=true +vars-text-templating: true +var-file-paths: + - pkg/input/formats/testdata/ytt/ytt-vars.yaml +dast: true \ No newline at end of file diff --git a/pkg/input/formats/testdata/ytt/ytt-vars.yaml b/pkg/input/formats/testdata/ytt/ytt-vars.yaml new file mode 100644 index 000000000..be973bee8 --- /dev/null +++ b/pkg/input/formats/testdata/ytt/ytt-vars.yaml @@ -0,0 +1,3 @@ +foo: tata +bar: foo +token: TOKEN-BEARER \ No newline at end of file diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index 679707672..1140d5509 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -56,7 +56,7 @@ func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRe } tpl := []string{string(data)} dvs := mapToKeyValueSlice(j.opts.Variables) - finalInput, err = ytt(tpl, dvs) + finalInput, err = ytt(tpl, dvs, j.opts.VarsFilePaths) if err != nil { return errors.Wrap(err, "could not apply ytt templating") } diff --git a/pkg/input/formats/yaml/ytt.go b/pkg/input/formats/yaml/ytt.go index c075119ca..3ccf5f6d2 100644 --- a/pkg/input/formats/yaml/ytt.go +++ b/pkg/input/formats/yaml/ytt.go @@ -12,7 +12,7 @@ import ( "gopkg.in/yaml.v2" ) -func ytt(tpl, dvs []string) (io.Reader, error) { +func ytt(tpl, dvs []string, varFiles []string) (io.Reader, error) { // create and invoke ytt "template" command templatingOptions := yttcmd.NewOptions() @@ -21,6 +21,11 @@ func ytt(tpl, dvs []string) (io.Reader, error) { return nil, err } + if len(varFiles) > 0 { + // Load vaarFiles into the templating options. + templatingOptions.DataValuesFlags.FromFiles = varFiles + } + // equivalent to `--data-value-yaml` templatingOptions.DataValuesFlags.KVsFromYAML = dvs diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index 769af60fb..c98214dd3 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -117,6 +117,7 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { SkipFormatValidation: opts.Options.SkipFormatValidation, RequiredOnly: opts.Options.FormatUseRequiredOnly, VarsTextTemplating: opts.Options.VarsTextTemplating, + VarsFilePaths: opts.Options.VarsFilePaths, }, }) } diff --git a/pkg/types/types.go b/pkg/types/types.go index 5b71225cb..47c434072 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -421,6 +421,8 @@ type Options struct { SkipFormatValidation bool // VarsTextTemplating is used to inject variables into yaml input files VarsTextTemplating bool + // VarsFilePaths is used to inject variables into yaml input files from a file + VarsFilePaths goflags.StringSlice // PayloadConcurrency is the number of concurrent payloads to run per template PayloadConcurrency int // ProbeConcurrency is the number of concurrent http probes to run with httpx From 1a9a7563c0c7f957d1cc8424ddc309b33af98126 Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Tue, 24 Jun 2025 18:39:29 +0200 Subject: [PATCH 05/12] feat: send struct from var file --- pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml | 2 +- pkg/input/formats/testdata/ytt/ytt-profile.yaml | 1 + pkg/input/formats/testdata/ytt/ytt-vars.yaml | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml index a43bca070..b409ad214 100644 --- a/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml +++ b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml @@ -22,4 +22,4 @@ request: Connection: close User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 - foo=(@= get_value("foo") @)&bar=(@= get_value("bar") @)&debug=(@= get_value("debug", "false") @) \ No newline at end of file + foo=(@= json.encode(data.values.foo) @)&bar=(@= get_value("bar") @)&debug=(@= get_value("debug", "false") @) \ No newline at end of file diff --git a/pkg/input/formats/testdata/ytt/ytt-profile.yaml b/pkg/input/formats/testdata/ytt/ytt-profile.yaml index 2cad498bc..01e794ad4 100644 --- a/pkg/input/formats/testdata/ytt/ytt-profile.yaml +++ b/pkg/input/formats/testdata/ytt/ytt-profile.yaml @@ -4,6 +4,7 @@ templates: - integration_tests/fuzz/fuzz-body.yaml var: - debug=true + - bar=bar vars-text-templating: true var-file-paths: - pkg/input/formats/testdata/ytt/ytt-vars.yaml diff --git a/pkg/input/formats/testdata/ytt/ytt-vars.yaml b/pkg/input/formats/testdata/ytt/ytt-vars.yaml index be973bee8..625ba7ddb 100644 --- a/pkg/input/formats/testdata/ytt/ytt-vars.yaml +++ b/pkg/input/formats/testdata/ytt/ytt-vars.yaml @@ -1,3 +1,3 @@ -foo: tata -bar: foo -token: TOKEN-BEARER \ No newline at end of file +token: foobar +foo: + bar: baz \ No newline at end of file From 99914e1a32f01a06c1e9781d99dcd22e4dd43853 Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Tue, 24 Jun 2025 18:45:35 +0200 Subject: [PATCH 06/12] fix code rabbit --- cmd/nuclei/main.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 13fb0d69d..a2663b505 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -582,7 +582,7 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started } if !options.Vars.IsEmpty() { - // Maybe we should add vars to the config file as well? + // Maybe we should add vars to the config file as well even if they are set via flags? file, err := os.Open(cfgFile) if err != nil { gologger.Fatal().Msgf("Could not open config file: %s\n", err) @@ -596,15 +596,17 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started variables := data["var"] if variables != nil { - for _, value := range variables.([]interface{}) { - if strVal, ok := value.(string); ok { - options.Vars.Set(strVal) - } else { - gologger.Warning().Msgf("Skipping non-string variable in config: %#v", value) + if varSlice, ok := variables.([]interface{}); ok { + for _, value := range varSlice { + if strVal, ok := value.(string); ok { + options.Vars.Set(strVal) + } else { + gologger.Warning().Msgf("Skipping non-string variable in config: %#v", value) + } } + } else { + gologger.Warning().Msgf("No 'var' section found in config file: %s", cfgFile) } - } else { - gologger.Warning().Msgf("No 'var' section found in config file: %s", cfgFile) } } From 937fa1252b6bb2bec051f4b9980e692188974ee7 Mon Sep 17 00:00:00 2001 From: Alban Stourbe Date: Thu, 26 Jun 2025 09:42:14 +0200 Subject: [PATCH 07/12] fix(main.go): add errcheck --- cmd/nuclei/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index a2663b505..2991fb81e 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -599,7 +599,10 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started if varSlice, ok := variables.([]interface{}); ok { for _, value := range varSlice { if strVal, ok := value.(string); ok { - options.Vars.Set(strVal) + err = options.Vars.Set(strVal) + if err != nil { + gologger.Warning().Msgf("Could not set variable from config file: %s\n", err) + } } else { gologger.Warning().Msgf("Skipping non-string variable in config: %#v", value) } From 8304462420c010edea514d3f12f9fa25d179f0ee Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 3 Jul 2025 16:50:21 +0200 Subject: [PATCH 08/12] retain required empty spaces --- pkg/input/formats/yaml/multidoc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index 1140d5509..3539b07f0 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -78,7 +78,7 @@ func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRe return errors.Wrap(err, "could not decode yaml file") } - raw := strings.TrimSpace(request.Request.Raw) + raw := request.Request.Raw if raw == "" { continue } From 4baf46f080488fda49d18c6b036caf84d928dd3c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 3 Jul 2025 17:05:14 +0200 Subject: [PATCH 09/12] fixing path --- pkg/input/formats/yaml/multidoc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index dd0264b81..1d9c44f2e 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -38,7 +38,7 @@ func TestYamlFormatterParse(t *testing.T) { func TestYamlFormatterParseWithVariables(t *testing.T) { format := New() - proxifyYttFile := "../testdata/ginandjuice.ytt.yaml" + proxifyYttFile := "../testdata/ytt/ginandjuice.ytt.yaml" expectedUrls := []string{ "https://ginandjuice.shop/users/3", From cf8d067feaac613edb1f7e956c8bdad06402cfc4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 3 Jul 2025 17:28:55 +0200 Subject: [PATCH 10/12] fixing test --- pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml | 2 +- pkg/input/formats/yaml/multidoc_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml index b409ad214..b40ece639 100644 --- a/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml +++ b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml @@ -16,7 +16,7 @@ request: raw: |+ POST /users/3 HTTP/1.1 Host: ginandjuice.shop - Authorization: Bearer (@= get_value("token") @) + Authorization: Bearer (@= get_value("token", "3x4mpl3t0k3n") @) Accept-Encoding: gzip Content-Type: application/x-www-form-urlencoded Connection: close diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index 1d9c44f2e..54efe0326 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -66,7 +66,7 @@ Content-Type: application/x-www-form-urlencoded Connection: close User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 -foo=catalog&bar=product` +foo="catalog"&bar=product&debug=false` normalised := strings.ReplaceAll(request.Request.Raw, "\r\n", "\n") require.Equal(t, expectedRaw, strings.TrimSuffix(normalised, "\n"), "request raw does not match expected value") From d55ab2f82783a7cfe222a798d1973a7783d22e5c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 3 Jul 2025 18:05:08 +0200 Subject: [PATCH 11/12] use bytes slice --- pkg/input/formats/yaml/multidoc.go | 11 +++-------- pkg/input/formats/yaml/ytt.go | 11 ++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index 3539b07f0..a77375c77 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -1,8 +1,8 @@ package yaml import ( + "bytes" "io" - "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -56,16 +56,11 @@ func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRe } tpl := []string{string(data)} dvs := mapToKeyValueSlice(j.opts.Variables) - finalInput, err = ytt(tpl, dvs, j.opts.VarsFilePaths) + finalData, err := ytt(tpl, dvs, j.opts.VarsFilePaths) if err != nil { return errors.Wrap(err, "could not apply ytt templating") } - finalData, err := io.ReadAll(finalInput) - if err != nil { - return errors.Wrap(err, "could not read templated input") - } - finalInput = strings.NewReader(string(finalData)) - + finalInput = bytes.NewReader(finalData) } decoder := YamlUtil.NewDecoder(finalInput) diff --git a/pkg/input/formats/yaml/ytt.go b/pkg/input/formats/yaml/ytt.go index 3ccf5f6d2..faaf6ccdc 100644 --- a/pkg/input/formats/yaml/ytt.go +++ b/pkg/input/formats/yaml/ytt.go @@ -1,9 +1,7 @@ package yaml import ( - "bytes" "fmt" - "io" "strings" yttcmd "carvel.dev/ytt/pkg/cmd/template" @@ -12,7 +10,7 @@ import ( "gopkg.in/yaml.v2" ) -func ytt(tpl, dvs []string, varFiles []string) (io.Reader, error) { +func ytt(tpl, dvs []string, varFiles []string) ([]byte, error) { // create and invoke ytt "template" command templatingOptions := yttcmd.NewOptions() @@ -38,12 +36,7 @@ func ytt(tpl, dvs []string, varFiles []string) (io.Reader, error) { return nil, output.Err } - // output.DocSet contains the full set of resulting YAML documents, in order. - bs, err := output.DocSet.AsBytes() - if err != nil { - return nil, err - } - return bytes.NewReader(bs), nil + return output.DocSet.AsBytes() } // templatesAsInput conveniently wraps one or more strings, each in a files.File, into a template.Input. From c487e596021792596f76a38ba46105dc56be25d6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 11 Sep 2025 21:41:59 +0200 Subject: [PATCH 12/12] lint --- cmd/nuclei/main.go | 4 +++- pkg/input/formats/yaml/multidoc_test.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 021d75d6a..c08f52bfb 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -590,7 +590,9 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started if err != nil { gologger.Fatal().Msgf("Could not open config file: %s\n", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() data := make(map[string]interface{}) err = yaml.NewDecoder(file).Decode(&data) if err != nil { diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index 54efe0326..40d078a0a 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -53,7 +53,9 @@ func TestYamlFormatterParseWithVariables(t *testing.T) { }) file, err := os.Open(proxifyYttFile) require.Nilf(t, err, "error opening proxify ytt input file: %v", err) - defer file.Close() + defer func() { + _ = file.Close() + }() var urls []string err = format.Parse(file, func(request *types.RequestResponse) bool {