diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 0698fb96d..c08f52bfb 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -20,6 +20,7 @@ import ( _ "github.com/projectdiscovery/utils/pprof" stringsutil "github.com/projectdiscovery/utils/strings" "github.com/rs/xid" + "gopkg.in/yaml.v2" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger/levels" @@ -263,6 +264,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.InputFileMode, "input-mode", "im", "list", fmt.Sprintf("mode of input file (%v)", provider.SupportedInputFormats())), flagSet.BoolVarP(&options.FormatUseRequiredOnly, "required-only", "ro", false, "use only required fields in input format when generating requests"), flagSet.BoolVarP(&options.SkipFormatValidation, "skip-format-validation", "sfv", false, "skip format validation (like missing vars) when parsing input file"), + flagSet.BoolVarP(&options.VarsTextTemplating, "vars-text-templating", "vtt", false, "enable text templating for vars in input file (only for yaml input mode)"), + flagSet.StringSliceVarP(&options.VarsFilePaths, "var-file-paths", "vfp", nil, "list of yaml file contained vars to inject into yaml input", goflags.CommaSeparatedStringSliceOptions), ) flagSet.CreateGroup("templates", "Templates", @@ -571,6 +574,7 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started config.DefaultConfig.SetConfigDir(customConfigDir) readFlagsConfig(flagSet) } + if cfgFile != "" { if !fileutil.FileExists(cfgFile) { options.Logger.Fatal().Msgf("given config file '%s' does not exist", cfgFile) @@ -579,6 +583,41 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started if err := flagSet.MergeConfigFile(cfgFile); err != nil { options.Logger.Fatal().Msgf("Could not read config: %s\n", err) } + + if !options.Vars.IsEmpty() { + // Maybe we should add vars to the config file as well even if they are set via flags? + file, err := os.Open(cfgFile) + if err != nil { + gologger.Fatal().Msgf("Could not open config file: %s\n", err) + } + defer func() { + _ = file.Close() + }() + data := make(map[string]interface{}) + err = yaml.NewDecoder(file).Decode(&data) + if err != nil { + gologger.Fatal().Msgf("Could not decode config file: %s\n", err) + } + + variables := data["var"] + if variables != nil { + if varSlice, ok := variables.([]interface{}); ok { + for _, value := range varSlice { + if strVal, ok := value.(string); ok { + err = options.Vars.Set(strVal) + if err != nil { + gologger.Warning().Msgf("Could not set variable from config file: %s\n", err) + } + } else { + gologger.Warning().Msgf("Skipping non-string variable in config: %#v", value) + } + } + } else { + gologger.Warning().Msgf("No 'var' section found in config file: %s", cfgFile) + } + } + + } } if options.NewTemplatesDirectory != "" { config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory) diff --git a/go.mod b/go.mod index f8ac019fa..62f9ad0c4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/projectdiscovery/nuclei/v3 -go 1.24.1 +go 1.24.2 + +toolchain go1.24.4 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible @@ -44,17 +46,18 @@ require ( ) require ( - code.gitea.io/sdk/gitea v0.21.0 + carvel.dev/ytt v0.52.0 + code.gitea.io/sdk/gitea v0.17.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/DataDog/gostackparse v0.7.0 - github.com/Masterminds/semver/v3 v3.4.0 + github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/Mzack9999/goja v0.0.0-20250507184235-e46100e9c697 github.com/Mzack9999/goja_nodejs v0.0.0-20250507184139-66bcbf65c883 github.com/alitto/pond v1.9.2 github.com/antchfx/xmlquery v1.4.4 - github.com/antchfx/xpath v1.3.4 + github.com/antchfx/xpath v1.3.3 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go-v2 v1.36.5 github.com/aws/aws-sdk-go-v2/config v1.29.17 @@ -128,7 +131,6 @@ require ( dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.0 // indirect git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect - github.com/42wim/httpsig v1.2.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect @@ -255,6 +257,7 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 // indirect github.com/kataras/jwt v0.1.10 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.18.0 // indirect diff --git a/go.sum b/go.sum index ae2cdea3d..deb089746 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +carvel.dev/ytt v0.52.0 h1:tkJPL8Gun5snVfypNXbmMKwnbwMyspcTi3Ypyso3nRY= +carvel.dev/ytt v0.52.0/go.mod h1:QgmuU7E15EXW1r2wxTt7zExVz14IHwEG4WNMmaFBkJo= cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -270,8 +272,8 @@ cloud.google.com/go/vpcaccess v1.7.11/go.mod h1:a2cuAiSCI4TVK0Dt6/dRjf22qQvfY+po cloud.google.com/go/webrisk v1.9.11/go.mod h1:mK6M8KEO0ZI7VkrjCq3Tjzw4vYq+3c4DzlMUDVaiswE= cloud.google.com/go/websecurityscanner v1.6.11/go.mod h1:vhAZjksELSg58EZfUQ1BMExD+hxqpn0G0DuyCZQjiTg= cloud.google.com/go/workflows v1.12.10/go.mod h1:RcKqCiOmKs8wFUEf3EwWZPH5eHc7Oq0kamIyOUCk0IE= -code.gitea.io/sdk/gitea v0.21.0 h1:69n6oz6kEVHRo1+APQQyizkhrZrLsTLXey9142pfkD4= -code.gitea.io/sdk/gitea v0.21.0/go.mod h1:tnBjVhuKJCn8ibdyyhvUyxrR1Ca2KHEoTWoukNhXQPA= +code.gitea.io/sdk/gitea v0.17.0 h1:8JPBss4+Jf7AE1YcfyiGrngTXE8dFSG3si/bypsTH34= +code.gitea.io/sdk/gitea v0.17.0/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -281,8 +283,6 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= -github.com/42wim/httpsig v1.2.2 h1:ofAYoHUNs/MJOLqQ8hIxeyz2QxOz8qdSVvp3PX/oPgA= -github.com/42wim/httpsig v1.2.2/go.mod h1:P/UYo7ytNBFwc+dg35IubuAUIs8zj5zzFIgUCEl55WY= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= @@ -293,14 +293,14 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= @@ -310,6 +310,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= @@ -317,8 +319,8 @@ github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4p github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -391,9 +393,8 @@ github.com/antchfx/htmlquery v1.3.4 h1:Isd0srPkni2iNTWCwVj/72t7uCphFeor5Q8nCzj1j github.com/antchfx/htmlquery v1.3.4/go.mod h1:K9os0BwIEmLAvTqaNSua8tXLWRWZpocZIH73OzWQbwM= github.com/antchfx/xmlquery v1.4.4 h1:mxMEkdYP3pjKSftxss4nUHfjBhnMk4imGoR96FRY2dg= github.com/antchfx/xmlquery v1.4.4/go.mod h1:AEPEEPYE9GnA2mj5Ur2L5Q5/2PycJ0N9Fusrx9b12fc= +github.com/antchfx/xpath v1.3.3 h1:tmuPQa1Uye0Ym1Zn65vxPgfltWb/Lxu2jeqIGteJSRs= github.com/antchfx/xpath v1.3.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antchfx/xpath v1.3.4 h1:1ixrW1VnXd4HurCj7qnqnR0jo14g8JMe20Fshg1Vgz4= -github.com/antchfx/xpath v1.3.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= @@ -849,6 +850,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -952,6 +955,7 @@ github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -1018,6 +1022,10 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57 h1:CwBRArr+BWBopnUJhDjJw86rPL/jGbEjfHWKzTasSqE= +github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57/go.mod h1:B0xN2MiNBGWOWi9CcfAo9LBI8IU4J1utlbOIJCsmKr4= +github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 h1:4bcRTTSx+LKSxMWibIwzHnDNmaN1x52oEpvnjCy+8vk= +github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368/go.mod h1:lKGj1op99m4GtQISxoD2t+K+WO/q2NzEPKvfXFQfbCA= github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ= github.com/kataras/jwt v0.1.10/go.mod h1:xkimAtDhU/aGlQqjwvgtg+VyuPwMiyZHaY8LJRh0mYo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= @@ -1659,6 +1667,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= @@ -1919,6 +1928,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/integration_tests/fuzz/fuzz-body.yaml b/integration_tests/fuzz/fuzz-body.yaml new file mode 100644 index 000000000..01a3006f6 --- /dev/null +++ b/integration_tests/fuzz/fuzz-body.yaml @@ -0,0 +1,38 @@ +id: fuzz-body + +info: + name: fuzzing error sqli payloads in http req body + author: pdteam + severity: info + description: | + This template attempts to find SQL injection vulnerabilities by fuzzing http body + It automatically handles and parses json,xml,multipart form and x-www-form-urlencoded data + and performs fuzzing on the value of every key + +http: + - pre-condition: + - type: dsl + dsl: + - method != "GET" + - method != "HEAD" + condition: and + + payloads: + injection: + - "'" + - "\"" + - ";" + + fuzzing: + - part: body + type: postfix + mode: single + fuzz: + - '{{injection}}' + + stop-at-first-match: true + matchers: + - type: word + words: + - "unrecognized token:" + - "null" diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index a680888ae..c7798286a 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -28,6 +28,12 @@ type InputFormatOptions struct { // RequiredOnly only uses required fields when generating requests // instead of all fields RequiredOnly bool + // VarsTextTemplating uses Variables and inject it into the input + // this is used for text templating of variables based on carvel ytt + // Only available for Yaml formats + VarsTextTemplating bool + // VarsFilePaths is the path to the file containing variables + VarsFilePaths []string } // Format is an interface implemented by all input formats diff --git a/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml new file mode 100644 index 000000000..b40ece639 --- /dev/null +++ b/pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml @@ -0,0 +1,25 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:json", "json") + +#@ def get_value(key, default=""): +#@ if hasattr(data.values, key): +#@ return str(getattr(data.values, key)) +#@ else: +#@ return default +#@ end +#@ end + +timestamp: 2024-02-20T19:24:13+05:32 +url: https://ginandjuice.shop/users/3 +request: + #@yaml/text-templated-strings + raw: |+ + POST /users/3 HTTP/1.1 + Host: ginandjuice.shop + Authorization: Bearer (@= get_value("token", "3x4mpl3t0k3n") @) + Accept-Encoding: gzip + Content-Type: application/x-www-form-urlencoded + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 + + foo=(@= json.encode(data.values.foo) @)&bar=(@= get_value("bar") @)&debug=(@= get_value("debug", "false") @) \ No newline at end of file diff --git a/pkg/input/formats/testdata/ytt/ytt-profile.yaml b/pkg/input/formats/testdata/ytt/ytt-profile.yaml new file mode 100644 index 000000000..01e794ad4 --- /dev/null +++ b/pkg/input/formats/testdata/ytt/ytt-profile.yaml @@ -0,0 +1,11 @@ +list: pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml +input-mode: yaml +templates: + - integration_tests/fuzz/fuzz-body.yaml +var: + - debug=true + - bar=bar +vars-text-templating: true +var-file-paths: + - pkg/input/formats/testdata/ytt/ytt-vars.yaml +dast: true \ No newline at end of file diff --git a/pkg/input/formats/testdata/ytt/ytt-vars.yaml b/pkg/input/formats/testdata/ytt/ytt-vars.yaml new file mode 100644 index 000000000..625ba7ddb --- /dev/null +++ b/pkg/input/formats/testdata/ytt/ytt-vars.yaml @@ -0,0 +1,3 @@ +token: foobar +foo: + bar: baz \ No newline at end of file diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index 6d75e0334..a77375c77 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -1,8 +1,8 @@ package yaml import ( + "bytes" "io" - "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -46,23 +46,41 @@ func (j *YamlMultiDocFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { - decoder := YamlUtil.NewDecoder(input) + finalInput := input + + // Apply text templating if enabled + if j.opts.VarsTextTemplating { + data, err := io.ReadAll(input) + if err != nil { + return errors.Wrap(err, "could not read input") + } + tpl := []string{string(data)} + dvs := mapToKeyValueSlice(j.opts.Variables) + finalData, err := ytt(tpl, dvs, j.opts.VarsFilePaths) + if err != nil { + return errors.Wrap(err, "could not apply ytt templating") + } + finalInput = bytes.NewReader(finalData) + } + + decoder := YamlUtil.NewDecoder(finalInput) for { var request proxifyRequest - err := decoder.Decode(&request) - if err == io.EOF { - break + if err := decoder.Decode(&request); err != nil { + if err == io.EOF { + break + } + return errors.Wrap(err, "could not decode yaml file") } - if err != nil { - return errors.Wrap(err, "could not decode json file") - } - if strings.TrimSpace(request.Request.Raw) == "" { + + raw := request.Request.Raw + if raw == "" { continue } - rawRequest, err := types.ParseRawRequestWithURL(request.Request.Raw, request.URL) + rawRequest, err := types.ParseRawRequestWithURL(raw, request.URL) if err != nil { - gologger.Warning().Msgf("multidoc-yaml: Could not parse raw request %s: %s\n", request.URL, err) + gologger.Warning().Msgf("multidoc-yaml: Could not parse raw request %s: %s", request.URL, err) continue } resultsCb(rawRequest) diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index efc7733ea..40d078a0a 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -2,8 +2,10 @@ package yaml import ( "os" + "strings" "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" "github.com/stretchr/testify/require" ) @@ -33,3 +35,48 @@ func TestYamlFormatterParse(t *testing.T) { require.Len(t, urls, len(expectedUrls), "invalid number of urls") require.ElementsMatch(t, urls, expectedUrls, "invalid urls") } + +func TestYamlFormatterParseWithVariables(t *testing.T) { + format := New() + proxifyYttFile := "../testdata/ytt/ginandjuice.ytt.yaml" + + expectedUrls := []string{ + "https://ginandjuice.shop/users/3", + } + + format.SetOptions(formats.InputFormatOptions{ + VarsTextTemplating: true, + Variables: map[string]interface{}{ + "foo": "catalog", + "bar": "product", + }, + }) + file, err := os.Open(proxifyYttFile) + require.Nilf(t, err, "error opening proxify ytt input file: %v", err) + defer func() { + _ = file.Close() + }() + + var urls []string + err = format.Parse(file, func(request *types.RequestResponse) bool { + urls = append(urls, request.URL.String()) + expectedRaw := `POST /users/3 HTTP/1.1 +Host: ginandjuice.shop +Authorization: Bearer 3x4mpl3t0k3n +Accept-Encoding: gzip +Content-Type: application/x-www-form-urlencoded +Connection: close +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 + +foo="catalog"&bar=product&debug=false` + normalised := strings.ReplaceAll(request.Request.Raw, "\r\n", "\n") + require.Equal(t, expectedRaw, strings.TrimSuffix(normalised, "\n"), "request raw does not match expected value") + + return false + }, proxifyYttFile) + + require.Nilf(t, err, "error parsing yaml file: %v", err) + require.Len(t, urls, len(expectedUrls), "invalid number of urls") + require.ElementsMatch(t, urls, expectedUrls, "invalid urls") + +} diff --git a/pkg/input/formats/yaml/ytt.go b/pkg/input/formats/yaml/ytt.go new file mode 100644 index 000000000..faaf6ccdc --- /dev/null +++ b/pkg/input/formats/yaml/ytt.go @@ -0,0 +1,70 @@ +package yaml + +import ( + "fmt" + "strings" + + yttcmd "carvel.dev/ytt/pkg/cmd/template" + yttui "carvel.dev/ytt/pkg/cmd/ui" + yttfiles "carvel.dev/ytt/pkg/files" + "gopkg.in/yaml.v2" +) + +func ytt(tpl, dvs []string, varFiles []string) ([]byte, error) { + // create and invoke ytt "template" command + templatingOptions := yttcmd.NewOptions() + + input, err := templatesAsInput(tpl...) + if err != nil { + return nil, err + } + + if len(varFiles) > 0 { + // Load vaarFiles into the templating options. + templatingOptions.DataValuesFlags.FromFiles = varFiles + } + + // equivalent to `--data-value-yaml` + templatingOptions.DataValuesFlags.KVsFromYAML = dvs + + // for in-memory use, pipe output to "/dev/null" + noopUI := yttui.NewCustomWriterTTY(false, noopWriter{}, noopWriter{}) + + // Evaluate the template given the configured data values... + output := templatingOptions.RunWithFiles(input, noopUI) + if output.Err != nil { + return nil, output.Err + } + + return output.DocSet.AsBytes() +} + +// templatesAsInput conveniently wraps one or more strings, each in a files.File, into a template.Input. +func templatesAsInput(tpl ...string) (yttcmd.Input, error) { + var files []*yttfiles.File + for i, t := range tpl { + // to make this less brittle, you'll probably want to use well-defined names for `path`, here, for each input. + // this matters when you're processing errors which report based on these paths. + file, err := yttfiles.NewFileFromSource(yttfiles.NewBytesSource(fmt.Sprintf("tpl%d.yml", i), []byte(t))) + if err != nil { + return yttcmd.Input{}, err + } + + files = append(files, file) + } + + return yttcmd.Input{Files: files}, nil +} + +func mapToKeyValueSlice(m map[string]interface{}) []string { + var result []string + for k, v := range m { + y, _ := yaml.Marshal(v) + result = append(result, fmt.Sprintf("%s=%s", k, strings.TrimSpace(string(y)))) + } + return result +} + +type noopWriter struct{} + +func (w noopWriter) Write(data []byte) (int, error) { return len(data), nil } diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index 32b923dd6..9e1d09ab2 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -116,6 +116,8 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { Variables: generators.MergeMaps(extraVars, opts.Options.Vars.AsMap()), SkipFormatValidation: opts.Options.SkipFormatValidation, RequiredOnly: opts.Options.FormatUseRequiredOnly, + VarsTextTemplating: opts.Options.VarsTextTemplating, + VarsFilePaths: opts.Options.VarsFilePaths, }, }) } diff --git a/pkg/types/types.go b/pkg/types/types.go index 48c186c84..26e98ba69 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -421,6 +421,10 @@ type Options struct { FormatUseRequiredOnly bool // SkipFormatValidation is used to skip format validation SkipFormatValidation bool + // VarsTextTemplating is used to inject variables into yaml input files + VarsTextTemplating bool + // VarsFilePaths is used to inject variables into yaml input files from a file + VarsFilePaths goflags.StringSlice // PayloadConcurrency is the number of concurrent payloads to run per template PayloadConcurrency int // ProbeConcurrency is the number of concurrent http probes to run with httpx