diff --git a/integration_tests/code/test.json b/integration_tests/code/test.json new file mode 100644 index 000000000..247700a51 --- /dev/null +++ b/integration_tests/code/test.json @@ -0,0 +1,27 @@ +{ + "id": "go-integration-test", + "info": { + "name": "Basic Go Integration Test", + "author": "pdteam", + "severity": "info" + }, + "requests": [ + { + "method": "GET", + "path": [ + "{{BaseURL}}" + ], + "headers": { + "test": "nuclei" + }, + "matchers": [ + { + "type": "word", + "words": [ + "This is test headers matcher text" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index 8b8fe5b33..dc2ce1c31 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -36,6 +36,7 @@ import ( var codeTestcases = map[string]testutils.TestCase{ "code/test.yaml": &goIntegrationTest{}, + "code/test.json": &goIntegrationTest{}, } type goIntegrationTest struct{} diff --git a/v2/cmd/sign-templates/main.go b/v2/cmd/sign-templates/main.go new file mode 100644 index 000000000..3c1e800cb --- /dev/null +++ b/v2/cmd/sign-templates/main.go @@ -0,0 +1,148 @@ +package main + +import ( + "io/fs" + "log" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" + stringsutil "github.com/projectdiscovery/utils/strings" +) + +type options struct { + Templates goflags.StringSlice + Algorithm string + PrivateKeyName string + PrivateKeyPassPhrase string + PublicKeyName string +} + +func ParseOptions() (*options, error) { + opts := &options{} + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`sign-templates is a utility to perform template signature`) + + flagSet.CreateGroup("sign", "sign", + flagSet.StringSliceVarP(&opts.Templates, "templates", "t", nil, "templates files/folders to sign", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringVarP(&opts.Algorithm, "algorithm", "a", "ecdsa", "signature algorithm (ecdsa, rsa)"), + flagSet.StringVarP(&opts.PrivateKeyName, "private-key", "prk", "", "private key env var name or file location"), + flagSet.StringVarP(&opts.PrivateKeyPassPhrase, "private-key-pass", "prkp", "", "private key passphrase env var name or file location"), + flagSet.StringVarP(&opts.PublicKeyName, "public-key", "puk", "", "public key env var name or file location"), + ) + + if err := flagSet.Parse(); err != nil { + return nil, err + } + + return opts, nil +} + +func main() { + opts, err := ParseOptions() + if err != nil { + log.Fatalf("couldn't parse options: %s\n", err) + } + + var algo signer.AlgorithmType + switch opts.Algorithm { + case "rsa": + algo = signer.RSA + case "ecdsa": + algo = signer.ECDSA + default: + log.Fatal("unknown algorithm type") + } + + signerOptions := &signer.Options{ + PrivateKeyName: opts.PrivateKeyName, + PassphraseName: opts.PrivateKeyPassPhrase, + PublicKeyName: opts.PublicKeyName, + Algorithm: algo, + } + sign, err := signer.New(signerOptions) + if err != nil { + log.Fatalf("couldn't create crypto engine: %s\n", err) + } + + for _, templateItem := range opts.Templates { + if err := processItem(sign, templateItem); err != nil { + log.Fatalf("Could not walk directory: %s\n", err) + } + } +} + +func processItem(sign *signer.Signer, item string) error { + return filepath.WalkDir(item, func(iterItem string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + + if err := processFile(sign, iterItem); err != nil { + return err + } + + return nil + }) +} + +func processFile(sign *signer.Signer, filePath string) error { + ext := filepath.Ext(filePath) + if !stringsutil.EqualFoldAny(ext, ".yaml") { + return nil + } + err := signTemplate(sign, filePath) + if err != nil { + return errors.Wrapf(err, "could not sign template: %s", filePath) + } + + ok, err := verifyTemplateSignature(sign, filePath) + if err != nil { + return errors.Wrapf(err, "could not verify template: %s", filePath) + } + if !ok { + return errors.Wrapf(err, "template signature doesn't match: %s", filePath) + } + + return nil +} + +func appendToFile(path string, data []byte, digest string) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + if _, err := file.Write(data); err != nil { + return err + } + + if _, err := file.WriteString("\n" + digest); err != nil { + return err + } + return nil +} + +func signTemplate(sign *signer.Signer, templatePath string) error { + templateData, err := os.ReadFile(templatePath) + if err != nil { + return err + } + signatureData, err := signer.Sign(sign, templateData) + if err != nil { + return err + } + dataWithoutSignature := signer.RemoveSignatureFromData(templateData) + return appendToFile(templatePath, dataWithoutSignature, signatureData) +} + +func verifyTemplateSignature(sign *signer.Signer, templatePath string) (bool, error) { + templateData, err := os.ReadFile(templatePath) + if err != nil { + return false, err + } + return signer.Verify(sign, templateData) +} diff --git a/v2/go.mod b/v2/go.mod index b2b2cd77d..fc6df71fb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -62,11 +62,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.30.3 github.com/docker/go-units v0.5.0 github.com/fatih/structs v1.1.0 + github.com/go-faker/faker/v4 v4.0.0 github.com/go-git/go-git/v5 v5.5.2 github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-version v1.6.0 github.com/kataras/jwt v0.1.8 - github.com/klauspost/compress v1.15.15 + github.com/klauspost/compress v1.16.0 github.com/labstack/echo/v4 v4.10.0 github.com/mholt/archiver v3.1.1+incompatible github.com/mitchellh/go-homedir v1.1.0 @@ -74,6 +75,7 @@ require ( github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/httpx v1.2.7 + github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/nvd v1.0.9 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 @@ -82,7 +84,7 @@ require ( github.com/projectdiscovery/uncover v1.0.2 github.com/projectdiscovery/utils v0.0.10-0.20230217185600-008d111dd1c1 github.com/projectdiscovery/wappalyzergo v0.0.81 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -107,7 +109,7 @@ require ( github.com/karlseguin/expect v1.0.8 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/pjbgf/sha1cd v0.2.3 // indirect - github.com/projectdiscovery/asnmap v0.0.1 // indirect + github.com/projectdiscovery/asnmap v1.0.0 // indirect github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect github.com/projectdiscovery/freeport v0.0.4 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -187,8 +189,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect - github.com/projectdiscovery/mapcidr v1.0.3 + github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect github.com/projectdiscovery/networkpolicy v0.0.4 github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect @@ -209,7 +210,7 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.23.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.5.0 golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/mod v0.8.0 // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 54d7045c6..85323435c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -210,6 +210,8 @@ github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4x github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-faker/faker/v4 v4.0.0 h1:tfgFaeizVlYGOS1tVo/vcWcKhkNgG1NWm8ibRG0f+aQ= +github.com/go-faker/faker/v4 v4.0.0/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= @@ -395,8 +397,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= @@ -519,10 +521,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/projectdiscovery/asnmap v0.0.1 h1:n4YCz1ljUaDA3dOUCkjI/bUOtiS7ge1KJ39qpURCd/o= -github.com/projectdiscovery/asnmap v0.0.1/go.mod h1:CjCVDhQPVtmlE247L6YFeIVX9c4m8pOX8V8BmB0JkX8= -github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= -github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= +github.com/projectdiscovery/asnmap v1.0.0 h1:h9aUEHT3gEWgeTxDCd0UMxBw1yPthDwL1ogqUWnkBXo= +github.com/projectdiscovery/asnmap v1.0.0/go.mod h1:m+qedSAZERz7Ds942hWANjU9kg4ADAikFHfwSXN1Ey8= +github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ/zkotVodSJLOch3pV/UYt1vQcwyIs5HX0sm1ljE= +github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.12 h1:KLYJxpiwEFidduU4PbcwEcCQ2L7c5wrf7DI5IN5fZ+8= @@ -547,8 +549,8 @@ github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053 h1:8 github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053/go.mod h1:7lLz3Rt+Lxt8xhK0EUYkgxoa9RXRL3honxHeAu+ivuk= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= -github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= -github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= +github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oTFX/xzGhVU= +github.com/projectdiscovery/mapcidr v1.1.0/go.mod h1:hck0bWXka5ZkUaBG+TWt99bzLy+4hAg9oANhEmm3GNs= github.com/projectdiscovery/networkpolicy v0.0.4 h1:zcGjEqZbyECZEdyCy1jVuwOS7Ww1mzgCefQU75XqdJA= github.com/projectdiscovery/networkpolicy v0.0.4/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w= github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE= @@ -565,7 +567,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.12-0.20230220094538-f406add578 github.com/projectdiscovery/retryablehttp-go v1.0.12-0.20230220094538-f406add578ab/go.mod h1:dnJK347etdVKz7ljhemUhBK6EpDnVf0U1O8dGj7MM+0= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= -github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.0.5 h1:ZDMcwqjwXB0x2XBzvdra7HYiN8yLGBhHc5qE2mjJchM= @@ -655,8 +656,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8a3d01ffe..6fb109491 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -70,7 +70,7 @@ type Runner struct { catalog catalog.Catalog progress progress.Progress colorizer aurora.Aurora - issuesClient *reporting.Client + issuesClient reporting.Client hmapInputProvider *hybrid.Input browser *engine.Browser ratelimiter *ratelimit.Limiter diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 30693adb7..2cbc9c84b 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -3,15 +3,13 @@ package filter import ( "testing" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - - "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/stretchr/testify/require" ) func TestTagBasedFilter(t *testing.T) { diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 78f820c52..dc2ab33d3 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -17,7 +17,7 @@ import ( "github.com/projectdiscovery/hmap/filekv" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/mapcidr" - asn "github.com/projectdiscovery/mapcidr/asn" + "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover" @@ -330,8 +330,7 @@ func (i *Input) expandCIDRInputValue(value string) { // expandASNInputValue expands CIDRs for given ASN and stores expanded IPs func (i *Input) expandASNInputValue(value string) { - asnClient := asn.New() - cidrs, _ := asnClient.GetCIDRsForASNNum(value) + cidrs, _ := asn.GetCIDRsForASNNum(value) for _, cidr := range cidrs { i.expandCIDRInputValue(cidr.String()) } diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go index a64ec1f0b..1a9c92236 100644 --- a/v2/pkg/core/workflow_execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -3,8 +3,6 @@ package core import ( "testing" - "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -13,6 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "github.com/stretchr/testify/require" ) func TestWorkflowsSimple(t *testing.T) { diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index 2a31329ff..8a3710824 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -47,7 +47,7 @@ type Info struct { // examples: // - value: > // []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"} - Reference stringslice.StringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` + Reference stringslice.RawStringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` // description: | // Severity of the template. SeverityHolder severity.Holder `json:"severity,omitempty" yaml:"severity,omitempty"` diff --git a/v2/pkg/model/model_test.go b/v2/pkg/model/model_test.go index d706cdce8..aba1cf1c9 100644 --- a/v2/pkg/model/model_test.go +++ b/v2/pkg/model/model_test.go @@ -7,10 +7,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" - - "gopkg.in/yaml.v2" - "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" ) func TestInfoJsonMarshal(t *testing.T) { @@ -20,7 +18,7 @@ func TestInfoJsonMarshal(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.StringSlice{Value: "reference1"}, + Reference: stringslice.NewRaw("Reference1"), Metadata: map[string]interface{}{ "string_key": "string_value", "array_key": []string{"array_value1", "array_value2"}, @@ -33,7 +31,7 @@ func TestInfoJsonMarshal(t *testing.T) { result, err := json.Marshal(&info) assert.Nil(t, err) - expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"reference1","severity":"high","metadata":{"array_key":["array_value1","array_value2"],"map_key":{"key1":"val1"},"string_key":"string_value"}}` + expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"Reference1","severity":"high","metadata":{"array_key":["array_value1","array_value2"],"map_key":{"key1":"val1"},"string_key":"string_value"}}` assert.Equal(t, expected, string(result)) } @@ -44,7 +42,7 @@ func TestInfoYamlMarshal(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.StringSlice{Value: "reference1"}, + Reference: stringslice.NewRaw("Reference1"), Metadata: map[string]interface{}{ "string_key": "string_value", "array_key": []string{"array_value1", "array_value2"}, @@ -65,7 +63,7 @@ tags: - cve - misc description: Test description -reference: reference1 +reference: Reference1 severity: high metadata: array_key: diff --git a/v2/pkg/model/types/severity/severity.go b/v2/pkg/model/types/severity/severity.go index b235375b6..c186ec44b 100644 --- a/v2/pkg/model/types/severity/severity.go +++ b/v2/pkg/model/types/severity/severity.go @@ -113,7 +113,7 @@ func (severityHolder *Holder) UnmarshalJSON(data []byte) error { return nil } -func (severityHolder *Holder) MarshalJSON() ([]byte, error) { +func (severityHolder Holder) MarshalJSON() ([]byte, error) { return json.Marshal(severityHolder.Severity.String()) } diff --git a/v2/pkg/model/types/stringslice/stringslice.go b/v2/pkg/model/types/stringslice/stringslice.go index 018acc614..8474ef31a 100644 --- a/v2/pkg/model/types/stringslice/stringslice.go +++ b/v2/pkg/model/types/stringslice/stringslice.go @@ -16,6 +16,10 @@ type StringSlice struct { Value interface{} } +func New(value interface{}) StringSlice { + return StringSlice{Value: value} +} + func (stringSlice StringSlice) JSONSchemaType() *jsonschema.Type { gotType := &jsonschema.Type{ OneOf: []*jsonschema.Type{{Type: "string"}, {Type: "array"}}, @@ -52,13 +56,13 @@ func (stringSlice *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) result := make([]string, 0, len(marshalledSlice)) for _, value := range marshalledSlice { - result = append(result, stringSlice.normalize(value)) + result = append(result, stringSlice.Normalize(value)) } stringSlice.Value = result return nil } -func (stringSlice StringSlice) normalize(value string) string { +func (stringSlice StringSlice) Normalize(value string) string { return strings.ToLower(strings.TrimSpace(value)) } @@ -94,7 +98,7 @@ func (stringSlice *StringSlice) UnmarshalJSON(data []byte) error { values := make([]string, 0, len(result)) for _, value := range result { - values = append(values, stringSlice.normalize(value)) + values = append(values, stringSlice.Normalize(value)) } stringSlice.Value = values return nil diff --git a/v2/pkg/model/types/stringslice/stringslice_raw.go b/v2/pkg/model/types/stringslice/stringslice_raw.go new file mode 100644 index 000000000..8ca38c5e8 --- /dev/null +++ b/v2/pkg/model/types/stringslice/stringslice_raw.go @@ -0,0 +1,13 @@ +package stringslice + +type RawStringSlice struct { + StringSlice +} + +func NewRaw(value interface{}) RawStringSlice { + return RawStringSlice{StringSlice: StringSlice{Value: value}} +} + +func (rawStringSlice RawStringSlice) Normalize(value string) string { + return value +} diff --git a/v2/pkg/model/types/userAgent/user_agent.go b/v2/pkg/model/types/userAgent/user_agent.go index 998bb67a6..30173ca2b 100644 --- a/v2/pkg/model/types/userAgent/user_agent.go +++ b/v2/pkg/model/types/userAgent/user_agent.go @@ -86,6 +86,20 @@ func (userAgentHolder *UserAgentHolder) UnmarshalYAML(unmarshal func(interface{} return nil } +func (userAgentHolder *UserAgentHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedUserAgent, err := toUserAgent(s) + if err != nil { + return err + } + + userAgentHolder.Value = computedUserAgent + return nil +} + func (userAgentHolder *UserAgentHolder) MarshalJSON() ([]byte, error) { return json.Marshal(userAgentHolder.Value.String()) } diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index c996a2e7d..c7f9c2e6e 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -49,20 +49,18 @@ const ( letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" ) -var invalidDslFunctionError = errors.New("invalid DSL function signature") -var invalidDslFunctionMessageTemplate = "%w. correct method signature %q" - -var dslFunctions map[string]dslFunction - var ( + ErrinvalidDslFunction = errors.New("invalid DSL function signature") + dslFunctions map[string]dslFunction + // FunctionNames is a list of function names for expression evaluation usages FunctionNames []string // HelperFunctions is a pre-compiled list of govaluate DSL functions HelperFunctions map[string]govaluate.ExpressionFunction -) -var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`) -var dateFormatRegex = regexp.MustCompile("%([A-Za-z])") + functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`) + dateFormatRegex = regexp.MustCompile("%([A-Za-z])") +) type dslFunction struct { signatures []string @@ -98,7 +96,7 @@ func init() { func(args ...interface{}) (interface{}, error) { argCount := len(args) if argCount == 0 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } else if argCount == 1 { runes := []rune(types.ToString(args[0])) sort.Slice(runes, func(i int, j int) bool { @@ -122,7 +120,7 @@ func init() { func(args ...interface{}) (interface{}, error) { argCount := len(args) if argCount == 0 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } else if argCount == 1 { builder := &strings.Builder{} visited := make(map[rune]struct{}) @@ -149,7 +147,7 @@ func init() { "repeat": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { count, err := strconv.Atoi(types.ToString(args[1])) if err != nil { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } return strings.Repeat(types.ToString(args[0]), count), nil }), @@ -243,7 +241,7 @@ func init() { argumentsSize := len(arguments) if argumentsSize < 1 && argumentsSize > 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } currentTime, err := getCurrentTimeFromUserInput(arguments) @@ -353,7 +351,7 @@ func init() { "(str string, prefix ...string) bool", func(args ...interface{}) (interface{}, error) { if len(args) < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } for _, prefix := range args[1:] { if strings.HasPrefix(types.ToString(args[0]), types.ToString(prefix)) { @@ -366,7 +364,7 @@ func init() { "line_starts_with": makeDslWithOptionalArgsFunction( "(str string, prefix ...string) bool", func(args ...interface{}) (interface{}, error) { if len(args) < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } for _, line := range strings.Split(types.ToString(args[0]), "\n") { for _, prefix := range args[1:] { @@ -382,7 +380,7 @@ func init() { "(str string, suffix ...string) bool", func(args ...interface{}) (interface{}, error) { if len(args) < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } for _, suffix := range args[1:] { if strings.HasSuffix(types.ToString(args[0]), types.ToString(suffix)) { @@ -395,7 +393,7 @@ func init() { "line_ends_with": makeDslWithOptionalArgsFunction( "(str string, suffix ...string) bool", func(args ...interface{}) (interface{}, error) { if len(args) < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } for _, line := range strings.Split(types.ToString(args[0]), "\n") { for _, suffix := range args[1:] { @@ -436,11 +434,11 @@ func init() { separator := types.ToString(arguments[1]) count, err := strconv.Atoi(types.ToString(arguments[2])) if err != nil { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } return strings.SplitN(input, separator, count), nil } else { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } }, ), @@ -450,7 +448,7 @@ func init() { func(arguments ...interface{}) (interface{}, error) { argumentsSize := len(arguments) if argumentsSize < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } else if argumentsSize == 2 { separator := types.ToString(arguments[0]) elements, ok := arguments[1].([]string) @@ -495,7 +493,7 @@ func init() { argSize := len(args) if argSize != 0 && argSize != 1 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } if argSize >= 1 { @@ -516,7 +514,7 @@ func init() { argSize := len(args) if argSize < 1 || argSize > 3 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } length = int(args[0].(float64)) @@ -538,7 +536,7 @@ func init() { argSize := len(args) if argSize != 1 && argSize != 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } length = int(args[0].(float64)) @@ -558,7 +556,7 @@ func init() { argSize := len(args) if argSize != 1 && argSize != 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } length = int(args[0].(float64)) @@ -575,7 +573,7 @@ func init() { func(args ...interface{}) (interface{}, error) { argSize := len(args) if argSize != 1 && argSize != 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } length := int(args[0].(float64)) @@ -594,7 +592,7 @@ func init() { func(args ...interface{}) (interface{}, error) { argSize := len(args) if argSize > 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } min := 0 @@ -613,7 +611,7 @@ func init() { "(cidr ...string) string", func(args ...interface{}) (interface{}, error) { if len(args) == 0 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } var cidrs []string for _, arg := range args { @@ -635,7 +633,7 @@ func init() { argSize := len(args) if argSize != 0 && argSize != 1 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } else if argSize == 1 { seconds = int(args[0].(float64)) } @@ -670,7 +668,7 @@ func init() { } return parsedTime.Unix(), err } else { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } }, ), @@ -678,7 +676,7 @@ func init() { "(seconds uint)", func(args ...interface{}) (interface{}, error) { if len(args) != 1 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } seconds := args[0].(float64) time.Sleep(time.Duration(seconds) * time.Second) @@ -689,7 +687,7 @@ func init() { "(firstVersion, constraints ...string) bool", func(args ...interface{}) (interface{}, error) { if len(args) < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } firstParsed, parseErr := version.NewVersion(types.ToString(args[0])) @@ -713,7 +711,7 @@ func init() { "(args ...interface{})", func(args ...interface{}) (interface{}, error) { if len(args) < 1 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) return true, nil @@ -753,7 +751,7 @@ func init() { "(str string, start int, optionalEnd int)", func(args ...interface{}) (interface{}, error) { if len(args) < 2 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } argStr := types.ToString(args[0]) start, err := strconv.Atoi(types.ToString(args[1])) @@ -817,7 +815,7 @@ func init() { argSize := len(args) if argSize < 1 || argSize > 4 { - return nil, invalidDslFunctionError + return nil, ErrinvalidDslFunction } jsonString := args[0].(string) @@ -968,7 +966,7 @@ func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.Expressi []string{signature}, func(args ...interface{}) (interface{}, error) { if len(args) != numberOfParameters { - return nil, fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature) + return nil, fmt.Errorf("%w. correct method signature %q", ErrinvalidDslFunction, signature) } return dslFunctionLogic(args...) }, diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 05cd10314..b9e908a8f 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -53,7 +53,7 @@ func TestDSLGzipSerialize(t *testing.T) { func TestDslFunctionSignatures(t *testing.T) { createSignatureError := func(signature string) string { - return fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature).Error() + return fmt.Errorf("%w. correct method signature %q", ErrinvalidDslFunction, signature).Error() } toUpperSignatureError := createSignatureError("to_upper(arg1 interface{}) interface{}") diff --git a/v2/pkg/operators/extractors/extractor_types.go b/v2/pkg/operators/extractors/extractor_types.go index 6e6bd3774..f2ee89d78 100644 --- a/v2/pkg/operators/extractors/extractor_types.go +++ b/v2/pkg/operators/extractors/extractor_types.go @@ -99,6 +99,20 @@ func (holder *ExtractorTypeHolder) UnmarshalYAML(unmarshal func(interface{}) err return nil } +func (holder *ExtractorTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toExtractorTypes(s) + if err != nil { + return err + } + + holder.ExtractorType = computedType + return nil +} + func (holder *ExtractorTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.ExtractorType.String()) } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 0e81d90ab..7071a0bda 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -14,10 +14,10 @@ type Extractor struct { // spaces or underscores (_). // examples: // - value: "\"cookie-extractor\"" - Name string `yaml:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"` // description: | // Type is the type of the extractor. - Type ExtractorTypeHolder `json:"name,omitempty" yaml:"type"` + Type ExtractorTypeHolder `json:"type" yaml:"type"` // extractorType is the internal type of the extractor extractorType ExtractorType @@ -33,13 +33,13 @@ type Extractor struct { // - name: Wordpress Author Extraction regex // value: > // []string{"Author:(?:[A-Za-z0-9 -\\_=\"]+)?([A-Za-z0-9]+)<\\/span>"} - Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to extract from part,description=Regex to extract from part"` + Regex []string `yaml:"regex,omitempty" json:"regex,omitempty" jsonschema:"title=regex to extract from part,description=Regex to extract from part"` // description: | // Group specifies a numbered group to extract from the regex. // examples: // - name: Example Regex Group // value: "1" - RegexGroup int `yaml:"group,omitempty" jsonschema:"title=group to extract from regex,description=Group to extract from regex"` + RegexGroup int `yaml:"group,omitempty" json:"group,omitempty" jsonschema:"title=group to extract from regex,description=Group to extract from regex"` // regexCompiled is the compiled variant regexCompiled []*regexp.Regexp @@ -60,7 +60,7 @@ type Extractor struct { // - name: Extracting value of Content-Type Cookie // value: > // []string{"content_type"} - KVal []string `yaml:"kval,omitempty" jsonschema:"title=kval pairs to extract from response,description=Kval pairs to extract from response"` + KVal []string `yaml:"kval,omitempty" json:"kval,omitempty" jsonschema:"title=kval pairs to extract from response,description=Kval pairs to extract from response"` // description: | // JSON allows using jq-style syntax to extract items from json response @@ -70,27 +70,27 @@ type Extractor struct { // []string{".[] | .id"} // - value: > // []string{".batters | .batter | .[] | .id"} - JSON []string `yaml:"json,omitempty" jsonschema:"title=json jq expressions to extract data,description=JSON JQ expressions to evaluate from response part"` + JSON []string `yaml:"json,omitempty" json:"json,omitempty" jsonschema:"title=json jq expressions to extract data,description=JSON JQ expressions to evaluate from response part"` // description: | // XPath allows using xpath expressions to extract items from html response // // examples: // - value: > // []string{"/html/body/div/p[2]/a"} - XPath []string `yaml:"xpath,omitempty" jsonschema:"title=html xpath expressions to extract data,description=XPath allows using xpath expressions to extract items from html response"` + XPath []string `yaml:"xpath,omitempty" json:"xpath,omitempty" jsonschema:"title=html xpath expressions to extract data,description=XPath allows using xpath expressions to extract items from html response"` // description: | // Attribute is an optional attribute to extract from response XPath. // // examples: // - value: "\"href\"" - Attribute string `yaml:"attribute,omitempty" jsonschema:"title=optional attribute to extract from xpath,description=Optional attribute to extract from response XPath"` + Attribute string `yaml:"attribute,omitempty" json:"attribute,omitempty" jsonschema:"title=optional attribute to extract from xpath,description=Optional attribute to extract from response XPath"` // jsonCompiled is the compiled variant jsonCompiled []*gojq.Code // description: | // Extracts using DSL expressions. - DSL []string `yaml:"dsl,omitempty" jsonschema:"title=dsl expressions to extract,description=Optional attribute to extract from response dsl"` + DSL []string `yaml:"dsl,omitempty" json:"dsl,omitempty" jsonschema:"title=dsl expressions to extract,description=Optional attribute to extract from response dsl"` dslCompiled []*govaluate.EvaluableExpression // description: | @@ -101,16 +101,16 @@ type Extractor struct { // examples: // - value: "\"body\"" // - value: "\"raw\"" - Part string `yaml:"part,omitempty" jsonschema:"title=part of response to extract data from,description=Part of the request response to extract data from"` + Part string `yaml:"part,omitempty" json:"part,omitempty" jsonschema:"title=part of response to extract data from,description=Part of the request response to extract data from"` // description: | // Internal, when set to true will allow using the value extracted // in the next request for some protocols (like HTTP). - Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"` + Internal bool `yaml:"internal,omitempty" json:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"` // description: | // CaseInsensitive enables case-insensitive extractions. Default is false. // values: // - false // - true - CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` + CaseInsensitive bool `yaml:"case-insensitive,omitempty" json:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index f5aa306ec..e3f60481f 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -10,14 +10,14 @@ import ( type Matcher struct { // description: | // Type is the type of the matcher. - Type MatcherTypeHolder `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` + Type MatcherTypeHolder `yaml:"type" json:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` // description: | // Condition is the optional condition between two matcher variables. By default, // the condition is assumed to be OR. // values: // - "and" // - "or" - Condition string `yaml:"condition,omitempty" jsonschema:"title=condition between matcher variables,description=Condition between the matcher variables,enum=and,enum=or"` + Condition string `yaml:"condition,omitempty" json:"condition,omitempty" jsonschema:"title=condition between matcher variables,description=Condition between the matcher variables,enum=and,enum=or"` // description: | // Part is the part of the request response to match data from. @@ -27,31 +27,31 @@ type Matcher struct { // examples: // - value: "\"body\"" // - value: "\"raw\"" - Part string `yaml:"part,omitempty" jsonschema:"title=part of response to match,description=Part of response to match data from"` + Part string `yaml:"part,omitempty" json:"part,omitempty" jsonschema:"title=part of response to match,description=Part of response to match data from"` // description: | // Negative specifies if the match should be reversed // It will only match if the condition is not true. - Negative bool `yaml:"negative,omitempty" jsonschema:"title=negative specifies if match reversed,description=Negative specifies if the match should be reversed. It will only match if the condition is not true"` + Negative bool `yaml:"negative,omitempty" json:"negative,omitempty" jsonschema:"title=negative specifies if match reversed,description=Negative specifies if the match should be reversed. It will only match if the condition is not true"` // description: | // Name of the matcher. Name should be lowercase and must not contain // spaces or underscores (_). // examples: // - value: "\"cookie-matcher\"" - Name string `yaml:"name,omitempty" jsonschema:"title=name of the matcher,description=Name of the matcher"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=name of the matcher,description=Name of the matcher"` // description: | // Status are the acceptable status codes for the response. // examples: // - value: > // []int{200, 302} - Status []int `yaml:"status,omitempty" jsonschema:"title=status to match,description=Status to match for the response"` + Status []int `yaml:"status,omitempty" json:"status,omitempty" jsonschema:"title=status to match,description=Status to match for the response"` // description: | // Size is the acceptable size for the response // examples: // - value: > // []int{3029, 2042} - Size []int `yaml:"size,omitempty" jsonschema:"title=acceptable size for response,description=Size is the acceptable size for the response"` + Size []int `yaml:"size,omitempty" json:"size,omitempty" jsonschema:"title=acceptable size for response,description=Size is the acceptable size for the response"` // description: | // Words contains word patterns required to be present in the response part. // examples: @@ -61,7 +61,7 @@ type Matcher struct { // - name: Match for application/json in response headers // value: > // []string{"application/json"} - Words []string `yaml:"words,omitempty" jsonschema:"title=words to match in response,description= Words contains word patterns required to be present in the response part"` + Words []string `yaml:"words,omitempty" json:"words,omitempty" jsonschema:"title=words to match in response,description= Words contains word patterns required to be present in the response part"` // description: | // Regex contains Regular Expression patterns required to be present in the response part. // examples: @@ -71,7 +71,7 @@ type Matcher struct { // - name: Match for Open Redirect via Location header // value: > // []string{`(?m)^(?:Location\\s*?:\\s*?)(?:https?://|//)?(?:[a-zA-Z0-9\\-_\\.@]*)example\\.com.*$`} - Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to match in response,description=Regex contains regex patterns required to be present in the response part"` + Regex []string `yaml:"regex,omitempty" json:"regex,omitempty" jsonschema:"title=regex to match in response,description=Regex contains regex patterns required to be present in the response part"` // description: | // Binary are the binary patterns required to be present in the response part. // examples: @@ -81,7 +81,7 @@ type Matcher struct { // - name: Match for 7zip files // value: > // []string{"377ABCAF271C"} - Binary []string `yaml:"binary,omitempty" jsonschema:"title=binary patterns to match in response,description=Binary are the binary patterns required to be present in the response part"` + Binary []string `yaml:"binary,omitempty" json:"binary,omitempty" jsonschema:"title=binary patterns to match in response,description=Binary are the binary patterns required to be present in the response part"` // description: | // DSL are the dsl expressions that will be evaluated as part of nuclei matching rules. // A list of these helper functions are available [here](https://nuclei.projectdiscovery.io/templating-guide/helper-functions/). @@ -92,24 +92,24 @@ type Matcher struct { // - name: DSL Matcher for missing strict transport security header // value: > // []string{"!contains(tolower(all_headers), ''strict-transport-security'')"} - DSL []string `yaml:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"` + DSL []string `yaml:"dsl,omitempty" json:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"` // description: | // Encoding specifies the encoding for the words field if any. // values: // - "hex" - Encoding string `yaml:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"` + Encoding string `yaml:"encoding,omitempty" json:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"` // description: | // CaseInsensitive enables case-insensitive matches. Default is false. // values: // - false // - true - CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive match,description=use case insensitive match"` + CaseInsensitive bool `yaml:"case-insensitive,omitempty" json:"case-insensitive,omitempty" jsonschema:"title=use case insensitive match,description=use case insensitive match"` // description: | // MatchAll enables matching for all matcher values. Default is false. // values: // - false // - true - MatchAll bool `yaml:"match-all,omitempty" jsonschema:"title=match all values,description=match all matcher values ignoring condition"` + MatchAll bool `yaml:"match-all,omitempty" json:"match-all,omitempty" jsonschema:"title=match all values,description=match all matcher values ignoring condition"` // cached data for the compiled matcher condition ConditionType diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 47b70710c..0ce6c7e69 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -106,6 +106,20 @@ func (holder *MatcherTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error return nil } +func (holder *MatcherTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toMatcherTypes(s) + if err != nil { + return err + } + + holder.MatcherType = computedType + return nil +} + func (holder MatcherTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.MatcherType.String()) } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index aeeab8959..02593554a 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -23,17 +23,17 @@ type Operators struct { // // Multiple matchers can be combined with `matcher-condition` flag // which accepts either `and` or `or` as argument. - Matchers []*matchers.Matcher `yaml:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"` + Matchers []*matchers.Matcher `yaml:"matchers,omitempty" json:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"` // description: | // Extractors contains the extraction mechanism for the request to identify // and extract parts of the response. - Extractors []*extractors.Extractor `yaml:"extractors,omitempty" jsonschema:"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response"` + Extractors []*extractors.Extractor `yaml:"extractors,omitempty" json:"extractors,omitempty" jsonschema:"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response"` // description: | // MatchersCondition is the condition between the matchers. Default is OR. // values: // - "and" // - "or" - MatchersCondition string `yaml:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"` + MatchersCondition string `yaml:"matchers-condition,omitempty" json:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"` // cached variables that may be used along with request. matchersCondition matchers.ConditionType diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 745639105..6c8c6cc17 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -9,14 +9,15 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" "gopkg.in/yaml.v2" ) const ( - mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" - invalidFieldFormatTemplate = "invalid field format for '%s' (allowed format is %s)" + errMandatoryFieldMissingFmt = "mandatory '%s' field is missing" + errInvalidFieldFmt = "invalid field format for '%s' (allowed format is %s)" ) // LoadTemplate returns true if the template is valid and matches the filtering criteria. @@ -71,17 +72,17 @@ func validateTemplateFields(template *templates.Template) error { var errors []string if utils.IsBlank(info.Name) { - errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name")) + errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "name")) } if info.Authors.IsEmpty() { - errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author")) + errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "author")) } if template.ID == "" { - errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id")) + errors = append(errors, fmt.Sprintf(errMandatoryFieldMissingFmt, "id")) } else if !templateIDRegexp.MatchString(template.ID) { - errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())) + errors = append(errors, fmt.Sprintf(errInvalidFieldFmt, "id", templateIDRegexp.String())) } if len(errors) > 0 { @@ -105,7 +106,6 @@ const ( ) func init() { - parsedTemplatesCache = cache.New() stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") @@ -124,6 +124,12 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem } template := &templates.Template{} + + // check if the template is verified + if signer.DefaultVerifier != nil { + template.Verified, _ = signer.Verify(signer.DefaultVerifier, data) + } + if NoStrictSyntax { err = yaml.Unmarshal(data, template) } else { @@ -133,6 +139,7 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem stats.Increment(SyntaxErrorStats) return nil, err } + parsedTemplatesCache.Store(templatePath, template, nil) return template, nil } diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index a16e00ddf..01460638d 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -5,13 +5,12 @@ import ( "fmt" "testing" - "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/stretchr/testify/require" ) func TestLoadTemplate(t *testing.T) { diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index a0a9eb78d..662a72399 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -88,6 +88,20 @@ func (holder *AttackTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) return nil } +func (holder *AttackTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toAttackType(s) + if err != nil { + return err + } + + holder.Value = computedType + return nil +} + func (holder *AttackTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.Value.String()) } diff --git a/v2/pkg/protocols/common/helpers/writer/writer.go b/v2/pkg/protocols/common/helpers/writer/writer.go index 91b98f33b..c81b7d3d2 100644 --- a/v2/pkg/protocols/common/helpers/writer/writer.go +++ b/v2/pkg/protocols/common/helpers/writer/writer.go @@ -8,7 +8,7 @@ import ( ) // WriteResult is a helper for writing results to the output -func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient *reporting.Client) bool { +func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool { // Handle the case where no result found for the template. // In this case, we just show misc information about the failed // match for the template. diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 81ee94993..d4923276c 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -85,7 +85,7 @@ type Options struct { // Output is the output writer for nuclei Output output.Writer // IssuesClient is a client for issue exporting - IssuesClient *reporting.Client + IssuesClient reporting.Client // Progress is the nuclei progress bar implementation. Progress progress.Progress // Debug specifies whether debugging output should be shown for interactsh-client @@ -133,7 +133,7 @@ func New(options *Options) (*Client, error) { } // NewDefaultOptions returns the default options for interactsh client -func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options { +func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options { return &Options{ ServerURL: client.DefaultOptions.ServerURL, CacheSize: 5000, diff --git a/v2/pkg/protocols/common/variables/variables.go b/v2/pkg/protocols/common/variables/variables.go index 5acfaed98..2ab94569f 100644 --- a/v2/pkg/protocols/common/variables/variables.go +++ b/v2/pkg/protocols/common/variables/variables.go @@ -1,6 +1,7 @@ package variables import ( + "encoding/json" "strings" "github.com/alecthomas/jsonschema" @@ -39,6 +40,19 @@ func (variables *Variable) UnmarshalYAML(unmarshal func(interface{}) error) erro return nil } +func (variables *Variable) UnmarshalJSON(data []byte) error { + variables.InsertionOrderedStringMap = utils.InsertionOrderedStringMap{} + if err := json.Unmarshal(data, &variables.InsertionOrderedStringMap); err != nil { + return err + } + evaluated := variables.Evaluate(map[string]interface{}{}) + + for k, v := range evaluated { + variables.Set(k, v) + } + return nil +} + // Evaluate returns a finished map of variables based on set values func (variables *Variable) Evaluate(values map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}, variables.Len()) diff --git a/v2/pkg/protocols/common/variables/variables_test.go b/v2/pkg/protocols/common/variables/variables_test.go index e2ddb676a..7501b8ddd 100644 --- a/v2/pkg/protocols/common/variables/variables_test.go +++ b/v2/pkg/protocols/common/variables/variables_test.go @@ -1,6 +1,7 @@ package variables import ( + "encoding/json" "testing" "time" @@ -22,4 +23,21 @@ a6: "123456"` result := variables.Evaluate(map[string]interface{}{"hostname": "google.com"}) a4 := time.Now().Format("2006-01-02") require.Equal(t, map[string]interface{}{"a2": "098f6bcd4621d373cade4e832627b4f6", "a3": "this_is_random_text", "a4": a4, "a5": "moc.elgoog", "a6": "123456"}, result, "could not get correct elements") + + // json + data = `{ + "a2": "{{md5('test')}}", + "a3": "this_is_random_text", + "a4": "{{date_time('%Y-%M-%D')}}", + "a5": "{{reverse(hostname)}}", + "a6": "123456" +}` + variables = Variable{} + err = json.Unmarshal([]byte(data), &variables) + require.NoError(t, err, "could not unmarshal json variables") + + result = variables.Evaluate(map[string]interface{}{"hostname": "google.com"}) + a4 = time.Now().Format("2006-01-02") + require.Equal(t, map[string]interface{}{"a2": "098f6bcd4621d373cade4e832627b4f6", "a3": "this_is_random_text", "a4": a4, "a5": "moc.elgoog", "a6": "123456"}, result, "could not get correct elements") + } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 37429e6ff..242eaa04b 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -22,7 +22,7 @@ type Request struct { operators.Operators `yaml:",inline"` // ID is the optional id of the request - ID string `yaml:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` // description: | // Name is the Hostname to make DNS request for. @@ -30,10 +30,10 @@ type Request struct { // Generally, it is set to {{FQDN}} which is the domain we get from input. // examples: // - value: "\"{{FQDN}}\"" - Name string `yaml:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"` // description: | // RequestType is the type of DNS request to make. - RequestType DNSRequestTypeHolder `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` + RequestType DNSRequestTypeHolder `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` // description: | // Class is the class of the DNS request. // @@ -45,16 +45,16 @@ type Request struct { // - "hesiod" // - "none" // - "any" - Class string `yaml:"class,omitempty" jsonschema:"title=class of DNS request,description=Class is the class of the DNS request,enum=inet,enum=csnet,enum=chaos,enum=hesiod,enum=none,enum=any"` + Class string `yaml:"class,omitempty" json:"class,omitempty" jsonschema:"title=class of DNS request,description=Class is the class of the DNS request,enum=inet,enum=csnet,enum=chaos,enum=hesiod,enum=none,enum=any"` // description: | // Retries is the number of retries for the DNS request // examples: // - name: Use a retry of 3 to 5 generally // value: 5 - Retries int `yaml:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"` + Retries int `yaml:"retries,omitempty" json:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"` // description: | // Trace performs a trace operation for the target. - Trace bool `yaml:"trace,omitempty" jsonschema:"title=trace operation,description=Trace performs a trace operation for the target."` + Trace bool `yaml:"trace,omitempty" json:"trace,omitempty" jsonschema:"title=trace operation,description=Trace performs a trace operation for the target."` // description: | // TraceMaxRecursion is the number of max recursion allowed for trace operations // examples: @@ -72,9 +72,9 @@ type Request struct { // description: | // Recursion determines if resolver should recurse all records to get fresh results. - Recursion *bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` + Recursion *bool `yaml:"recursion,omitempty" json:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` // Resolvers to use for the dns requests - Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` + Resolvers []string `yaml:"resolvers,omitempty" json:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` } // RequestPartDefinitions contains a mapping of request part definitions and their diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go index c3ef1189c..2341e7591 100644 --- a/v2/pkg/protocols/dns/dns_types.go +++ b/v2/pkg/protocols/dns/dns_types.go @@ -116,6 +116,20 @@ func (holder *DNSRequestTypeHolder) UnmarshalYAML(unmarshal func(interface{}) er return nil } +func (holder *DNSRequestTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toDNSRequestTypes(s) + if err != nil { + return err + } + + holder.DNSRequestType = computedType + return nil +} + func (holder *DNSRequestTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.DNSRequestType.String()) } diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 8bbbf1ea7..26b4910ad 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -25,7 +25,7 @@ type Request struct { // Extensions is the list of extensions or mime types to perform matching on. // examples: // - value: '[]string{".txt", ".go", ".json"}' - Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"` + Extensions []string `yaml:"extensions,omitempty" json:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"` // description: | // DenyList is the list of file, directories, mime types or extensions to deny during matching. // @@ -33,10 +33,10 @@ type Request struct { // in nuclei. // examples: // - value: '[]string{".avi", ".mov", ".mp3"}' - DenyList []string `yaml:"denylist,omitempty" jsonschema:"title=denylist, directories and extensions to deny match,description=List of files, directories and extensions to deny during matching"` + DenyList []string `yaml:"denylist,omitempty" json:"denylist,omitempty" jsonschema:"title=denylist, directories and extensions to deny match,description=List of files, directories and extensions to deny during matching"` // ID is the optional id of the request - ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"` + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"` // description: | // MaxSize is the maximum size of the file to run request on. @@ -46,7 +46,7 @@ type Request struct { // If set to "no" then all content will be processed // examples: // - value: "\"5Mb\"" - MaxSize string `yaml:"max-size,omitempty" jsonschema:"title=max size data to run request on,description=Maximum size of the file to run request on"` + MaxSize string `yaml:"max-size,omitempty" json:"max-size,omitempty" jsonschema:"title=max size data to run request on,description=Maximum size of the file to run request on"` maxSize int64 // description: | @@ -57,7 +57,7 @@ type Request struct { // enables mime types check MimeType bool - CompiledOperators *operators.Operators `yaml:"-"` + CompiledOperators *operators.Operators `yaml:"-" json:"-"` // cache any variables that may be needed for operation. options *protocols.ExecuterOptions @@ -68,7 +68,7 @@ type Request struct { // description: | // NoRecursive specifies whether to not do recursive checks if folders are provided. - NoRecursive bool `yaml:"no-recursive,omitempty" jsonschema:"title=do not perform recursion,description=Specifies whether to not do recursive checks if folders are provided"` + NoRecursive bool `yaml:"no-recursive,omitempty" json:"no-recursive,omitempty" jsonschema:"title=do not perform recursion,description=Specifies whether to not do recursive checks if folders are provided"` allExtensions bool } diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index e65eed8be..8e3f7b327 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -13,20 +13,20 @@ type Action struct { // Args contain arguments for the headless action. // // Per action arguments are described in detail [here](https://nuclei.projectdiscovery.io/templating-guide/protocols/headless/). - Data map[string]string `yaml:"args,omitempty" jsonschema:"title=arguments for headless action,description=Args contain arguments for the headless action"` + Data map[string]string `yaml:"args,omitempty" json:"args,omitempty" jsonschema:"title=arguments for headless action,description=Args contain arguments for the headless action"` // description: | // Name is the name assigned to the headless action. // // This can be used to execute code, for instance in browser // DOM using script action, and get the result in a variable // which can be matched upon by nuclei. An Example template [here](https://github.com/projectdiscovery/nuclei-templates/blob/master/headless/prototype-pollution-check.yaml). - Name string `yaml:"name,omitempty" jsonschema:"title=name for headless action,description=Name is the name assigned to the headless action"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=name for headless action,description=Name is the name assigned to the headless action"` // description: | // Description is the optional description of the headless action - Description string `yaml:"description,omitempty" jsonschema:"title=description for headless action,description=Description of the headless action"` + Description string `yaml:"description,omitempty" json:"description,omitempty" jsonschema:"title=description for headless action,description=Description of the headless action"` // description: | // Action is the type of the action to perform. - ActionType ActionTypeHolder `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` + ActionType ActionTypeHolder `yaml:"action" json:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` } // String returns the string representation of an action diff --git a/v2/pkg/protocols/headless/engine/action_types.go b/v2/pkg/protocols/headless/engine/action_types.go index 4cf861184..d39cc0010 100644 --- a/v2/pkg/protocols/headless/engine/action_types.go +++ b/v2/pkg/protocols/headless/engine/action_types.go @@ -198,6 +198,20 @@ func (holder *ActionTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) return nil } +func (holder *ActionTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toActionTypes(s) + if err != nil { + return err + } + + holder.ActionType = computedType + return nil +} + func (holder *ActionTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.ActionType.String()) } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 9631c4a19..49ac08fcb 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -15,41 +15,41 @@ import ( // Request contains a Headless protocol request to be made from a template type Request struct { // ID is the optional id of the request - ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=Optional ID of the headless request"` + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=Optional ID of the headless request"` // description: | // Attack is the type of payload combinations to perform. // // Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. - AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" json:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // // Payloads support both key-values combinations where a list // of payloads is provided, or optionally a single file can also // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the headless request,description=Payloads contains any payloads for the current request"` + Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the headless request,description=Payloads contains any payloads for the current request"` // description: | // Steps is the list of actions to run for headless request - Steps []*engine.Action `yaml:"steps,omitempty" jsonschema:"title=list of actions for headless request,description=List of actions to run for headless request"` + Steps []*engine.Action `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"title=list of actions for headless request,description=List of actions to run for headless request"` // descriptions: | // User-Agent is the type of user-agent to use for the request. - UserAgent useragent.UserAgentHolder `yaml:"user_agent,omitempty" jsonschema:"title=user agent for the headless request,description=User agent for the headless request"` + UserAgent useragent.UserAgentHolder `yaml:"user_agent,omitempty" json:"user_agent,omitempty" jsonschema:"title=user agent for the headless request,description=User agent for the headless request"` // description: | // If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request. - CustomUserAgent string `yaml:"custom_user_agent,omitempty" jsonschema:"title=custom user agent for the headless request,description=Custom user agent for the headless request"` + CustomUserAgent string `yaml:"custom_user_agent,omitempty" json:"custom_user_agent,omitempty" jsonschema:"title=custom user agent for the headless request,description=Custom user agent for the headless request"` compiledUserAgent string // description: | // StopAtFirstMatch stops the execution of the requests and template as soon as a match is found. - StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` + StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` // Operators for the current request go here. - operators.Operators `yaml:",inline,omitempty"` - CompiledOperators *operators.Operators `yaml:"-"` + operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-" json:"-"` // cache any variables that may be needed for operation. options *protocols.ExecuterOptions diff --git a/v2/pkg/protocols/http/fuzz/fuzz.go b/v2/pkg/protocols/http/fuzz/fuzz.go index ad7fb6d14..63cecaec2 100644 --- a/v2/pkg/protocols/http/fuzz/fuzz.go +++ b/v2/pkg/protocols/http/fuzz/fuzz.go @@ -20,7 +20,7 @@ type Rule struct { // - "prefix" // - "postfix" // - "infix" - Type string `yaml:"type,omitempty" jsonschema:"title=type of rule,description=Type of fuzzing rule to perform,enum=replace,enum=prefix,enum=postfix,enum=infix"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"title=type of rule,description=Type of fuzzing rule to perform,enum=replace,enum=prefix,enum=postfix,enum=infix"` ruleType ruleType // description: | // Part is the part of request to fuzz. @@ -28,7 +28,7 @@ type Rule struct { // query fuzzes the query part of url. More parts will be added later. // values: // - "query" - Part string `yaml:"part,omitempty" jsonschema:"title=part of rule,description=Part of request rule to fuzz,enum=query"` + Part string `yaml:"part,omitempty" json:"part,omitempty" jsonschema:"title=part of rule,description=Part of request rule to fuzz,enum=query"` partType partType // description: | // Mode is the mode of fuzzing to perform. @@ -37,7 +37,7 @@ type Rule struct { // values: // - "single" // - "multiple" - Mode string `yaml:"mode,omitempty" jsonschema:"title=mode of rule,description=Mode of request rule to fuzz,enum=single,enum=multiple"` + Mode string `yaml:"mode,omitempty" json:"mode,omitempty" jsonschema:"title=mode of rule,description=Mode of request rule to fuzz,enum=single,enum=multiple"` modeType modeType // description: | @@ -46,7 +46,7 @@ type Rule struct { // - name: Examples of keys // value: > // []string{"url", "file", "host"} - Keys []string `yaml:"keys,omitempty" jsonschema:"title=keys of parameters to fuzz,description=Keys of parameters to fuzz"` + Keys []string `yaml:"keys,omitempty" json:"keys,omitempty" jsonschema:"title=keys of parameters to fuzz,description=Keys of parameters to fuzz"` keysMap map[string]struct{} // description: | // KeysRegex is the optional list of regex key parameters to fuzz. @@ -54,7 +54,7 @@ type Rule struct { // - name: Examples of key regex // value: > // []string{"url.*"} - KeysRegex []string `yaml:"keys-regex,omitempty" jsonschema:"title=keys regex to fuzz,description=Regex of parameter keys to fuzz"` + KeysRegex []string `yaml:"keys-regex,omitempty" json:"keys-regex,omitempty" jsonschema:"title=keys regex to fuzz,description=Regex of parameter keys to fuzz"` keysRegex []*regexp.Regexp // description: | // Values is the optional list of regex value parameters to fuzz. @@ -62,7 +62,7 @@ type Rule struct { // - name: Examples of value regex // value: > // []string{"https?://.*"} - ValuesRegex []string `yaml:"values,omitempty" jsonschema:"title=values regex to fuzz,description=Regex of parameter values to fuzz"` + ValuesRegex []string `yaml:"values,omitempty" json:"values,omitempty" jsonschema:"title=values regex to fuzz,description=Regex of parameter values to fuzz"` valuesRegex []*regexp.Regexp // description: | @@ -71,7 +71,7 @@ type Rule struct { // - name: Examples of fuzz // value: > // []string{"{{ssrf}}", "{{interactsh-url}}", "example-value"} - Fuzz []string `yaml:"fuzz,omitempty" jsonschema:"title=payloads of fuzz rule,description=Payloads to perform fuzzing substitutions with"` + Fuzz []string `yaml:"fuzz,omitempty" json:"fuzz,omitempty" jsonschema:"title=payloads of fuzz rule,description=Payloads to perform fuzzing substitutions with"` options *protocols.ExecuterOptions generator *generators.PayloadGenerator diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index c828c5264..19b49828d 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -22,7 +22,7 @@ import ( // Request contains a http request to be made from a template type Request struct { // Operators for the current request go here. - operators.Operators `yaml:",inline"` + operators.Operators `yaml:",inline" json:",inline"` // description: | // Path contains the path/s for the HTTP requests. It supports variables // as placeholders. @@ -30,22 +30,22 @@ type Request struct { // - name: Some example path values // value: > // []string{"{{BaseURL}}", "{{BaseURL}}/+CSCOU+/../+CSCOE+/files/file_list.json?path=/sessions"} - Path []string `yaml:"path,omitempty" jsonschema:"title=path(s) for the http request,description=Path(s) to send http requests to"` + Path []string `yaml:"path,omitempty" json:"path,omitempty" jsonschema:"title=path(s) for the http request,description=Path(s) to send http requests to"` // description: | // Raw contains HTTP Requests in Raw format. // examples: // - name: Some example raw requests // value: | // []string{"GET /etc/passwd HTTP/1.1\nHost:\nContent-Length: 4", "POST /.%0d./.%0d./.%0d./.%0d./bin/sh HTTP/1.1\nHost: {{Hostname}}\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0\nContent-Length: 1\nConnection: close\n\necho\necho\ncat /etc/passwd 2>&1"} - Raw []string `yaml:"raw,omitempty" jsonschema:"http requests in raw format,description=HTTP Requests in Raw Format"` + Raw []string `yaml:"raw,omitempty" json:"raw,omitempty" jsonschema:"http requests in raw format,description=HTTP Requests in Raw Format"` // ID is the optional id of the request - ID string `yaml:"id,omitempty" jsonschema:"title=id for the http request,description=ID for the HTTP Request"` + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id for the http request,description=ID for the HTTP Request"` // description: | // Name is the optional name of the request. // // If a name is specified, all the named request in a template can be matched upon // in a combined manner allowing multi-request based matchers. - Name string `yaml:"name,omitempty" jsonschema:"title=name for the http request,description=Optional name for the HTTP Request"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=name for the http request,description=Optional name for the HTTP Request"` // description: | // Attack is the type of payload combinations to perform. // @@ -55,54 +55,54 @@ type Request struct { // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" json:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. - Method HTTPMethodTypeHolder `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` + Method HTTPMethodTypeHolder `yaml:"method,omitempty" json:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` // description: | // Body is an optional parameter which contains HTTP Request body. // examples: // - name: Same Body for a Login POST request // value: "\"username=test&password=test\"" - Body string `yaml:"body,omitempty" jsonschema:"title=body is the http request body,description=Body is an optional parameter which contains HTTP Request body"` + Body string `yaml:"body,omitempty" json:"body,omitempty" jsonschema:"title=body is the http request body,description=Body is an optional parameter which contains HTTP Request body"` // description: | // Payloads contains any payloads for the current request. // // Payloads support both key-values combinations where a list // of payloads is provided, or optionally a single file can also // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the http request,description=Payloads contains any payloads for the current request"` + Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the http request,description=Payloads contains any payloads for the current request"` // description: | // Headers contains HTTP Headers to send with the request. // examples: // - value: | // map[string]string{"Content-Type": "application/x-www-form-urlencoded", "Content-Length": "1", "Any-Header": "Any-Value"} - Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers to send with the http request,description=Headers contains HTTP Headers to send with the request"` + Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty" jsonschema:"title=headers to send with the http request,description=Headers contains HTTP Headers to send with the request"` // description: | // RaceCount is the number of times to send a request in Race Condition Attack. // examples: // - name: Send a request 5 times // value: "5" - RaceNumberRequests int `yaml:"race_count,omitempty" jsonschema:"title=number of times to repeat request in race condition,description=Number of times to send a request in Race Condition Attack"` + RaceNumberRequests int `yaml:"race_count,omitempty" json:"race_count,omitempty" jsonschema:"title=number of times to repeat request in race condition,description=Number of times to send a request in Race Condition Attack"` // description: | // MaxRedirects is the maximum number of redirects that should be followed. // examples: // - name: Follow up to 5 redirects // value: "5" - MaxRedirects int `yaml:"max-redirects,omitempty" jsonschema:"title=maximum number of redirects to follow,description=Maximum number of redirects that should be followed"` + MaxRedirects int `yaml:"max-redirects,omitempty" json:"max-redirects,omitempty" jsonschema:"title=maximum number of redirects to follow,description=Maximum number of redirects that should be followed"` // description: | // PipelineConcurrentConnections is number of connections to create during pipelining. // examples: // - name: Create 40 concurrent connections // value: 40 - PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty" jsonschema:"title=number of pipelining connections,description=Number of connections to create during pipelining"` + PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty" json:"pipeline-concurrent-connections,omitempty" jsonschema:"title=number of pipelining connections,description=Number of connections to create during pipelining"` // description: | // PipelineRequestsPerConnection is number of requests to send per connection when pipelining. // examples: // - name: Send 100 requests per pipeline connection // value: 100 - PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty" jsonschema:"title=number of requests to send per pipelining connections,description=Number of requests to send per connection when pipelining"` + PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty" json:"pipeline-requests-per-connection,omitempty" jsonschema:"title=number of requests to send per pipelining connections,description=Number of requests to send per connection when pipelining"` // description: | // Threads specifies number of threads to use sending requests. This enables Connection Pooling. // @@ -111,18 +111,18 @@ type Request struct { // examples: // - name: Send requests using 10 concurrent threads // value: 10 - Threads int `yaml:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"` + Threads int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"` // description: | // MaxSize is the maximum size of http response body to read in bytes. // examples: // - name: Read max 2048 bytes of the response // value: 2048 - MaxSize int `yaml:"max-size,omitempty" jsonschema:"title=maximum http response body size,description=Maximum size of http response body to read in bytes"` + MaxSize int `yaml:"max-size,omitempty" json:"max-size,omitempty" jsonschema:"title=maximum http response body size,description=Maximum size of http response body to read in bytes"` // Fuzzing describes schema to fuzz http requests - Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz http requests"` + Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" json:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz http requests"` - CompiledOperators *operators.Operators `yaml:"-"` + CompiledOperators *operators.Operators `yaml:"-" json:"-"` options *protocols.ExecuterOptions connConfiguration *httpclientpool.Configuration @@ -140,63 +140,63 @@ type Request struct { // Signature is the request signature method // values: // - "AWS" - Signature SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` + Signature SignatureTypeHolder `yaml:"signature,omitempty" json:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` // description: | // CookieReuse is an optional setting that enables cookie reuse for // all requests defined in raw section. - CookieReuse bool `yaml:"cookie-reuse,omitempty" jsonschema:"title=optional cookie reuse enable,description=Optional setting that enables cookie reuse"` + CookieReuse bool `yaml:"cookie-reuse,omitempty" json:"cookie-reuse,omitempty" jsonschema:"title=optional cookie reuse enable,description=Optional setting that enables cookie reuse"` // description: | // Enables force reading of the entire raw unsafe request body ignoring // any specified content length headers. - ForceReadAllBody bool `yaml:"read-all,omitempty" jsonschema:"title=force read all body,description=Enables force reading of entire unsafe http request body"` + ForceReadAllBody bool `yaml:"read-all,omitempty" json:"read-all,omitempty" jsonschema:"title=force read all body,description=Enables force reading of entire unsafe http request body"` // description: | // Redirects specifies whether redirects should be followed by the HTTP Client. // // This can be used in conjunction with `max-redirects` to control the HTTP request redirects. - Redirects bool `yaml:"redirects,omitempty" jsonschema:"title=follow http redirects,description=Specifies whether redirects should be followed by the HTTP Client"` + Redirects bool `yaml:"redirects,omitempty" json:"redirects,omitempty" jsonschema:"title=follow http redirects,description=Specifies whether redirects should be followed by the HTTP Client"` // description: | // Redirects specifies whether only redirects to the same host should be followed by the HTTP Client. // // This can be used in conjunction with `max-redirects` to control the HTTP request redirects. - HostRedirects bool `yaml:"host-redirects,omitempty" jsonschema:"title=follow same host http redirects,description=Specifies whether redirects to the same host should be followed by the HTTP Client"` + HostRedirects bool `yaml:"host-redirects,omitempty" json:"host-redirects,omitempty" jsonschema:"title=follow same host http redirects,description=Specifies whether redirects to the same host should be followed by the HTTP Client"` // description: | // Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining // // All requests must be idempotent (GET/POST). This can be used for race conditions/billions requests. - Pipeline bool `yaml:"pipeline,omitempty" jsonschema:"title=perform HTTP 1.1 pipelining,description=Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"` + Pipeline bool `yaml:"pipeline,omitempty" json:"pipeline,omitempty" jsonschema:"title=perform HTTP 1.1 pipelining,description=Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"` // description: | // Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests. // // This uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete // control over the request, with no normalization performed by the client. - Unsafe bool `yaml:"unsafe,omitempty" jsonschema:"title=use rawhttp non-strict-rfc client,description=Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests"` + Unsafe bool `yaml:"unsafe,omitempty" json:"unsafe,omitempty" jsonschema:"title=use rawhttp non-strict-rfc client,description=Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests"` // description: | // Race determines if all the request have to be attempted at the same time (Race Condition) // // The actual number of requests that will be sent is determined by the `race_count` field. - Race bool `yaml:"race,omitempty" jsonschema:"title=perform race-http request coordination attack,description=Race determines if all the request have to be attempted at the same time (Race Condition)"` + Race bool `yaml:"race,omitempty" json:"race,omitempty" jsonschema:"title=perform race-http request coordination attack,description=Race determines if all the request have to be attempted at the same time (Race Condition)"` // description: | // ReqCondition automatically assigns numbers to requests and preserves their history. // // This allows matching on them later for multi-request conditions. // Deprecated: request condition will be detected automatically (https://github.com/projectdiscovery/nuclei/issues/2393) - ReqCondition bool `yaml:"req-condition,omitempty" jsonschema:"title=preserve request history,description=Automatically assigns numbers to requests and preserves their history"` + ReqCondition bool `yaml:"req-condition,omitempty" json:"req-condition,omitempty" jsonschema:"title=preserve request history,description=Automatically assigns numbers to requests and preserves their history"` // description: | // StopAtFirstMatch stops the execution of the requests and template as soon as a match is found. - StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` + StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` // description: | // SkipVariablesCheck skips the check for unresolved variables in request - SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` + SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" json:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` // description: | // IterateAll iterates all the values extracted from internal extractors - IterateAll bool `yaml:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"` + IterateAll bool `yaml:"iterate-all,omitempty" json:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"` // description: | // DigestAuthUsername specifies the username for digest authentication - DigestAuthUsername string `yaml:"digest-username,omitempty" jsonschema:"title=specifies the username for digest authentication,description=Optional parameter which specifies the username for digest auth"` + DigestAuthUsername string `yaml:"digest-username,omitempty" json:"digest-username,omitempty" jsonschema:"title=specifies the username for digest authentication,description=Optional parameter which specifies the username for digest auth"` // description: | // DigestAuthPassword specifies the password for digest authentication - DigestAuthPassword string `yaml:"digest-password,omitempty" jsonschema:"title=specifies the password for digest authentication,description=Optional parameter which specifies the password for digest auth"` + DigestAuthPassword string `yaml:"digest-password,omitempty" json:"digest-password,omitempty" jsonschema:"title=specifies the password for digest authentication,description=Optional parameter which specifies the password for digest auth"` } // Options returns executer options for http request diff --git a/v2/pkg/protocols/http/http_method_types.go b/v2/pkg/protocols/http/http_method_types.go index 1a50fc546..571446ab7 100644 --- a/v2/pkg/protocols/http/http_method_types.go +++ b/v2/pkg/protocols/http/http_method_types.go @@ -116,6 +116,20 @@ func (holder *HTTPMethodTypeHolder) UnmarshalYAML(unmarshal func(interface{}) er return nil } +func (holder *HTTPMethodTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toHTTPMethodTypes(s) + if err != nil { + return err + } + + holder.MethodType = computedType + return nil +} + func (holder *HTTPMethodTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.MethodType.String()) } diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go index a72a7ff61..7ffda1af4 100644 --- a/v2/pkg/protocols/http/signature.go +++ b/v2/pkg/protocols/http/signature.go @@ -2,6 +2,7 @@ package http import ( "encoding/json" + "strings" "github.com/alecthomas/jsonschema" "github.com/pkg/errors" @@ -77,7 +78,21 @@ func (holder *SignatureTypeHolder) UnmarshalYAML(unmarshal func(interface{}) err return nil } -func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) { +func (holder *SignatureTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toSignatureType(s) + if err != nil { + return err + } + + holder.Value = computedType + return nil +} + +func (holder SignatureTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.Value.String()) } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index b49a20cf3..f3189be5a 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -17,7 +17,7 @@ import ( // Request contains a Network protocol request to be made from a template type Request struct { // ID is the optional id of the request - ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` // description: | // Host to send network requests to. @@ -27,7 +27,7 @@ type Request struct { // examples: // - value: | // []string{"{{Hostname}}"} - Address []string `yaml:"host,omitempty" jsonschema:"title=host to send requests to,description=Host to send network requests to"` + Address []string `yaml:"host,omitempty" json:"host,omitempty" jsonschema:"title=host to send requests to,description=Host to send network requests to"` addresses []addressKV // description: | @@ -35,32 +35,32 @@ type Request struct { // // Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. - AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" json:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // // Payloads support both key-values combinations where a list // of payloads is provided, or optionally a single file can also // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the network request,description=Payloads contains any payloads for the current request"` + Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the network request,description=Payloads contains any payloads for the current request"` // description: | // Inputs contains inputs for the network socket - Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the network request,description=Inputs contains any input/output for the current request"` + Inputs []*Input `yaml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"title=inputs for the network request,description=Inputs contains any input/output for the current request"` // description: | // ReadSize is the size of response to read at the end // // Default value for read-size is 1024. // examples: // - value: "2048" - ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + ReadSize int `yaml:"read-size,omitempty" json:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` // description: | // ReadAll determines if the data stream should be read till the end regardless of the size // // Default value for read-all is false. // examples: // - value: false - ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` + ReadAll bool `yaml:"read-all,omitempty" json:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` // description: | // SelfContained specifies if the request is self-contained. @@ -105,7 +105,7 @@ type Input struct { // examples: // - value: "\"TEST\"" // - value: "\"hex_decode('50494e47')\"" - Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` + Data string `yaml:"data,omitempty" json:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` // description: | // Type is the type of input specified in `data` field. // @@ -113,7 +113,7 @@ type Input struct { // values: // - "hex" // - "text" - Type NetworkInputTypeHolder `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"` + Type NetworkInputTypeHolder `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"` // description: | // Read is the number of bytes to read from socket. // @@ -124,12 +124,12 @@ type Input struct { // The [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this. // examples: // - value: "1024" - Read int `yaml:"read,omitempty" jsonschema:"title=bytes to read from socket,description=Number of bytes to read from socket"` + Read int `yaml:"read,omitempty" json:"read,omitempty" jsonschema:"title=bytes to read from socket,description=Number of bytes to read from socket"` // description: | // Name is the optional name of the data read to provide matching on. // examples: // - value: "\"prefix\"" - Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/protocols/network/network_input_types.go b/v2/pkg/protocols/network/network_input_types.go index 694d58c6d..9180757c9 100644 --- a/v2/pkg/protocols/network/network_input_types.go +++ b/v2/pkg/protocols/network/network_input_types.go @@ -93,6 +93,20 @@ func (holder *NetworkInputTypeHolder) UnmarshalYAML(unmarshal func(interface{}) return nil } +func (holder *NetworkInputTypeHolder) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "" { + return nil + } + computedType, err := toNetworkInputTypes(s) + if err != nil { + return err + } + + holder.NetworkInputType = computedType + return nil +} + func (holder *NetworkInputTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.NetworkInputType.String()) } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 6d0f4c0b6..a11412b27 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -50,7 +50,7 @@ type ExecuterOptions struct { // Options contains configuration options for the executer. Options *types.Options // IssuesClient is a client for nuclei issue tracker reporting - IssuesClient *reporting.Client + IssuesClient reporting.Client // Progress is a progress client for scan reporting Progress progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 3910714f1..57a7dcc09 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -37,12 +37,12 @@ import ( // Request is a request for the SSL protocol type Request struct { // Operators for the current request go here. - operators.Operators `yaml:",inline,omitempty"` - CompiledOperators *operators.Operators `yaml:"-"` + operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-" json:"-"` // description: | // Address contains address for the request - Address string `yaml:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"` + Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"` // description: | // Minimum tls version - auto if not specified. // values: @@ -51,7 +51,7 @@ type Request struct { // - "tls11" // - "tls12" // - "tls13" - MinVersion string `yaml:"min_version,omitempty" jsonschema:"title=Min. TLS version,description=Minimum tls version - automatic if not specified.,enum=sslv3,enum=tls10,enum=tls11,enum=tls12,enum=tls13"` + MinVersion string `yaml:"min_version,omitempty" json:"min_version,omitempty" jsonschema:"title=Min. TLS version,description=Minimum tls version - automatic if not specified.,enum=sslv3,enum=tls10,enum=tls11,enum=tls12,enum=tls13"` // description: | // Max tls version - auto if not specified. // values: @@ -60,17 +60,17 @@ type Request struct { // - "tls11" // - "tls12" // - "tls13" - MaxVersion string `yaml:"max_version,omitempty" jsonschema:"title=Max. TLS version,description=Max tls version - automatic if not specified.,enum=sslv3,enum=tls10,enum=tls11,enum=tls12,enum=tls13"` + MaxVersion string `yaml:"max_version,omitempty" json:"max_version,omitempty" jsonschema:"title=Max. TLS version,description=Max tls version - automatic if not specified.,enum=sslv3,enum=tls10,enum=tls11,enum=tls12,enum=tls13"` // description: | // Client Cipher Suites - auto if not specified. - CiperSuites []string `yaml:"cipher_suites,omitempty"` + CiperSuites []string `yaml:"cipher_suites,omitempty" json:"cipher_suites,omitempty"` // description: | // Tls Scan Mode - auto if not specified // values: // - "ctls" // - "ztls" // - "auto" - ScanMode string `yaml:"scan_mode,omitempty" jsonschema:"title=Scan Mode,description=Scan Mode - auto if not specified.,enum=ctls,enum=ztls,enum=auto"` + ScanMode string `yaml:"scan_mode,omitempty" json:"scan_mode,omitempty" jsonschema:"title=Scan Mode,description=Scan Mode - auto if not specified.,enum=ctls,enum=ztls,enum=auto"` // cache any variables that may be needed for operation. dialer *fastdialer.Dialer diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index e2e461ddf..7f65341dd 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -38,32 +38,32 @@ import ( // Request is a request for the Websocket protocol type Request struct { // Operators for the current request go here. - operators.Operators `yaml:",inline,omitempty"` - CompiledOperators *operators.Operators `yaml:"-"` + operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-" json:"-"` // description: | // Address contains address for the request - Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` + Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` // description: | // Inputs contains inputs for the websocket protocol - Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + Inputs []*Input `yaml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` // description: | // Headers contains headers for the request. - Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` + Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` // description: | // Attack is the type of payload combinations to perform. // // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. - AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" json:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // // Payloads support both key-values combinations where a list // of payloads is provided, or optionally a single file can also // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + Payloads map[string]interface{} `yaml:"payloads,omitempty" json:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` generator *generators.PayloadGenerator @@ -81,12 +81,12 @@ type Input struct { // examples: // - value: "\"TEST\"" // - value: "\"hex_decode('50494e47')\"" - Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` + Data string `yaml:"data,omitempty" json:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` // description: | // Name is the optional name of the data read to provide matching on. // examples: // - value: "\"prefix\"" - Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` } const ( diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 79f94f353..ddabfc0d9 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -28,19 +28,19 @@ import ( // Request is a request for the WHOIS protocol type Request struct { // Operators for the current request go here. - operators.Operators `yaml:",inline,omitempty"` - CompiledOperators *operators.Operators `yaml:"-"` + operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-" json:"-"` // description: | // Query contains query for the request - Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"` + Query string `yaml:"query,omitempty" json:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"` // description: | // Optional WHOIS server URL. // // If present, specifies the WHOIS server to execute the Request on. // Otherwise, nil enables bootstrapping - Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"` + Server string `yaml:"server,omitempty" json:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"` // cache any variables that may be needed for operation. client *rdap.Client options *protocols.ExecuterOptions diff --git a/v2/pkg/reporting/client.go b/v2/pkg/reporting/client.go new file mode 100644 index 000000000..582e30657 --- /dev/null +++ b/v2/pkg/reporting/client.go @@ -0,0 +1,15 @@ +package reporting + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" +) + +// Client is a client for nuclei issue tracking module +type Client interface { + RegisterTracker(tracker Tracker) + RegisterExporter(exporter Exporter) + Close() + Clear() + CreateIssue(event *output.ResultEvent) error + GetReportingOptions() *Options +} diff --git a/v2/pkg/reporting/dedupe/dedupe.go b/v2/pkg/reporting/dedupe/dedupe.go index c56e2205b..a30de782a 100644 --- a/v2/pkg/reporting/dedupe/dedupe.go +++ b/v2/pkg/reporting/dedupe/dedupe.go @@ -51,6 +51,18 @@ func New(dbPath string) (*Storage, error) { return storage, nil } +func (s *Storage) Clear() { + var keys [][]byte + iter := s.storage.NewIterator(nil, nil) + for iter.Next() { + keys = append(keys, iter.Key()) + } + iter.Release() + for _, key := range keys { + _ = s.storage.Delete(key, nil) + } +} + // Close closes the storage for further operations func (s *Storage) Close() { s.storage.Close() diff --git a/v2/pkg/reporting/format/format_test.go b/v2/pkg/reporting/format/format_test.go index 101d1b8a2..6be42bb17 100644 --- a/v2/pkg/reporting/format/format_test.go +++ b/v2/pkg/reporting/format/format_test.go @@ -18,7 +18,7 @@ func TestToMarkdownTableString(t *testing.T) { Description: "Test description", SeverityHolder: severity.Holder{Severity: severity.High}, Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}}, - Reference: stringslice.StringSlice{Value: "reference1"}, + Reference: stringslice.NewRaw("reference1"), Metadata: map[string]interface{}{ "customDynamicKey1": "customDynamicValue1", "customDynamicKey2": "customDynamicValue2", diff --git a/v2/pkg/reporting/options.go b/v2/pkg/reporting/options.go new file mode 100644 index 000000000..1397f47b2 --- /dev/null +++ b/v2/pkg/reporting/options.go @@ -0,0 +1,36 @@ +package reporting + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/splunk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/retryablehttp-go" +) + +// Options is a configuration file for nuclei reporting module +type Options struct { + // AllowList contains a list of allowed events for reporting module + AllowList *Filter `yaml:"allow-list"` + // DenyList contains a list of denied events for reporting module + DenyList *Filter `yaml:"deny-list"` + // GitHub contains configuration options for GitHub Issue Tracker + GitHub *github.Options `yaml:"github"` + // GitLab contains configuration options for GitLab Issue Tracker + GitLab *gitlab.Options `yaml:"gitlab"` + // Jira contains configuration options for Jira Issue Tracker + Jira *jira.Options `yaml:"jira"` + // MarkdownExporter contains configuration options for Markdown Exporter Module + MarkdownExporter *markdown.Options `yaml:"markdown"` + // SarifExporter contains configuration options for Sarif Exporter Module + SarifExporter *sarif.Options `yaml:"sarif"` + // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module + ElasticsearchExporter *es.Options `yaml:"elasticsearch"` + // SplunkExporter contains configuration options for splunkhec Exporter Module + SplunkExporter *splunk.Options `yaml:"splunkhec"` + + HttpClient *retryablehttp.Client `yaml:"-"` +} diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 3610b7437..f98b84d35 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -3,12 +3,12 @@ package reporting import ( "os" "path/filepath" - "strings" - "github.com/pkg/errors" "go.uber.org/multierr" "gopkg.in/yaml.v2" + "errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" @@ -21,34 +21,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" - "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + sliceutil "github.com/projectdiscovery/utils/slice" ) -// Options is a configuration file for nuclei reporting module -type Options struct { - // AllowList contains a list of allowed events for reporting module - AllowList *Filter `yaml:"allow-list"` - // DenyList contains a list of denied events for reporting module - DenyList *Filter `yaml:"deny-list"` - // GitHub contains configuration options for GitHub Issue Tracker - GitHub *github.Options `yaml:"github"` - // GitLab contains configuration options for GitLab Issue Tracker - GitLab *gitlab.Options `yaml:"gitlab"` - // Jira contains configuration options for Jira Issue Tracker - Jira *jira.Options `yaml:"jira"` - // MarkdownExporter contains configuration options for Markdown Exporter Module - MarkdownExporter *markdown.Options `yaml:"markdown"` - // SarifExporter contains configuration options for Sarif Exporter Module - SarifExporter *sarif.Options `yaml:"sarif"` - // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module - ElasticsearchExporter *es.Options `yaml:"elasticsearch"` - // SplunkExporter contains configuration options for splunkhec Exporter Module - SplunkExporter *splunk.Options `yaml:"splunkhec"` - - HttpClient *retryablehttp.Client `yaml:"-"` -} - // Filter filters the received event and decides whether to perform // reporting for it or not. type Filter struct { @@ -56,9 +33,9 @@ type Filter struct { Tags stringslice.StringSlice `yaml:"tags"` } -const ( - reportingClientCreationErrorMessage = "could not create reporting client" - exportClientCreationErrorMessage = "could not create exporting client" +var ( + ErrReportingClientCreation = errors.New("could not create reporting client") + ErrExportClientCreation = errors.New("could not create exporting client") ) // GetMatch returns true if a filter matches result event @@ -73,8 +50,8 @@ func isTagMatch(event *output.ResultEvent, filter *Filter) bool { } tags := event.Info.Tags.ToSlice() - for _, tag := range filterTags.ToSlice() { - if stringSliceContains(tags, tag) { + for _, filterTag := range filterTags.ToSlice() { + if sliceutil.Contains(tags, filterTag) { return true } } @@ -89,13 +66,7 @@ func isSeverityMatch(event *output.ResultEvent, filter *Filter) bool { return true } - for _, current := range filter.Severities { - if current == resultEventSeverity { - return true - } - } - - return false + return sliceutil.Contains(filter.Severities, resultEventSeverity) } // Tracker is an interface implemented by an issue tracker @@ -112,8 +83,8 @@ type Exporter interface { Export(event *output.ResultEvent) error } -// Client is a client for nuclei issue tracking module -type Client struct { +// ReportingClient is a client for nuclei issue tracking module +type ReportingClient struct { trackers []Tracker exporters []Exporter options *Options @@ -121,14 +92,14 @@ type Client struct { } // New creates a new nuclei issue tracker reporting client -func New(options *Options, db string) (*Client, error) { - client := &Client{options: options} +func New(options *Options, db string) (Client, error) { + client := &ReportingClient{options: options} if options.GitHub != nil { options.GitHub.HttpClient = options.HttpClient tracker, err := github.New(options.GitHub) if err != nil { - return nil, errors.Wrap(err, reportingClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) } client.trackers = append(client.trackers, tracker) } @@ -136,7 +107,7 @@ func New(options *Options, db string) (*Client, error) { options.GitLab.HttpClient = options.HttpClient tracker, err := gitlab.New(options.GitLab) if err != nil { - return nil, errors.Wrap(err, reportingClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) } client.trackers = append(client.trackers, tracker) } @@ -144,21 +115,21 @@ func New(options *Options, db string) (*Client, error) { options.Jira.HttpClient = options.HttpClient tracker, err := jira.New(options.Jira) if err != nil { - return nil, errors.Wrap(err, reportingClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) } client.trackers = append(client.trackers, tracker) } if options.MarkdownExporter != nil { exporter, err := markdown.New(options.MarkdownExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } if options.SarifExporter != nil { exporter, err := sarif.New(options.SarifExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } @@ -166,7 +137,7 @@ func New(options *Options, db string) (*Client, error) { options.ElasticsearchExporter.HttpClient = options.HttpClient exporter, err := es.New(options.ElasticsearchExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } @@ -174,7 +145,7 @@ func New(options *Options, db string) (*Client, error) { options.SplunkExporter.HttpClient = options.HttpClient exporter, err := splunk.New(options.SplunkExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } @@ -191,7 +162,7 @@ func New(options *Options, db string) (*Client, error) { func CreateConfigIfNotExists() error { config, err := config.GetConfigDir() if err != nil { - return errors.Wrap(err, "could not get config directory") + return errorutil.NewWithErr(err).Msgf("could not get config directory") } reportingConfig := filepath.Join(config, "report-config.yaml") @@ -213,7 +184,7 @@ func CreateConfigIfNotExists() error { } reportingFile, err := os.Create(reportingConfig) if err != nil { - return errors.Wrap(err, "could not create config file") + return errorutil.NewWithErr(err).Msgf("could not create config file") } defer reportingFile.Close() @@ -222,17 +193,17 @@ func CreateConfigIfNotExists() error { } // RegisterTracker registers a custom tracker to the reporter -func (c *Client) RegisterTracker(tracker Tracker) { +func (c *ReportingClient) RegisterTracker(tracker Tracker) { c.trackers = append(c.trackers, tracker) } // RegisterExporter registers a custom exporter to the reporter -func (c *Client) RegisterExporter(exporter Exporter) { +func (c *ReportingClient) RegisterExporter(exporter Exporter) { c.exporters = append(c.exporters, exporter) } // Close closes the issue tracker reporting client -func (c *Client) Close() { +func (c *ReportingClient) Close() { c.dedupe.Close() for _, exporter := range c.exporters { exporter.Close() @@ -240,7 +211,7 @@ func (c *Client) Close() { } // CreateIssue creates an issue in the tracker -func (c *Client) CreateIssue(event *output.ResultEvent) error { +func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error { if c.options.AllowList != nil && !c.options.AllowList.GetMatch(event) { return nil } @@ -264,15 +235,10 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error { return err } -func stringSliceContains(slice []string, item string) bool { - for _, i := range slice { - if strings.EqualFold(i, item) { - return true - } - } - return false -} - -func (c *Client) GetReportingOptions() *Options { +func (c *ReportingClient) GetReportingOptions() *Options { return c.options } + +func (c *ReportingClient) Clear() { + c.dedupe.Clear() +} diff --git a/v2/pkg/templates/signer/default.go b/v2/pkg/templates/signer/default.go new file mode 100644 index 000000000..bf8b08e03 --- /dev/null +++ b/v2/pkg/templates/signer/default.go @@ -0,0 +1,7 @@ +package signer + +var DefaultVerifier *Signer + +func init() { + DefaultVerifier, _ = NewVerifier(&Options{PublicKeyData: ecdsaPublicKey, Algorithm: ECDSA}) +} diff --git a/v2/pkg/templates/signer/ecdsa_public_key.go b/v2/pkg/templates/signer/ecdsa_public_key.go new file mode 100644 index 000000000..ea32cbfc1 --- /dev/null +++ b/v2/pkg/templates/signer/ecdsa_public_key.go @@ -0,0 +1,8 @@ +package signer + +import ( + _ "embed" +) + +//go:embed ecdsa_public_key.pem +var ecdsaPublicKey []byte diff --git a/v2/pkg/templates/signer/ecdsa_public_key.pem b/v2/pkg/templates/signer/ecdsa_public_key.pem new file mode 100644 index 000000000..3fe5969e7 --- /dev/null +++ b/v2/pkg/templates/signer/ecdsa_public_key.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +-----END PUBLIC KEY----- diff --git a/v2/pkg/templates/signer/options.go b/v2/pkg/templates/signer/options.go new file mode 100644 index 000000000..8cb94bb6e --- /dev/null +++ b/v2/pkg/templates/signer/options.go @@ -0,0 +1,34 @@ +package signer + +import ( + "errors" + "math/big" + "regexp" +) + +type AlgorithmType uint8 + +const ( + RSA AlgorithmType = iota + ECDSA +) + +type Options struct { + PrivateKeyName string + PrivateKeyData []byte + PassphraseName string + PassphraseData []byte + PublicKeyName string + PublicKeyData []byte + Algorithm AlgorithmType +} + +type EcdsaSignature struct { + R *big.Int + S *big.Int +} + +var ( + ReDigest = regexp.MustCompile(`(?m)^#\sdigest:\s.+$`) + ErrUnknownAlgorithm = errors.New("unknown algorithm") +) diff --git a/v2/pkg/templates/signer/signer.go b/v2/pkg/templates/signer/signer.go new file mode 100644 index 000000000..87e3cff73 --- /dev/null +++ b/v2/pkg/templates/signer/signer.go @@ -0,0 +1,234 @@ +package signer + +import ( + "bytes" + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/gob" + "encoding/pem" + "errors" + "fmt" + "os" + + fileutil "github.com/projectdiscovery/utils/file" + "golang.org/x/crypto/ssh" +) + +type Signer struct { + options *Options + sshSigner ssh.Signer + sshVerifier ssh.PublicKey + ecdsaSigner *ecdsa.PrivateKey + ecdsaVerifier *ecdsa.PublicKey +} + +func New(options *Options) (*Signer, error) { + var ( + privateKeyData, passphraseData, publicKeyData []byte + err error + ) + if options.PrivateKeyName != "" { + privateKeyData, err = readKeyFromFileOrEnv(options.PrivateKeyName) + if err != nil { + return nil, err + } + } else { + privateKeyData = options.PrivateKeyData + } + + if options.PassphraseName != "" { + passphraseData = readKeyFromFileOrEnvWithDefault(options.PassphraseName, []byte{}) + } else { + passphraseData = options.PassphraseData + } + + if options.PublicKeyName != "" { + publicKeyData, err = readKeyFromFileOrEnv(options.PublicKeyName) + if err != nil { + return nil, err + } + } else { + publicKeyData = options.PublicKeyData + } + + signer := &Signer{options: options} + + switch signer.options.Algorithm { + case RSA: + signer.sshSigner, signer.sshVerifier, err = parseRsa(privateKeyData, publicKeyData, passphraseData) + case ECDSA: + signer.ecdsaSigner, signer.ecdsaVerifier, err = parseECDSA(privateKeyData, publicKeyData) + default: + return nil, ErrUnknownAlgorithm + } + + if err != nil { + return nil, err + } + + return signer, nil +} + +func NewVerifier(options *Options) (*Signer, error) { + var ( + publicKeyData []byte + err error + ) + if options.PublicKeyName != "" { + publicKeyData, err = readKeyFromFileOrEnv(options.PrivateKeyName) + if err != nil { + return nil, err + } + } else { + publicKeyData = options.PublicKeyData + } + + signer := &Signer{options: options} + + switch signer.options.Algorithm { + case RSA: + signer.sshVerifier, err = parseRsaPublicKey(publicKeyData) + case ECDSA: + signer.ecdsaVerifier, err = parseECDSAPublicKey(publicKeyData) + default: + return nil, ErrUnknownAlgorithm + } + + if err != nil { + return nil, err + } + + return signer, nil +} + +func (s *Signer) Sign(data []byte) ([]byte, error) { + dataHash := sha256.Sum256(data) + switch s.options.Algorithm { + case RSA: + sshSignature, err := s.sshSigner.Sign(rand.Reader, dataHash[:]) + if err != nil { + return nil, err + } + var signatureData bytes.Buffer + if err := gob.NewEncoder(&signatureData).Encode(sshSignature); err != nil { + return nil, err + } + return signatureData.Bytes(), nil + case ECDSA: + r, s, err := ecdsa.Sign(rand.Reader, s.ecdsaSigner, dataHash[:]) + if err != nil { + return nil, err + } + ecdsaSignature := &EcdsaSignature{R: r, S: s} + var signatureData bytes.Buffer + if err := gob.NewEncoder(&signatureData).Encode(ecdsaSignature); err != nil { + return nil, err + } + return signatureData.Bytes(), nil + default: + return nil, ErrUnknownAlgorithm + } +} + +func (s *Signer) Verify(data, signatureData []byte) (bool, error) { + dataHash := sha256.Sum256(data) + switch s.options.Algorithm { + case RSA: + signature := &ssh.Signature{} + if err := gob.NewDecoder(bytes.NewReader(signatureData)).Decode(&signature); err != nil { + return false, err + } + if err := s.sshVerifier.Verify(dataHash[:], signature); err != nil { + return false, err + } + return true, nil + case ECDSA: + signature := &EcdsaSignature{} + if err := gob.NewDecoder(bytes.NewReader(signatureData)).Decode(&signature); err != nil { + return false, err + } + return ecdsa.Verify(s.ecdsaVerifier, dataHash[:], signature.R, signature.S), nil + default: + return false, ErrUnknownAlgorithm + } +} + +func parseRsa(privateKeyData, passphraseData, publicKeyData []byte) (ssh.Signer, ssh.PublicKey, error) { + privateKey, err := parseRsaPrivateKey(privateKeyData, passphraseData) + if err != nil { + return nil, nil, err + } + + publicKey, err := parseRsaPublicKey(publicKeyData) + if err != nil { + return nil, nil, err + } + + return privateKey, publicKey, nil +} + +func parseRsaPrivateKey(privateKeyData, passphraseData []byte) (ssh.Signer, error) { + if len(passphraseData) > 0 { + return ssh.ParsePrivateKeyWithPassphrase(privateKeyData, passphraseData) + } + return ssh.ParsePrivateKey(privateKeyData) +} + +func parseRsaPublicKey(publicKeyData []byte) (ssh.PublicKey, error) { + publicKey, _, _, _, err := ssh.ParseAuthorizedKey(publicKeyData) + if err != nil { + return nil, err + } + + return publicKey, nil +} + +func parseECDSA(privateKeyData, publicKeyData []byte) (*ecdsa.PrivateKey, *ecdsa.PublicKey, error) { + privateKey, err := parseECDSAPrivateKey(privateKeyData) + if err != nil { + return nil, nil, err + } + publicKey, err := parseECDSAPublicKey(publicKeyData) + if err != nil { + return nil, nil, err + } + return privateKey, publicKey, nil +} + +func parseECDSAPrivateKey(privateKeyData []byte) (*ecdsa.PrivateKey, error) { + blockPriv, _ := pem.Decode(privateKeyData) + return x509.ParseECPrivateKey(blockPriv.Bytes) +} + +func parseECDSAPublicKey(publicKeyData []byte) (*ecdsa.PublicKey, error) { + blockPub, _ := pem.Decode(publicKeyData) + genericPublicKey, err := x509.ParsePKIXPublicKey(blockPub.Bytes) + if err != nil { + return nil, err + } + if publicKey, ok := genericPublicKey.(*ecdsa.PublicKey); ok { + return publicKey, nil + } + + return nil, errors.New("couldn't parse ecdsa public key") +} + +func readKeyFromFileOrEnvWithDefault(keypath string, defaultValue []byte) []byte { + keyValue, err := readKeyFromFileOrEnv(keypath) + if err != nil { + return defaultValue + } + return keyValue +} + +func readKeyFromFileOrEnv(keypath string) ([]byte, error) { + if fileutil.FileExists(keypath) { + return os.ReadFile(keypath) + } + if keydata := os.Getenv(keypath); keydata != "" { + return []byte(keydata), nil + } + return nil, fmt.Errorf("Private key not found in file or environment variable: %s", keypath) +} diff --git a/v2/pkg/templates/signer/util.go b/v2/pkg/templates/signer/util.go new file mode 100644 index 000000000..70f4308c7 --- /dev/null +++ b/v2/pkg/templates/signer/util.go @@ -0,0 +1,50 @@ +package signer + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" +) + +const ( + SignaturePattern = "# digest: " + SignatureFmt = SignaturePattern + "%x" +) + +func RemoveSignatureFromData(data []byte) []byte { + return bytes.Trim(ReDigest.ReplaceAll(data, []byte("")), "\n") +} + +func Sign(sign *Signer, data []byte) (string, error) { + if sign == nil { + return "", errors.New("invalid nil signer") + } + cleanedData := RemoveSignatureFromData(data) + signatureData, err := sign.Sign(cleanedData) + if err != nil { + return "", err + } + + return fmt.Sprintf(SignatureFmt, signatureData), nil +} + +func Verify(sign *Signer, data []byte) (bool, error) { + if sign == nil { + return false, errors.New("invalid nil verifier") + } + digestData := ReDigest.Find(data) + if len(digestData) == 0 { + return false, errors.New("digest not found") + } + + digestData = bytes.TrimSpace(bytes.TrimPrefix(digestData, []byte(SignaturePattern))) + digest, err := hex.DecodeString(string(digestData)) + if err != nil { + return false, err + } + + cleanedData := RemoveSignatureFromData(data) + + return sign.Verify(cleanedData, digest) +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 95bed41e7..54e5f4871 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -42,12 +42,12 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` + ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` // description: | // Info contains metadata information about the template. // examples: // - value: exampleInfoStructure - Info model.Info `yaml:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` + Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` // description: | // Requests contains the http request to make in the template. // examples: @@ -88,20 +88,20 @@ type Template struct { // description: | // Self Contained marks Requests for the template as self-contained - SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + SelfContained bool `yaml:"self-contained,omitempty" json:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` // description: | // Stop execution once first match is found - StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"` + StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"` // description: | // Signature is the request signature method // values: // - "AWS" - Signature http.SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` + Signature http.SignatureTypeHolder `yaml:"signature,omitempty" json:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` // description: | // Variables contains any variables for the current request. - Variables variables.Variable `yaml:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request"` + Variables variables.Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` @@ -109,6 +109,9 @@ type Template struct { Executer protocols.Executer `yaml:"-" json:"-"` Path string `yaml:"-" json:"-"` + + // Verified defines if the template signature is digitally verified + Verified bool `yaml:"-" json:"-"` } // TemplateProtocols is a list of accepted template protocols diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 8f5eb4ac4..496bc10b8 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -20,7 +20,7 @@ var ( Name: "Argument Injection in Ruby Dragonfly", Authors: stringslice.StringSlice{Value: "0xspara"}, SeverityHolder: severity.Holder{Severity: severity.High}, - Reference: stringslice.StringSlice{Value: "https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"}, + Reference: stringslice.NewRaw("https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"), Tags: stringslice.StringSlice{Value: "cve,cve2021,rce,ruby"}, } exampleNormalHTTPRequest = &http.Request{ diff --git a/v2/pkg/templates/templates_test.go b/v2/pkg/templates/templates_test.go index dac8432fd..8b8f3f6b2 100644 --- a/v2/pkg/templates/templates_test.go +++ b/v2/pkg/templates/templates_test.go @@ -1 +1,36 @@ package templates + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func TestTemplateStruct(t *testing.T) { + templatePath := "./tests/match-1.yaml" + bin, err := os.ReadFile(templatePath) + require.Nil(t, err, "failed to load example template") + var yamlTemplate Template + err = yaml.Unmarshal(bin, &yamlTemplate) + require.Nil(t, err, "failed to unmarshal yaml template") + jsonBin, err := json.Marshal(yamlTemplate) + require.Nil(t, err, "failed to marshal template to json") + var jsonTemplate Template + err = json.Unmarshal(jsonBin, &jsonTemplate) + require.Nil(t, err, "failed to unmarshal json template") + + templatePath = "./tests/json-template.json" + bin, err = os.ReadFile(templatePath) + require.Nil(t, err, "failed to load example template") + jsonTemplate = Template{} + err = json.Unmarshal(bin, &jsonTemplate) + require.Nil(t, err, "failed to unmarshal json template") + yamlBin, err := yaml.Marshal(jsonTemplate) + require.Nil(t, err, "failed to marshal template to yaml") + yamlTemplate = Template{} + err = yaml.Unmarshal(yamlBin, &yamlTemplate) + require.Nil(t, err, "failed to unmarshal yaml template") +} diff --git a/v2/pkg/templates/tests/json-template.json b/v2/pkg/templates/tests/json-template.json new file mode 100644 index 000000000..247700a51 --- /dev/null +++ b/v2/pkg/templates/tests/json-template.json @@ -0,0 +1,27 @@ +{ + "id": "go-integration-test", + "info": { + "name": "Basic Go Integration Test", + "author": "pdteam", + "severity": "info" + }, + "requests": [ + { + "method": "GET", + "path": [ + "{{BaseURL}}" + ], + "headers": { + "test": "nuclei" + }, + "matchers": [ + { + "type": "word", + "words": [ + "This is test headers matcher text" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index f7ae8cd33..3c561d152 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -7,7 +7,6 @@ import ( "github.com/alecthomas/jsonschema" "github.com/pkg/errors" - "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" ) diff --git a/v2/pkg/utils/insertion_ordered_map.go b/v2/pkg/utils/insertion_ordered_map.go index 5ec8bdfb6..a9586778a 100644 --- a/v2/pkg/utils/insertion_ordered_map.go +++ b/v2/pkg/utils/insertion_ordered_map.go @@ -1,6 +1,7 @@ package utils import ( + "encoding/json" "fmt" "strconv" @@ -44,6 +45,18 @@ func (insertionOrderedStringMap *InsertionOrderedStringMap) UnmarshalYAML(unmars return nil } +func (insertionOrderedStringMap *InsertionOrderedStringMap) UnmarshalJSON(data []byte) error { + var dataMap map[string]interface{} + if err := json.Unmarshal(data, &dataMap); err != nil { + return err + } + insertionOrderedStringMap.values = make(map[string]interface{}) + for k, v := range dataMap { + insertionOrderedStringMap.Set(k, toString(v)) + } + return nil +} + // toString converts an interface to string in a quick way func toString(data interface{}) string { switch s := data.(type) { diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index 5c71e5f89..b9d5f7c31 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -13,9 +13,9 @@ import ( type Workflow struct { // description: | // Workflows is a list of workflows to execute for a template. - Workflows []*WorkflowTemplate `yaml:"workflows,omitempty" jsonschema:"title=list of workflows to execute,description=List of workflows to execute for template"` + Workflows []*WorkflowTemplate `yaml:"workflows,omitempty" json:"workflows,omitempty" jsonschema:"title=list of workflows to execute,description=List of workflows to execute for template"` - Options *protocols.ExecuterOptions `yaml:"-"` + Options *protocols.ExecuterOptions `yaml:"-" json:"-"` } // WorkflowTemplate is a template to be run as part of a workflow @@ -27,18 +27,18 @@ type WorkflowTemplate struct { // value: "\"dns/worksites-detection.yaml\"" // - name: A template directory // value: "\"misconfigurations/aem\"" - Template string `yaml:"template,omitempty" jsonschema:"title=template/directory to execute,description=Template or directory to execute as part of workflow"` + Template string `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"title=template/directory to execute,description=Template or directory to execute as part of workflow"` // description: | // Tags to run templates based on. - Tags stringslice.StringSlice `yaml:"tags,omitempty" jsonschema:"title=tags to execute,description=Tags to run template based on"` + Tags stringslice.StringSlice `yaml:"tags,omitempty" json:"tags,omitempty" jsonschema:"title=tags to execute,description=Tags to run template based on"` // description: | // Matchers perform name based matching to run subtemplates for a workflow. - Matchers []*Matcher `yaml:"matchers,omitempty" jsonschema:"title=name based template result matchers,description=Matchers perform name based matching to run subtemplates for a workflow"` + Matchers []*Matcher `yaml:"matchers,omitempty" json:"matchers,omitempty" jsonschema:"title=name based template result matchers,description=Matchers perform name based matching to run subtemplates for a workflow"` // description: | // Subtemplates are run if the `template` field Template matches. - Subtemplates []*WorkflowTemplate `yaml:"subtemplates,omitempty" jsonschema:"title=subtemplate based result matchers,description=Subtemplates are ran if the template field Template matches"` + Subtemplates []*WorkflowTemplate `yaml:"subtemplates,omitempty" json:"subtemplates,omitempty" jsonschema:"title=subtemplate based result matchers,description=Subtemplates are ran if the template field Template matches"` // Executers perform the actual execution for the workflow template - Executers []*ProtocolExecuterPair `yaml:"-"` + Executers []*ProtocolExecuterPair `yaml:"-" json:"-"` } // ProtocolExecuterPair is a pair of protocol executer and its options @@ -52,17 +52,17 @@ type ProtocolExecuterPair struct { type Matcher struct { // description: | // Name is the name of the items to match. - Name stringslice.StringSlice `yaml:"name,omitempty" jsonschema:"title=name of items to match,description=Name of items to match"` + Name stringslice.StringSlice `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=name of items to match,description=Name of items to match"` // description: | // Condition is the optional condition between names. By default, // the condition is assumed to be OR. // values: // - "and" // - "or" - Condition string `yaml:"condition,omitempty" jsonschema:"title=condition between names,description=Condition between the names,enum=and,enum=or"` + Condition string `yaml:"condition,omitempty" json:"condition,omitempty" jsonschema:"title=condition between names,description=Condition between the names,enum=and,enum=or"` // description: | // Subtemplates are run if the name of matcher matches. - Subtemplates []*WorkflowTemplate `yaml:"subtemplates,omitempty" jsonschema:"title=templates to run after match,description=Templates to run after match"` + Subtemplates []*WorkflowTemplate `yaml:"subtemplates,omitempty" json:"subtemplates,omitempty" jsonschema:"title=templates to run after match,description=Templates to run after match"` condition ConditionType }