diff --git a/Dockerfile b/Dockerfile index 322fd54d8..21822611b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.3-alpine as build-env +FROM golang:1.17.4-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.15.0 diff --git a/v2/go.mod b/v2/go.mod index 490908476..38a1594d3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -65,7 +65,7 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e +require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect diff --git a/v2/go.sum b/v2/go.sum index bd73d8ced..3df87760e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -595,10 +595,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/folderutil v0.0.0-20210804143510-68474319fd84 h1:+VqGxv8ywpIHwGGSCOcGn/q5kkuB6F1AZtY42I8VnXc= -github.com/projectdiscovery/folderutil v0.0.0-20210804143510-68474319fd84/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= -github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e h1:ozfSeEc5j1f7NCEZAiAskP/KYfBD/TzPmFTIfh+CEwE= -github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= +github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e h1:RJJuYyuwskYtzZi2gziy6SE/b7saWEzyskaA252E0VY= +github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk= github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 99d94cc88..1cf4ccf5b 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -24,6 +24,7 @@ import ( "github.com/olekukonko/tablewriter" "github.com/pkg/errors" + "github.com/projectdiscovery/folderutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei-updatecheck-api/client" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" @@ -401,8 +402,17 @@ func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory stri return "", true, nil } - directoryPathChunks := strings.Split(directory, string(os.PathSeparator)) - relativeDirectoryPathWithoutZipRoot := filepath.Join(directoryPathChunks[1:]...) + var ( + directoryPathChunks []string + relativeDirectoryPathWithoutZipRoot string + ) + if folderutil.IsUnixOS() { + directoryPathChunks = strings.Split(directory, string(os.PathSeparator)) + } else if folderutil.IsWindowsOS() { + pathInfo, _ := folderutil.NewPathInfo(directory) + directoryPathChunks = pathInfo.Parts + } + relativeDirectoryPathWithoutZipRoot = filepath.Join(directoryPathChunks[1:]...) if strings.HasPrefix(relativeDirectoryPathWithoutZipRoot, ".") { return "", true, nil diff --git a/v2/pkg/catalog/path.go b/v2/pkg/catalog/path.go index 76f387d63..06ebb2d88 100644 --- a/v2/pkg/catalog/path.go +++ b/v2/pkg/catalog/path.go @@ -4,6 +4,10 @@ import ( "fmt" "os" "path/filepath" + + "github.com/pkg/errors" + + "github.com/projectdiscovery/folderutil" ) // ResolvePath resolves the path to an absolute one in various ways. @@ -15,11 +19,10 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) { if filepath.IsAbs(templateName) { return templateName, nil } - if second != "" { secondBasePath := filepath.Join(filepath.Dir(second), templateName) - if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) { - return secondBasePath, nil + if potentialPath, err := c.tryResolve(secondBasePath); err != errNoValidCombination { + return potentialPath, nil } } @@ -29,15 +32,37 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) { } templatePath := filepath.Join(curDirectory, templateName) - if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - return templatePath, nil + if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { + return potentialPath, nil } if c.templatesDirectory != "" { templatePath := filepath.Join(c.templatesDirectory, templateName) - if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - return templatePath, nil + if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { + return potentialPath, nil } } return "", fmt.Errorf("no such path found: %s", templateName) } + +var errNoValidCombination = errors.New("no valid combination found") + +// tryResolve attempts to load locate the target by iterating across all the folders tree +func (c *Catalog) tryResolve(fullpath string) (string, error) { + dir, filename := filepath.Split(fullpath) + pathInfo, err := folderutil.NewPathInfo(dir) + if err != nil { + return "", err + } + pathInfoItems, err := pathInfo.MeshWith(filename) + if err != nil { + return "", err + } + for _, pathInfoItem := range pathInfoItems { + if _, err := os.Stat(pathInfoItem); !os.IsNotExist(err) { + return pathInfoItem, nil + } + } + + return "", errNoValidCombination +} diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 9e1a9f2ad..5b4431027 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -15,10 +15,12 @@ import ( "math/rand" "net/url" "regexp" + "strconv" "strings" "time" "github.com/Knetic/govaluate" + "github.com/logrusorgru/aurora" "github.com/spaolacci/murmur3" "github.com/projectdiscovery/gologger" @@ -27,374 +29,434 @@ import ( ) const ( - numbers = "1234567890" - letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - withCutSetArgsSize = 2 - withBaseRandArgsSize = 3 - withMaxRandArgsSize = withCutSetArgsSize + numbers = "1234567890" + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" ) -var ErrDSLArguments = errors.New("invalid arguments provided to dsl") +var invalidDslFunctionError = errors.New("invalid DSL function signature") +var invalidDslFunctionMessageTemplate = "%w. correct method signature %q" -var functions = map[string]govaluate.ExpressionFunction{ - "len": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - length := len(types.ToString(args[0])) - return float64(length), nil - }, - "toupper": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return strings.ToUpper(types.ToString(args[0])), nil - }, - "tolower": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return strings.ToLower(types.ToString(args[0])), nil - }, - "replace": func(args ...interface{}) (interface{}, error) { - if len(args) != 3 { - return nil, ErrDSLArguments - } - return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil - }, - "replace_regex": func(args ...interface{}) (interface{}, error) { - if len(args) != 3 { - return nil, ErrDSLArguments - } - compiled, err := regexp.Compile(types.ToString(args[1])) - if err != nil { - return nil, err - } - return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil - }, - "trim": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimleft": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimright": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimspace": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return strings.TrimSpace(types.ToString(args[0])), nil - }, - "trimprefix": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimsuffix": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "reverse": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return reverseString(types.ToString(args[0])), nil - }, - // encoding - "base64": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) +var dslFunctions map[string]dslFunction - return sEnc, nil - }, - "gzip": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - buffer := &bytes.Buffer{} - writer := gzip.NewWriter(buffer) - if _, err := writer.Write([]byte(args[0].(string))); err != nil { - return "", err - } - _ = writer.Close() +type dslFunction struct { + signature string + expressFunc govaluate.ExpressionFunction +} - return buffer.String(), nil - }, - // python encodes to base64 with lines of 76 bytes terminated by new line "\n" - "base64_py": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) - return deserialization.InsertInto(sEnc, 76, '\n'), nil - }, - "base64_decode": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return base64.StdEncoding.DecodeString(types.ToString(args[0])) - }, - "url_encode": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return url.QueryEscape(types.ToString(args[0])), nil - }, - "url_decode": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return url.QueryUnescape(types.ToString(args[0])) - }, - "hex_encode": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return hex.EncodeToString([]byte(types.ToString(args[0]))), nil - }, - "hex_decode": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - hx, _ := hex.DecodeString(types.ToString(args[0])) - return string(hx), nil - }, - "html_escape": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return html.EscapeString(types.ToString(args[0])), nil - }, - "html_unescape": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return html.UnescapeString(types.ToString(args[0])), nil - }, - // hashing - "md5": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - hash := md5.Sum([]byte(types.ToString(args[0]))) +func init() { + tempDslFunctions := map[string]func(string) dslFunction{ + "len": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + length := len(types.ToString(args[0])) + return float64(length), nil + }), + "to_upper": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return strings.ToUpper(types.ToString(args[0])), nil + }), + "to_lower": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return strings.ToLower(types.ToString(args[0])), nil + }), + "replace": makeDslFunction(3, func(args ...interface{}) (interface{}, error) { + return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil + }), + "replace_regex": makeDslFunction(3, func(args ...interface{}) (interface{}, error) { + compiled, err := regexp.Compile(types.ToString(args[1])) + if err != nil { + return nil, err + } + return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil + }), + "trim": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil + }), + "trim_left": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil + }), + "trim_right": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil + }), + "trim_space": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return strings.TrimSpace(types.ToString(args[0])), nil + }), + "trim_prefix": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil + }), + "trim_suffix": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil + }), + "reverse": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return reverseString(types.ToString(args[0])), nil + }), + "base64": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))), nil + }), + "gzip": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + buffer := &bytes.Buffer{} + writer := gzip.NewWriter(buffer) + if _, err := writer.Write([]byte(args[0].(string))); err != nil { + return "", err + } + _ = writer.Close() - return hex.EncodeToString(hash[:]), nil - }, - "sha256": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - h := sha256.New() - if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { - return nil, err - } - return hex.EncodeToString(h.Sum(nil)), nil - }, - "sha1": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - h := sha1.New() - if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { - return nil, err - } - return hex.EncodeToString(h.Sum(nil)), nil - }, - "mmh3": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil - }, - // search - "contains": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "regex": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - compiled, err := regexp.Compile(types.ToString(args[0])) - if err != nil { - return nil, err - } - return compiled.MatchString(types.ToString(args[1])), nil - }, - // random generators - "rand_char": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - chars := letters + numbers - bad := "" - if len(args) >= 1 { - chars = types.ToString(args[0]) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return chars[rand.Intn(len(chars))], nil - }, - "rand_base": func(args ...interface{}) (interface{}, error) { - if len(args) != 3 { - return nil, ErrDSLArguments - } - l := 0 - bad := "" - base := letters + numbers + return buffer.String(), nil + }), + "base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + // python encodes to base64 with lines of 76 bytes terminated by new line "\n" + stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) + return deserialization.InsertInto(stdBase64, 76, '\n'), nil + }), + "base64_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return base64.StdEncoding.DecodeString(types.ToString(args[0])) + }), + "url_encode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return url.QueryEscape(types.ToString(args[0])), nil + }), + "url_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return url.QueryUnescape(types.ToString(args[0])) + }), + "hex_encode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return hex.EncodeToString([]byte(types.ToString(args[0]))), nil + }), + "hex_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + decodeString, err := hex.DecodeString(types.ToString(args[0])) + return decodeString, err + }), + "html_escape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return html.EscapeString(types.ToString(args[0])), nil + }), + "html_unescape": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return html.UnescapeString(types.ToString(args[0])), nil + }), + "md5": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + hash := md5.Sum([]byte(types.ToString(args[0]))) + return hex.EncodeToString(hash[:]), nil + }), + "sha256": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + hash := sha256.New() + if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil { + return nil, err + } + return hex.EncodeToString(hash.Sum(nil)), nil + }), + "sha1": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + hash := sha1.New() + if _, err := hash.Write([]byte(types.ToString(args[0]))); err != nil { + return nil, err + } + return hex.EncodeToString(hash.Sum(nil)), nil + }), + "mmh3": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { + return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil + }), + "contains": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil + }), + "regex": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + compiled, err := regexp.Compile(types.ToString(args[0])) + if err != nil { + return nil, err + } + return compiled.MatchString(types.ToString(args[1])), nil + }), + "remove_bad_chars": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { + input := types.ToString(args[0]) + badChars := types.ToString(args[1]) + return trimAll(input, badChars), nil + }), + "rand_char": makeDslWithOptionalArgsFunction( + "(optionalCharSet string) string", + func(args ...interface{}) (interface{}, error) { + charSet := letters + numbers - if len(args) >= 1 { - l = int(args[0].(float64)) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - if len(args) >= withBaseRandArgsSize { - base = types.ToString(args[2]) - } - base = trimAll(base, bad) - return randSeq(base, l), nil - }, - "rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - l := 0 - bad := "" - chars := letters + numbers + argSize := len(args) + if argSize != 0 && argSize != 1 { + return nil, invalidDslFunctionError + } - if len(args) >= 1 { - l = int(args[0].(float64)) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return randSeq(chars, l), nil - }, - "rand_text_alpha": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - l := 0 - bad := "" - chars := letters + if argSize >= 1 { + charSet = types.ToString(args[0]) + } - if len(args) >= 1 { - l = int(args[0].(float64)) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return randSeq(chars, l), nil - }, - "rand_text_numeric": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - l := 0 - bad := "" - chars := numbers + return charSet[rand.Intn(len(charSet))], nil + }, + ), + "rand_base": makeDslWithOptionalArgsFunction( + "(length uint, optionalCharSet string) string", + func(args ...interface{}) (interface{}, error) { + var length int + charSet := letters + numbers - if len(args) >= 1 { - l = int(args[0].(float64)) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return randSeq(chars, l), nil - }, - "rand_int": func(args ...interface{}) (interface{}, error) { - if len(args) != 2 { - return nil, ErrDSLArguments - } - min := 0 - max := math.MaxInt32 + argSize := len(args) + if argSize < 1 || argSize > 3 { + return nil, invalidDslFunctionError + } - if len(args) >= 1 { - min = int(args[0].(float64)) - } - if len(args) >= withMaxRandArgsSize { - max = int(args[1].(float64)) - } - return rand.Intn(max-min) + min, nil - }, - "unixtime": func(args ...interface{}) (interface{}, error) { - seconds := 0 - if len(args) >= 1 { - seconds = int(args[0].(float64)) - } - now := time.Now() - offset := now.Add(time.Duration(seconds) * time.Second) - return float64(offset.Unix()), nil - }, - // Time Functions - "waitfor": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - seconds := args[0].(float64) - time.Sleep(time.Duration(seconds) * time.Second) - return true, nil - }, - // deserialization Functions - "generate_java_gadget": func(args ...interface{}) (interface{}, error) { - if len(args) != 3 { - return nil, ErrDSLArguments - } - gadget := args[0].(string) - cmd := args[1].(string) + length = int(args[0].(float64)) - var encoding string - if len(args) > 2 { - encoding = args[2].(string) + if argSize == 2 { + charSet = types.ToString(args[1]) + } + return randSeq(charSet, length), nil + }, + ), + "rand_text_alphanumeric": makeDslWithOptionalArgsFunction( + "(length uint, optionalBadChars string) string", + func(args ...interface{}) (interface{}, error) { + length := 0 + badChars := "" + + argSize := len(args) + if argSize != 1 && argSize != 2 { + return nil, invalidDslFunctionError + } + + length = int(args[0].(float64)) + + if argSize == 2 { + badChars = types.ToString(args[1]) + } + chars := trimAll(letters+numbers, badChars) + return randSeq(chars, length), nil + }, + ), + "rand_text_alpha": makeDslWithOptionalArgsFunction( + "(length uint, optionalBadChars string) string", + func(args ...interface{}) (interface{}, error) { + var length int + badChars := "" + + argSize := len(args) + if argSize != 1 && argSize != 2 { + return nil, invalidDslFunctionError + } + + length = int(args[0].(float64)) + + if argSize == 2 { + badChars = types.ToString(args[1]) + } + chars := trimAll(letters, badChars) + return randSeq(chars, length), nil + }, + ), + "rand_text_numeric": makeDslWithOptionalArgsFunction( + "(length uint, optionalBadNumbers string) string", + func(args ...interface{}) (interface{}, error) { + argSize := len(args) + if argSize != 1 && argSize != 2 { + return nil, invalidDslFunctionError + } + + length := args[0].(int) + badNumbers := "" + + if argSize == 2 { + badNumbers = types.ToString(args[1]) + } + + chars := trimAll(numbers, badNumbers) + return randSeq(chars, length), nil + }, + ), + "rand_int": makeDslWithOptionalArgsFunction( + "(optionalMin, optionalMax uint) int", + func(args ...interface{}) (interface{}, error) { + argSize := len(args) + if argSize >= 2 { + return nil, invalidDslFunctionError + } + + min := 0 + max := math.MaxInt32 + + if argSize >= 1 { + min = args[0].(int) + } + if argSize == 2 { + max = args[1].(int) + } + return rand.Intn(max-min) + min, nil + }, + ), + "generate_java_gadget": makeDslFunction(3, func(args ...interface{}) (interface{}, error) { + gadget := args[0].(string) + cmd := args[1].(string) + encoding := args[2].(string) + data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) + return data, nil + }), + "unix_time": makeDslWithOptionalArgsFunction( + "(optionalSeconds uint) float64", + func(args ...interface{}) (interface{}, error) { + seconds := 0 + + argSize := len(args) + if argSize != 0 && argSize != 1 { + return nil, invalidDslFunctionError + } else if argSize == 1 { + seconds = int(args[0].(uint)) + } + + offset := time.Now().Add(time.Duration(seconds) * time.Second) + return float64(offset.Unix()), nil + }, + ), + "wait_for": makeDslWithOptionalArgsFunction( + "(seconds uint)", + func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, invalidDslFunctionError + } + seconds := args[0].(uint) + time.Sleep(time.Duration(seconds) * time.Second) + return true, nil + }, + ), + "print_debug": makeDslWithOptionalArgsFunction( + "(args ...interface{})", + func(args ...interface{}) (interface{}, error) { + if len(args) < 1 { + return nil, invalidDslFunctionError + } + gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) + return true, nil + }, + ), + } + + dslFunctions = make(map[string]dslFunction, len(tempDslFunctions)) + for funcName, dslFunc := range tempDslFunctions { + dslFunctions[funcName] = dslFunc(funcName) + } +} + +func createSignaturePart(numberOfParameters int) string { + params := make([]string, 0, numberOfParameters) + for i := 1; i <= numberOfParameters; i++ { + params = append(params, "arg"+strconv.Itoa(i)) + } + return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", ")) +} + +func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction { + return func(functionName string) dslFunction { + return dslFunction{ + functionName + signaturePart, + dslFunctionLogic, } - data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) - return data, nil - }, - // for debug purposes - "print_debug": func(args ...interface{}) (interface{}, error) { - gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) - return true, nil - }, + } +} + +func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction { + return func(functionName string) dslFunction { + signature := functionName + createSignaturePart(numberOfParameters) + return dslFunction{ + signature, + func(args ...interface{}) (interface{}, error) { + if len(args) != numberOfParameters { + return nil, fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature) + } + return dslFunctionLogic(args...) + }, + } + } } // HelperFunctions returns the dsl helper functions func HelperFunctions() map[string]govaluate.ExpressionFunction { - return functions + helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions)) + + for functionName, dslFunction := range dslFunctions { + helperFunctions[functionName] = dslFunction.expressFunc + helperFunctions[strings.ReplaceAll(functionName, "_", "")] = dslFunction.expressFunc // for backwards compatibility + } + + return helperFunctions } // AddHelperFunction allows creation of additional helper functions to be supported with templates +//goland:noinspection GoUnusedExportedFunction func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error { - if _, ok := functions[key]; !ok { - functions[key] = value + if _, ok := dslFunctions[key]; !ok { + dslFunction := dslFunctions[key] + dslFunction.signature = "(args ...interface{}) interface{}" + dslFunction.expressFunc = value return nil } return errors.New("duplicate helper function key defined") } +func GetPrintableDslFunctionSignatures(noColor bool) string { + aggregateSignatures := func(values []string) string { + builder := &strings.Builder{} + for _, value := range values { + builder.WriteRune('\t') + builder.WriteString(value) + builder.WriteRune('\n') + } + return builder.String() + } + + if noColor { + return aggregateSignatures(getDslFunctionSignatures()) + } + return aggregateSignatures(colorizeDslFunctionSignatures()) +} + +func getDslFunctionSignatures() []string { + result := make([]string, 0, len(dslFunctions)) + + for _, dslFunction := range dslFunctions { + result = append(result, dslFunction.signature) + } + + return result +} + +var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`) + +func colorizeDslFunctionSignatures() []string { + signatures := getDslFunctionSignatures() + + colorToOrange := func(value string) string { + return aurora.Index(208, value).String() + } + + result := make([]string, 0, len(signatures)) + + for _, signature := range signatures { + subMatchSlices := functionSignaturePattern.FindAllStringSubmatch(signature, -1) + if len(subMatchSlices) != 1 { + // TODO log when #1166 is implemented + return signatures + } + matches := subMatchSlices[0] + if len(matches) != 5 { + // TODO log when #1166 is implemented + return signatures + } + + functionParameters := strings.Split(matches[2], ",") + + var coloredParameterAndTypes []string + for _, functionParameter := range functionParameters { + functionParameter = strings.TrimSpace(functionParameter) + paramAndType := strings.Split(functionParameter, " ") + if len(paramAndType) == 1 { + coloredParameterAndTypes = append(coloredParameterAndTypes, paramAndType[0]) + } else if len(paramAndType) == 2 { + coloredParameterAndTypes = append(coloredParameterAndTypes, fmt.Sprintf("%s %s", paramAndType[0], colorToOrange(paramAndType[1]))) + } + } + + highlightedParams := strings.TrimSpace(fmt.Sprintf("%s %s", strings.Join(coloredParameterAndTypes, ", "), colorToOrange(matches[3]))) + colorizedDslSignature := fmt.Sprintf("%s(%s)%s", aurora.BrightYellow(matches[1]).String(), highlightedParams, colorToOrange(matches[4])) + + result = append(result, colorizedDslSignature) + } + + return result +} + func reverseString(s string) string { runes := []rune(s) for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index e5bfb0d64..f50447d6d 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -2,12 +2,14 @@ package dsl import ( "compress/gzip" + "fmt" "io/ioutil" "strings" "testing" "time" "github.com/Knetic/govaluate" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -46,3 +48,91 @@ func TestDSLGzipSerialize(t *testing.T) { require.Equal(t, "hello world", string(data), "could not get gzip encoded data") } + +func Test1(t *testing.T) { + type testCase struct { + methodName string + arguments []interface{} + expected interface{} + err string + } + + toUpperSignatureError := createSignatureError("to_upper(arg1 interface{}) interface{}") + removeBadCharsSignatureError := createSignatureError("remove_bad_chars(arg1, arg2 interface{}) interface{}") + + testCases := []testCase{ + {"to_upper", []interface{}{}, nil, toUpperSignatureError}, + {"to_upper", []interface{}{"a"}, "A", ""}, + {"toupper", []interface{}{"a"}, "A", ""}, + {"to_upper", []interface{}{"a", "b", "c"}, nil, toUpperSignatureError}, + + {"remove_bad_chars", []interface{}{}, nil, removeBadCharsSignatureError}, + {"remove_bad_chars", []interface{}{"a"}, nil, removeBadCharsSignatureError}, + {"remove_bad_chars", []interface{}{"abba baab", "b"}, "aa aa", ""}, + {"remove_bad_chars", []interface{}{"a", "b", "c"}, nil, removeBadCharsSignatureError}, + } + + helperFunctions := HelperFunctions() + for _, currentTestCase := range testCases { + methodName := currentTestCase.methodName + t.Run(methodName, func(t *testing.T) { + actualResult, err := helperFunctions[methodName](currentTestCase.arguments...) + + if currentTestCase.err == "" { + assert.Nil(t, err) + } else { + assert.Equal(t, err.Error(), currentTestCase.err) + } + assert.Equal(t, currentTestCase.expected, actualResult) + }) + } +} + +func createSignatureError(signature string) string { + return fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature).Error() +} + +func Test(t *testing.T) { + expectedColorizedSignatures := []string{ + "\x1b[93mbase64_py\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mprint_debug\x1b[0m(args \x1b[38;5;208m...interface{}\x1b[0m)\x1b[38;5;208m\x1b[0m", + "\x1b[93mregex\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mmmh3\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mto_lower\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mmd5\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mreplace_regex\x1b[0m(arg1, arg2, arg3 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mhtml_unescape\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mhex_encode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mrand_base\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalCharSet \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m", + "\x1b[93msha1\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mtrim_right\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mwait_for\x1b[0m(seconds \x1b[38;5;208muint\x1b[0m)\x1b[38;5;208m\x1b[0m", + "\x1b[93mtrim\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93murl_encode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mto_upper\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mrand_text_alpha\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalBadChars \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m", + "\x1b[93msha256\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mgzip\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mlen\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mtrim_space\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mrand_int\x1b[0m(optionalMin, optionalMax \x1b[38;5;208muint\x1b[0m)\x1b[38;5;208m int\x1b[0m", + "\x1b[93mremove_bad_chars\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mrand_char\x1b[0m(optionalCharSet \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m", + "\x1b[93mreverse\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mhtml_escape\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mbase64\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mbase64_decode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mhex_decode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mtrim_prefix\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93murl_decode\x1b[0m(arg1 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mreplace\x1b[0m(arg1, arg2, arg3 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mtrim_suffix\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mrand_text_numeric\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalBadNumbers \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m", + "\x1b[93mcontains\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mgenerate_java_gadget\x1b[0m(arg1, arg2, arg3 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93munix_time\x1b[0m(optionalSeconds \x1b[38;5;208muint\x1b[0m)\x1b[38;5;208m float64\x1b[0m", + "\x1b[93mtrim_left\x1b[0m(arg1, arg2 \x1b[38;5;208minterface{}\x1b[0m)\x1b[38;5;208m interface{}\x1b[0m", + "\x1b[93mrand_text_alphanumeric\x1b[0m(length \x1b[38;5;208muint\x1b[0m, optionalBadChars \x1b[38;5;208mstring\x1b[0m)\x1b[38;5;208m string\x1b[0m", + } + assert.ElementsMatch(t, expectedColorizedSignatures, colorizeDslFunctionSignatures()) +} diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index b571085c4..3a199ce32 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -55,12 +55,12 @@ func (matcher *Matcher) CompileMatchers() error { } // Compile the dsl expressions - for _, expr := range matcher.DSL { - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) + for _, dslExpression := range matcher.DSL { + compiledExpression, err := govaluate.NewEvaluableExpressionWithFunctions(dslExpression, dsl.HelperFunctions()) if err != nil { - return fmt.Errorf("could not compile dsl: %s", expr) + return &DslCompilationError{DslSignature: dslExpression, WrappedError: err} } - matcher.dslCompiled = append(matcher.dslCompiled, compiled) + matcher.dslCompiled = append(matcher.dslCompiled, compiledExpression) } // Set up the condition type, if any. @@ -83,3 +83,16 @@ func (matcher *Matcher) CompileMatchers() error { } return nil } + +type DslCompilationError struct { + DslSignature string + WrappedError error +} + +func (e *DslCompilationError) Error() string { + return fmt.Sprintf("could not compile DSL expression: %s. %v", e.DslSignature, e.WrappedError) +} + +func (e *DslCompilationError) Unwrap() error { + return e.WrappedError +} diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 22fb9e108..898eaca0a 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -57,17 +58,19 @@ func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) ( var err error word, err = expressions.Evaluate(word, data) if err != nil { + gologger.Warning().Msgf("Error while evaluating word matcher: %q", word) continue } // Continue if the word doesn't match if !strings.Contains(corpus, word) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if matcher.condition == ANDCondition { + switch matcher.condition { + case ANDCondition: return false, []string{} + case ORCondition: + continue } - // Continue with the flow since it's an OR Condition. - continue } // If the condition was an OR, return on the first match. @@ -94,11 +97,12 @@ func (matcher *Matcher) MatchRegex(corpus string) (bool, []string) { if !regex.MatchString(corpus) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if matcher.condition == ANDCondition { + switch matcher.condition { + case ANDCondition: return false, []string{} + case ORCondition: + continue } - // Continue with the flow since it's an OR Condition. - continue } currentMatches := regex.FindAllString(corpus, -1) @@ -125,11 +129,12 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { if !strings.Contains(corpus, binary) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if matcher.condition == ANDCondition { + switch matcher.condition { + case ANDCondition: return false, []string{} + case ORCondition: + continue } - // Continue with the flow since it's an OR Condition. - continue } // If the condition was an OR, return on the first match. @@ -149,37 +154,43 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { // MatchDSL matches on a generic map result func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { + logExpressionEvaluationFailure := func (matcherName string, err error) { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error()) + } + // Iterate over all the expressions accepted as valid for i, expression := range matcher.dslCompiled { if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil { resolvedExpression, err := expressions.Evaluate(expression.String(), data) if err != nil { - gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error()) + logExpressionEvaluationFailure(matcher.Name, err) return false } expression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions()) if err != nil { - gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error()) + logExpressionEvaluationFailure(matcher.Name, err) return false } } + result, err := expression.Evaluate(data) if err != nil { + gologger.Warning().Msgf(err.Error()) continue } - var bResult bool - bResult, ok := result.(bool) - - // Continue if the regex doesn't match - if !ok || !bResult { + if boolResult, ok := result.(bool); !ok { + gologger.Warning().Msgf("The return value of a DSL statement must return a boolean value.") + continue + } else if !boolResult { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if matcher.condition == ANDCondition { + switch matcher.condition { + case ANDCondition: return false + case ORCondition: + continue } - // Continue with the flow since it's an OR Condition. - continue } // If the condition was an OR, return on the first match. diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 16a3ede71..4a4d88ec4 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -1,9 +1,14 @@ package executer import ( + "fmt" "strings" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" @@ -24,8 +29,20 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption // Compile compiles the execution generators preparing any requests possible. func (e *Executer) Compile() error { + cliOptions := e.options.Options + for _, request := range e.requests { if err := request.Compile(e.options); err != nil { + var dslCompilationError *matchers.DslCompilationError + if errors.As(err, &dslCompilationError) { + if cliOptions.Verbose { + rawErrorMessage := dslCompilationError.Error() + formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "." + gologger.Warning().Msgf(formattedErrorMessage) + gologger.Info().Msgf("The available custom DSL functions are:") + fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor)) + } + } return err } } diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index ca8f9b655..fb1597d25 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -37,7 +37,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } generator := &PayloadGenerator{} - if err := generator.validate(payloads, templatePath); err != nil { + if err := generator.validate(payloadsFinal, templatePath); err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index b04f8034c..7bdea4c84 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "github.com/projectdiscovery/folderutil" @@ -20,21 +21,17 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat return errors.New("invalid number of lines in payload") } - // check if it's a worldlist file and try to load it + // check if it's a file and try to load it if fileExists(payloadType) { continue } changed := false - templatePathInfo, err := folderutil.NewPathInfo(templatePath) - if err != nil { - return err - } - payloadPathsToProbe, err := templatePathInfo.MeshWith(payloadType) - if err != nil { - return err - } + dir, _ := filepath.Split(templatePath) + templatePathInfo, _ := folderutil.NewPathInfo(dir) + payloadPathsToProbe, _ := templatePathInfo.MeshWith(payloadType) + for _, payloadPath := range payloadPathsToProbe { if fileExists(payloadPath) { payloads[name] = payloadPath diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index a79e80124..fbb34f425 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -82,14 +82,14 @@ func New(option *Options) (*Exporter, error) { } // Export exports a passed result event to elasticsearch -func (i *Exporter) Export(event *output.ResultEvent) error { +func (exporter *Exporter) Export(event *output.ResultEvent) error { // creating a request - req, err := http.NewRequest(http.MethodPost, i.url, nil) + req, err := http.NewRequest(http.MethodPost, exporter.url, nil) if err != nil { return errors.Wrap(err, "could not make request") } - if len(i.authentication) > 0 { - req.Header.Add("Authorization", i.authentication) + if len(exporter.authentication) > 0 { + req.Header.Add("Authorization", exporter.authentication) } req.Header.Add("Content-Type", "application/json") @@ -103,7 +103,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { } req.Body = ioutil.NopCloser(bytes.NewReader(b)) - res, err := i.elasticsearch.Do(req) + res, err := exporter.elasticsearch.Do(req) if err != nil { return err } @@ -120,6 +120,6 @@ func (i *Exporter) Export(event *output.ResultEvent) error { } // Close closes the exporter after operation -func (i *Exporter) Close() error { +func (exporter *Exporter) Close() error { return nil } diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 31c356af9..b86e4e9ec 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -37,7 +37,7 @@ func New(options *Options) (*Exporter, error) { } // Export exports a passed result event to markdown -func (i *Exporter) Export(event *output.ResultEvent) error { +func (exporter *Exporter) Export(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) @@ -66,11 +66,11 @@ func (i *Exporter) Export(event *output.ResultEvent) error { dataBuilder.WriteString(description) data := dataBuilder.Bytes() - return ioutil.WriteFile(filepath.Join(i.directory, finalFilename), data, 0644) + return ioutil.WriteFile(filepath.Join(exporter.directory, finalFilename), data, 0644) } // Close closes the exporter after operation -func (i *Exporter) Close() error { +func (exporter *Exporter) Close() error { return nil } diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 6395ee319..6288b0098 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -51,8 +51,8 @@ func New(options *Options) (*Exporter, error) { } // Export exports a passed result event to sarif structure -func (i *Exporter) Export(event *output.ResultEvent) error { - templatePath := strings.TrimPrefix(event.TemplatePath, i.home) +func (exporter *Exporter) Export(event *output.ResultEvent) error { + templatePath := strings.TrimPrefix(event.TemplatePath, exporter.home) h := sha1.New() _, _ = h.Write([]byte(event.Host)) @@ -67,7 +67,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { } var templateURL string - if strings.HasPrefix(event.TemplatePath, i.home) { + if strings.HasPrefix(event.TemplatePath, exporter.home) { templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath } else { templateURL = "https://github.com/projectdiscovery/nuclei-templates" @@ -78,19 +78,20 @@ func (i *Exporter) Export(event *output.ResultEvent) error { ruleDescription = event.Info.Description } - i.mutex.Lock() - defer i.mutex.Unlock() + exporter.mutex.Lock() + defer exporter.mutex.Unlock() - _ = i.run.AddRule(templateID). + _ = exporter.run.AddRule(templateID). WithDescription(ruleName). WithHelp(fullDescription). WithHelpURI(templateURL). WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)) - result := i.run.AddResult(templateID). + + result := exporter.run.AddResult(templateID). WithMessage(sarif.NewMessage().WithText(event.Host)). WithLevel(sarifSeverity) - // Also write file match metadata to file + // Also write file match metadata to file if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) { for file, line := range event.FileToIndexPosition { result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation( @@ -124,18 +125,18 @@ func getSarifSeverity(event *output.ResultEvent) string { } // Close closes the exporter after operation -func (i *Exporter) Close() error { - i.mutex.Lock() - defer i.mutex.Unlock() +func (exporter *Exporter) Close() error { + exporter.mutex.Lock() + defer exporter.mutex.Unlock() - i.sarif.AddRun(i.run) - if len(i.run.Results) == 0 { + exporter.sarif.AddRun(exporter.run) + if len(exporter.run.Results) == 0 { return nil // do not write when no results } - file, err := os.Create(i.options.File) + file, err := os.Create(exporter.options.File) if err != nil { return errors.Wrap(err, "could not create sarif output file") } defer file.Close() - return i.sarif.Write(file) + return exporter.sarif.Write(file) }