From 7f6c72853f2345a1f25094a9aeac60edca874e15 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:32:15 +0200 Subject: [PATCH 1/6] feat: introduce 'concat' DSL function #1502 --- integration_tests/http/dsl-functions.yaml | 97 ++++++++++--------- v2/pkg/operators/common/dsl/dsl.go | 10 ++ v2/pkg/operators/common/dsl/dsl_test.go | 2 + .../common/expressions/expressions_test.go | 9 ++ .../common/expressions/variables_test.go | 1 + 5 files changed, 71 insertions(+), 48 deletions(-) diff --git a/integration_tests/http/dsl-functions.yaml b/integration_tests/http/dsl-functions.yaml index 8b078e8fc..85963bf93 100644 --- a/integration_tests/http/dsl-functions.yaml +++ b/integration_tests/http/dsl-functions.yaml @@ -15,54 +15,55 @@ requests: 03: {{base64_decode("SGVsbG8=")}} 04: {{base64_py("Hello")}} 05: {{contains("Hello", "lo")}} - 06: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}} - 07: {{gzip("Hello")}} - 08: {{hex_decode("6161")}} - 09: {{hex_encode("aa")}} - 10: {{html_escape("test")}} - 11: {{html_unescape("<body>test</body>")}} - 12: {{len("Hello")}} - 13: {{len(5555)}} - 14: {{md5("Hello")}} - 15: {{md5(1234)}} - 16: {{mmh3("Hello")}} - 17: {{print_debug(1+2, "Hello")}} - 18: {{rand_base(5, "abc")}} - 19: {{rand_base(5, "")}} - 20: {{rand_base(5)}} - 21: {{rand_char("abc")}} - 22: {{rand_char("")}} - 23: {{rand_char()}} - 24: {{rand_int(1, 10)}} - 25: {{rand_int(10)}} - 26: {{rand_int()}} - 27: {{rand_text_alpha(10, "abc")}} - 28: {{rand_text_alpha(10, "")}} - 29: {{rand_text_alpha(10)}} - 30: {{rand_text_alphanumeric(10, "ab12")}} - 31: {{rand_text_alphanumeric(10)}} - 32: {{rand_text_numeric(10, 123)}} - 33: {{rand_text_numeric(10)}} - 34: {{regex("H([a-z]+)o", "Hello")}} - 35: {{remove_bad_chars("abcd", "bc")}} - 36: {{repeat("a", 5)}} - 37: {{replace("Hello", "He", "Ha")}} - 38: {{replace_regex("He123llo", "(\\d+)", "")}} - 39: {{reverse("abc")}} - 40: {{sha1("Hello")}} - 41: {{sha256("Hello")}} - 42: {{to_lower("HELLO")}} - 43: {{to_upper("hello")}} - 44: {{trim("aaaHelloddd", "ad")}} - 45: {{trim_left("aaaHelloddd", "ad")}} - 46: {{trim_prefix("aaHelloaa", "aa")}} - 47: {{trim_right("aaaHelloddd", "ad")}} - 48: {{trim_space(" Hello ")}} - 49: {{trim_suffix("aaHelloaa", "aa")}} - 50: {{unix_time(10)}} - 51: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}} - 52: {{url_encode("https://projectdiscovery.io/test?a=1")}} - 53: {{wait_for(1)}} + 06: {{contains("Hello", "world")}} + 07: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}} + 08: {{gzip("Hello")}} + 09: {{hex_decode("6161")}} + 10: {{hex_encode("aa")}} + 11: {{html_escape("test")}} + 12: {{html_unescape("<body>test</body>")}} + 13: {{len("Hello")}} + 14: {{len(5555)}} + 15: {{md5("Hello")}} + 16: {{md5(1234)}} + 17: {{mmh3("Hello")}} + 18: {{print_debug(1+2, "Hello")}} + 19: {{rand_base(5, "abc")}} + 20: {{rand_base(5, "")}} + 21: {{rand_base(5)}} + 22: {{rand_char("abc")}} + 23: {{rand_char("")}} + 24: {{rand_char()}} + 25: {{rand_int(1, 10)}} + 26: {{rand_int(10)}} + 27: {{rand_int()}} + 28: {{rand_text_alpha(10, "abc")}} + 29: {{rand_text_alpha(10, "")}} + 30: {{rand_text_alpha(10)}} + 31: {{rand_text_alphanumeric(10, "ab12")}} + 32: {{rand_text_alphanumeric(10)}} + 33: {{rand_text_numeric(10, 123)}} + 34: {{rand_text_numeric(10)}} + 35: {{regex("H([a-z]+)o", "Hello")}} + 36: {{remove_bad_chars("abcd", "bc")}} + 37: {{repeat("a", 5)}} + 38: {{replace("Hello", "He", "Ha")}} + 39: {{replace_regex("He123llo", "(\\d+)", "")}} + 40: {{reverse("abc")}} + 41: {{sha1("Hello")}} + 42: {{sha256("Hello")}} + 43: {{to_lower("HELLO")}} + 44: {{to_upper("hello")}} + 45: {{trim("aaaHelloddd", "ad")}} + 46: {{trim_left("aaaHelloddd", "ad")}} + 47: {{trim_prefix("aaHelloaa", "aa")}} + 48: {{trim_right("aaaHelloddd", "ad")}} + 49: {{trim_space(" Hello ")}} + 50: {{trim_suffix("aaHelloaa", "aa")}} + 51: {{unix_time(10)}} + 52: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}} + 53: {{url_encode("https://projectdiscovery.io/test?a=1")}} + 54: {{wait_for(1)}} extractors: - type: regex diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 03f5da994..343cd36ee 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -160,6 +160,16 @@ func init() { "contains": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil }), + "concat": makeDslWithOptionalArgsFunction( + "(args ...interface{}) string", + func(arguments ...interface{}) (interface{}, error) { + builder := &strings.Builder{} + for _, argument := range arguments { + builder.WriteString(types.ToString(argument)) + } + return builder.String(), nil + }, + ), "regex": makeDslFunction(2, func(args ...interface{}) (interface{}, error) { compiled, err := regexp.Compile(types.ToString(args[0])) if err != nil { diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 946a9b405..c49bffdaa 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -98,6 +98,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) { expected := ` base64(arg1 interface{}) interface{} base64_decode(arg1 interface{}) interface{} base64_py(arg1 interface{}) interface{} + concat(args ...interface{}) string contains(arg1, arg2 interface{}) interface{} generate_java_gadget(arg1, arg2, arg3 interface{}) interface{} gzip(arg1 interface{}) interface{} @@ -161,6 +162,7 @@ func TestDslExpressions(t *testing.T) { `mmh3("Hello")`: "316307400", `remove_bad_chars("abcd", "bc")`: "ad", `replace("Hello", "He", "Ha")`: "Hallo", + `concat("Hello", 123, "world")`: "Hello123world", `repeat("a", 5)`: "aaaaa", `repeat("a", "5")`: "aaaaa", `repeat("../", "5")`: "../../../../../", diff --git a/v2/pkg/protocols/common/expressions/expressions_test.go b/v2/pkg/protocols/common/expressions/expressions_test.go index f2d5ec1c9..7e4b154aa 100644 --- a/v2/pkg/protocols/common/expressions/expressions_test.go +++ b/v2/pkg/protocols/common/expressions/expressions_test.go @@ -14,6 +14,15 @@ func TestEvaluate(t *testing.T) { }{ {input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}}, {input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}}, + // TODO #1501 + //{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}}, + //{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, + //{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, + {input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}}, + {input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}}, + {input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}}, + {input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}}, + {input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}}, {input: "test", expected: "test", extra: map[string]interface{}{}}, {input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}}, {input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}}, diff --git a/v2/pkg/protocols/common/expressions/variables_test.go b/v2/pkg/protocols/common/expressions/variables_test.go index 61d071351..113e06909 100644 --- a/v2/pkg/protocols/common/expressions/variables_test.go +++ b/v2/pkg/protocols/common/expressions/variables_test.go @@ -13,6 +13,7 @@ func TestUnresolvedVariablesCheck(t *testing.T) { err error }{ {"{{test}}", errors.New("unresolved variables found: test")}, + {"{{1+1}}", nil}, {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")}, {"test", nil}, {"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")}, From ae07dce014b8b0908f5da5f1e7f32e490935ee8c Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:32:47 +0200 Subject: [PATCH 2/6] refactor: removed redundant escape characters from the `unresolvedVariablesRegex` variable --- v2/pkg/protocols/common/expressions/variables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index d064d577f..c16a80d24 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -6,7 +6,7 @@ import ( "strings" ) -var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|{){2}([^}]+)(?:%7[D|d]|}){2}["')}]*`) +var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`) // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. From bff19522680159b762c6da39b85b9c77a2adb31f Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:41:05 +0200 Subject: [PATCH 3/6] test: introduce 'concat' DSL function * removed irrelevant test case (covered elsewhere) --- v2/pkg/protocols/common/expressions/variables_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/protocols/common/expressions/variables_test.go b/v2/pkg/protocols/common/expressions/variables_test.go index 113e06909..61d071351 100644 --- a/v2/pkg/protocols/common/expressions/variables_test.go +++ b/v2/pkg/protocols/common/expressions/variables_test.go @@ -13,7 +13,6 @@ func TestUnresolvedVariablesCheck(t *testing.T) { err error }{ {"{{test}}", errors.New("unresolved variables found: test")}, - {"{{1+1}}", nil}, {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")}, {"test", nil}, {"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")}, From d4a015fb47c4ed8e399cffe1704033cedb258ec6 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:45:16 +0200 Subject: [PATCH 4/6] test: introduce 'concat' DSL function * fixed typo in test --- integration_tests/http/dsl-functions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/http/dsl-functions.yaml b/integration_tests/http/dsl-functions.yaml index 85963bf93..5d7c18479 100644 --- a/integration_tests/http/dsl-functions.yaml +++ b/integration_tests/http/dsl-functions.yaml @@ -15,7 +15,7 @@ requests: 03: {{base64_decode("SGVsbG8=")}} 04: {{base64_py("Hello")}} 05: {{contains("Hello", "lo")}} - 06: {{contains("Hello", "world")}} + 06: {{concat("Hello", "world")}} 07: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}} 08: {{gzip("Hello")}} 09: {{hex_decode("6161")}} From 286e27510dacb4043458367f50df7ee1c8eea1e9 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:50:42 +0200 Subject: [PATCH 5/6] test: introduce 'concat' DSL function * correct order and number of expected results --- integration_tests/http/dsl-functions.yaml | 4 ++-- v2/cmd/integration-test/http.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/http/dsl-functions.yaml b/integration_tests/http/dsl-functions.yaml index 5d7c18479..f52a3b808 100644 --- a/integration_tests/http/dsl-functions.yaml +++ b/integration_tests/http/dsl-functions.yaml @@ -14,8 +14,8 @@ requests: 02: {{base64(1234)}} 03: {{base64_decode("SGVsbG8=")}} 04: {{base64_py("Hello")}} - 05: {{contains("Hello", "lo")}} - 06: {{concat("Hello", "world")}} + 05: {{concat("Hello", "world")}} + 06: {{contains("Hello", "lo")}} 07: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}} 08: {{gzip("Hello")}} 09: {{hex_decode("6161")}} diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index a95cea558..e69c8f367 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -224,7 +224,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error { } totalExtracted := strings.Split(submatch[1], ",") - numberOfDslFunctions := 53 + numberOfDslFunctions := 54 if len(totalExtracted) != numberOfDslFunctions { return errors.New("incorrect number of results") } From 8019fb8f08084f88ea95a88354b929b528f55e51 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 17 Jan 2022 14:04:09 +0200 Subject: [PATCH 6/6] test: additional use-cases for the expression test --- v2/pkg/protocols/common/expressions/expressions_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/v2/pkg/protocols/common/expressions/expressions_test.go b/v2/pkg/protocols/common/expressions/expressions_test.go index 7e4b154aa..668b3b455 100644 --- a/v2/pkg/protocols/common/expressions/expressions_test.go +++ b/v2/pkg/protocols/common/expressions/expressions_test.go @@ -18,8 +18,12 @@ func TestEvaluate(t *testing.T) { //{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}}, //{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, //{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, + //{input: `{{"{{" + '123*123' + "}}"}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, {input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}}, {input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}}, + {input: `{{"1"+"1"}}`, expected: "11", extra: map[string]interface{}{}}, + {input: `{{"1" + '*' + "1"}}`, expected: "1*1", extra: map[string]interface{}{}}, + {input: `{{"a" + 'b' + "c"}}`, expected: "abc", extra: map[string]interface{}{}}, {input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}}, {input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}}, {input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}},