From ff3bb04655f30ee540f605e528ca998ac05d118c Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Fri, 18 Jul 2025 18:37:57 +0530 Subject: [PATCH] chore: support legacy cols usage and address several gaps (#8552) --- grammar/FilterQuery.g4 | 2 +- pkg/parser/grammar/FilterQueryLexer.interp | 2 +- pkg/parser/grammar/filterquery_lexer.go | 225 +++++------ pkg/querier/builder_query.go | 15 + pkg/querier/consume.go | 11 + pkg/querier/postprocess.go | 130 +++++++ pkg/querier/querier.go | 11 + pkg/querybuilder/fallback_expr.go | 16 +- .../resourcefilter/statement_builder.go | 2 + pkg/querybuilder/time.go | 74 +++- pkg/querybuilder/where_clause_visitor.go | 50 ++- pkg/telemetrylogs/condition_builder_test.go | 4 +- pkg/telemetrylogs/field_mapper.go | 7 +- pkg/telemetrylogs/filter_compiler.go | 55 --- pkg/telemetrylogs/filter_expr_logs_test.go | 166 ++++---- pkg/telemetrylogs/statement_builder.go | 77 +++- pkg/telemetrylogs/stmt_builder_test.go | 103 ++++- pkg/telemetrymetadata/field_mapper.go | 5 +- pkg/telemetrymetadata/metadata.go | 14 +- pkg/telemetrymetadata/metadata_test.go | 2 +- pkg/telemetrymetrics/condition_builder.go | 14 +- .../condition_builder_test.go | 8 +- pkg/telemetrymetrics/field_mapper.go | 3 +- pkg/telemetrymetrics/statement_builder.go | 21 +- pkg/telemetrymetrics/stmt_builder_test.go | 14 +- pkg/telemetrytraces/condition_builder.go | 21 +- pkg/telemetrytraces/condition_builder_test.go | 4 +- pkg/telemetrytraces/const.go | 64 +++- pkg/telemetrytraces/field_mapper.go | 50 ++- pkg/telemetrytraces/statement_builder.go | 122 +++++- pkg/telemetrytraces/stmt_builder_test.go | 359 +++++++++++++++++- pkg/telemetrytraces/test_data.go | 22 ++ .../querybuildertypesv5/functions.go | 6 +- .../querybuildertypesv5/functions_test.go | 19 +- pkg/types/telemetrytypes/field.go | 25 +- .../telemetrytypestest/metadata_store.go | 3 + 36 files changed, 1385 insertions(+), 341 deletions(-) delete mode 100644 pkg/telemetrylogs/filter_compiler.go diff --git a/grammar/FilterQuery.g4 b/grammar/FilterQuery.g4 index 9b84cbd0e20d..204c5fdd3275 100644 --- a/grammar/FilterQuery.g4 +++ b/grammar/FilterQuery.g4 @@ -208,7 +208,7 @@ QUOTED_TEXT ) ; -fragment SEGMENT : [a-zA-Z$] [a-zA-Z0-9$_:\-]* ; +fragment SEGMENT : [a-zA-Z$_] [a-zA-Z0-9$_:\-/]* ; fragment EMPTY_BRACKS : '[' ']' ; fragment OLD_JSON_BRACKS: '[' '*' ']'; diff --git a/pkg/parser/grammar/FilterQueryLexer.interp b/pkg/parser/grammar/FilterQueryLexer.interp index 7c22ebda90b0..a5c3dfe3fd5e 100644 --- a/pkg/parser/grammar/FilterQueryLexer.interp +++ b/pkg/parser/grammar/FilterQueryLexer.interp @@ -118,4 +118,4 @@ mode names: DEFAULT_MODE atn: -[4, 0, 33, 334, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 3, 5, 91, 8, 5, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 4, 13, 118, 8, 13, 11, 13, 12, 13, 119, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 4, 15, 137, 8, 15, 11, 15, 12, 15, 138, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 161, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 3, 19, 178, 8, 19, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 3, 27, 221, 8, 27, 1, 28, 1, 28, 1, 29, 3, 29, 226, 8, 29, 1, 29, 4, 29, 229, 8, 29, 11, 29, 12, 29, 230, 1, 29, 1, 29, 5, 29, 235, 8, 29, 10, 29, 12, 29, 238, 9, 29, 3, 29, 240, 8, 29, 1, 29, 1, 29, 3, 29, 244, 8, 29, 1, 29, 4, 29, 247, 8, 29, 11, 29, 12, 29, 248, 3, 29, 251, 8, 29, 1, 29, 3, 29, 254, 8, 29, 1, 29, 1, 29, 4, 29, 258, 8, 29, 11, 29, 12, 29, 259, 1, 29, 1, 29, 3, 29, 264, 8, 29, 1, 29, 4, 29, 267, 8, 29, 11, 29, 12, 29, 268, 3, 29, 271, 8, 29, 3, 29, 273, 8, 29, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 279, 8, 30, 10, 30, 12, 30, 282, 9, 30, 1, 30, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 289, 8, 30, 10, 30, 12, 30, 292, 9, 30, 1, 30, 3, 30, 295, 8, 30, 1, 31, 1, 31, 5, 31, 299, 8, 31, 10, 31, 12, 31, 302, 9, 31, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 316, 8, 34, 10, 34, 12, 34, 319, 9, 34, 1, 35, 4, 35, 322, 8, 35, 11, 35, 12, 35, 323, 1, 35, 1, 35, 1, 36, 1, 36, 1, 37, 4, 37, 331, 8, 37, 11, 37, 12, 37, 332, 0, 0, 38, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43, 22, 45, 23, 47, 24, 49, 25, 51, 26, 53, 27, 55, 28, 57, 0, 59, 29, 61, 30, 63, 0, 65, 0, 67, 0, 69, 31, 71, 32, 73, 0, 75, 33, 1, 0, 30, 2, 0, 76, 76, 108, 108, 2, 0, 73, 73, 105, 105, 2, 0, 75, 75, 107, 107, 2, 0, 69, 69, 101, 101, 2, 0, 78, 78, 110, 110, 2, 0, 79, 79, 111, 111, 2, 0, 84, 84, 116, 116, 2, 0, 9, 9, 32, 32, 2, 0, 66, 66, 98, 98, 2, 0, 87, 87, 119, 119, 2, 0, 88, 88, 120, 120, 2, 0, 83, 83, 115, 115, 2, 0, 82, 82, 114, 114, 2, 0, 71, 71, 103, 103, 2, 0, 80, 80, 112, 112, 2, 0, 67, 67, 99, 99, 2, 0, 65, 65, 97, 97, 2, 0, 68, 68, 100, 100, 2, 0, 72, 72, 104, 104, 2, 0, 89, 89, 121, 121, 2, 0, 85, 85, 117, 117, 2, 0, 70, 70, 102, 102, 2, 0, 43, 43, 45, 45, 2, 0, 34, 34, 92, 92, 2, 0, 39, 39, 92, 92, 3, 0, 36, 36, 65, 90, 97, 122, 6, 0, 36, 36, 45, 45, 48, 58, 65, 90, 95, 95, 97, 122, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 8, 0, 9, 10, 13, 13, 32, 34, 39, 41, 44, 44, 60, 62, 91, 91, 93, 93, 358, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 71, 1, 0, 0, 0, 0, 75, 1, 0, 0, 0, 1, 77, 1, 0, 0, 0, 3, 79, 1, 0, 0, 0, 5, 81, 1, 0, 0, 0, 7, 83, 1, 0, 0, 0, 9, 85, 1, 0, 0, 0, 11, 90, 1, 0, 0, 0, 13, 92, 1, 0, 0, 0, 15, 95, 1, 0, 0, 0, 17, 98, 1, 0, 0, 0, 19, 100, 1, 0, 0, 0, 21, 103, 1, 0, 0, 0, 23, 105, 1, 0, 0, 0, 25, 108, 1, 0, 0, 0, 27, 113, 1, 0, 0, 0, 29, 126, 1, 0, 0, 0, 31, 132, 1, 0, 0, 0, 33, 146, 1, 0, 0, 0, 35, 154, 1, 0, 0, 0, 37, 162, 1, 0, 0, 0, 39, 169, 1, 0, 0, 0, 41, 179, 1, 0, 0, 0, 43, 182, 1, 0, 0, 0, 45, 186, 1, 0, 0, 0, 47, 190, 1, 0, 0, 0, 49, 193, 1, 0, 0, 0, 51, 197, 1, 0, 0, 0, 53, 204, 1, 0, 0, 0, 55, 220, 1, 0, 0, 0, 57, 222, 1, 0, 0, 0, 59, 272, 1, 0, 0, 0, 61, 294, 1, 0, 0, 0, 63, 296, 1, 0, 0, 0, 65, 303, 1, 0, 0, 0, 67, 306, 1, 0, 0, 0, 69, 310, 1, 0, 0, 0, 71, 321, 1, 0, 0, 0, 73, 327, 1, 0, 0, 0, 75, 330, 1, 0, 0, 0, 77, 78, 5, 40, 0, 0, 78, 2, 1, 0, 0, 0, 79, 80, 5, 41, 0, 0, 80, 4, 1, 0, 0, 0, 81, 82, 5, 91, 0, 0, 82, 6, 1, 0, 0, 0, 83, 84, 5, 93, 0, 0, 84, 8, 1, 0, 0, 0, 85, 86, 5, 44, 0, 0, 86, 10, 1, 0, 0, 0, 87, 91, 5, 61, 0, 0, 88, 89, 5, 61, 0, 0, 89, 91, 5, 61, 0, 0, 90, 87, 1, 0, 0, 0, 90, 88, 1, 0, 0, 0, 91, 12, 1, 0, 0, 0, 92, 93, 5, 33, 0, 0, 93, 94, 5, 61, 0, 0, 94, 14, 1, 0, 0, 0, 95, 96, 5, 60, 0, 0, 96, 97, 5, 62, 0, 0, 97, 16, 1, 0, 0, 0, 98, 99, 5, 60, 0, 0, 99, 18, 1, 0, 0, 0, 100, 101, 5, 60, 0, 0, 101, 102, 5, 61, 0, 0, 102, 20, 1, 0, 0, 0, 103, 104, 5, 62, 0, 0, 104, 22, 1, 0, 0, 0, 105, 106, 5, 62, 0, 0, 106, 107, 5, 61, 0, 0, 107, 24, 1, 0, 0, 0, 108, 109, 7, 0, 0, 0, 109, 110, 7, 1, 0, 0, 110, 111, 7, 2, 0, 0, 111, 112, 7, 3, 0, 0, 112, 26, 1, 0, 0, 0, 113, 114, 7, 4, 0, 0, 114, 115, 7, 5, 0, 0, 115, 117, 7, 6, 0, 0, 116, 118, 7, 7, 0, 0, 117, 116, 1, 0, 0, 0, 118, 119, 1, 0, 0, 0, 119, 117, 1, 0, 0, 0, 119, 120, 1, 0, 0, 0, 120, 121, 1, 0, 0, 0, 121, 122, 7, 0, 0, 0, 122, 123, 7, 1, 0, 0, 123, 124, 7, 2, 0, 0, 124, 125, 7, 3, 0, 0, 125, 28, 1, 0, 0, 0, 126, 127, 7, 1, 0, 0, 127, 128, 7, 0, 0, 0, 128, 129, 7, 1, 0, 0, 129, 130, 7, 2, 0, 0, 130, 131, 7, 3, 0, 0, 131, 30, 1, 0, 0, 0, 132, 133, 7, 4, 0, 0, 133, 134, 7, 5, 0, 0, 134, 136, 7, 6, 0, 0, 135, 137, 7, 7, 0, 0, 136, 135, 1, 0, 0, 0, 137, 138, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, 138, 139, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 141, 7, 1, 0, 0, 141, 142, 7, 0, 0, 0, 142, 143, 7, 1, 0, 0, 143, 144, 7, 2, 0, 0, 144, 145, 7, 3, 0, 0, 145, 32, 1, 0, 0, 0, 146, 147, 7, 8, 0, 0, 147, 148, 7, 3, 0, 0, 148, 149, 7, 6, 0, 0, 149, 150, 7, 9, 0, 0, 150, 151, 7, 3, 0, 0, 151, 152, 7, 3, 0, 0, 152, 153, 7, 4, 0, 0, 153, 34, 1, 0, 0, 0, 154, 155, 7, 3, 0, 0, 155, 156, 7, 10, 0, 0, 156, 157, 7, 1, 0, 0, 157, 158, 7, 11, 0, 0, 158, 160, 7, 6, 0, 0, 159, 161, 7, 11, 0, 0, 160, 159, 1, 0, 0, 0, 160, 161, 1, 0, 0, 0, 161, 36, 1, 0, 0, 0, 162, 163, 7, 12, 0, 0, 163, 164, 7, 3, 0, 0, 164, 165, 7, 13, 0, 0, 165, 166, 7, 3, 0, 0, 166, 167, 7, 10, 0, 0, 167, 168, 7, 14, 0, 0, 168, 38, 1, 0, 0, 0, 169, 170, 7, 15, 0, 0, 170, 171, 7, 5, 0, 0, 171, 172, 7, 4, 0, 0, 172, 173, 7, 6, 0, 0, 173, 174, 7, 16, 0, 0, 174, 175, 7, 1, 0, 0, 175, 177, 7, 4, 0, 0, 176, 178, 7, 11, 0, 0, 177, 176, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 40, 1, 0, 0, 0, 179, 180, 7, 1, 0, 0, 180, 181, 7, 4, 0, 0, 181, 42, 1, 0, 0, 0, 182, 183, 7, 4, 0, 0, 183, 184, 7, 5, 0, 0, 184, 185, 7, 6, 0, 0, 185, 44, 1, 0, 0, 0, 186, 187, 7, 16, 0, 0, 187, 188, 7, 4, 0, 0, 188, 189, 7, 17, 0, 0, 189, 46, 1, 0, 0, 0, 190, 191, 7, 5, 0, 0, 191, 192, 7, 12, 0, 0, 192, 48, 1, 0, 0, 0, 193, 194, 7, 18, 0, 0, 194, 195, 7, 16, 0, 0, 195, 196, 7, 11, 0, 0, 196, 50, 1, 0, 0, 0, 197, 198, 7, 18, 0, 0, 198, 199, 7, 16, 0, 0, 199, 200, 7, 11, 0, 0, 200, 201, 7, 16, 0, 0, 201, 202, 7, 4, 0, 0, 202, 203, 7, 19, 0, 0, 203, 52, 1, 0, 0, 0, 204, 205, 7, 18, 0, 0, 205, 206, 7, 16, 0, 0, 206, 207, 7, 11, 0, 0, 207, 208, 7, 16, 0, 0, 208, 209, 7, 0, 0, 0, 209, 210, 7, 0, 0, 0, 210, 54, 1, 0, 0, 0, 211, 212, 7, 6, 0, 0, 212, 213, 7, 12, 0, 0, 213, 214, 7, 20, 0, 0, 214, 221, 7, 3, 0, 0, 215, 216, 7, 21, 0, 0, 216, 217, 7, 16, 0, 0, 217, 218, 7, 0, 0, 0, 218, 219, 7, 11, 0, 0, 219, 221, 7, 3, 0, 0, 220, 211, 1, 0, 0, 0, 220, 215, 1, 0, 0, 0, 221, 56, 1, 0, 0, 0, 222, 223, 7, 22, 0, 0, 223, 58, 1, 0, 0, 0, 224, 226, 3, 57, 28, 0, 225, 224, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 228, 1, 0, 0, 0, 227, 229, 3, 73, 36, 0, 228, 227, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 228, 1, 0, 0, 0, 230, 231, 1, 0, 0, 0, 231, 239, 1, 0, 0, 0, 232, 236, 5, 46, 0, 0, 233, 235, 3, 73, 36, 0, 234, 233, 1, 0, 0, 0, 235, 238, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, 236, 237, 1, 0, 0, 0, 237, 240, 1, 0, 0, 0, 238, 236, 1, 0, 0, 0, 239, 232, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 250, 1, 0, 0, 0, 241, 243, 7, 3, 0, 0, 242, 244, 3, 57, 28, 0, 243, 242, 1, 0, 0, 0, 243, 244, 1, 0, 0, 0, 244, 246, 1, 0, 0, 0, 245, 247, 3, 73, 36, 0, 246, 245, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 246, 1, 0, 0, 0, 248, 249, 1, 0, 0, 0, 249, 251, 1, 0, 0, 0, 250, 241, 1, 0, 0, 0, 250, 251, 1, 0, 0, 0, 251, 273, 1, 0, 0, 0, 252, 254, 3, 57, 28, 0, 253, 252, 1, 0, 0, 0, 253, 254, 1, 0, 0, 0, 254, 255, 1, 0, 0, 0, 255, 257, 5, 46, 0, 0, 256, 258, 3, 73, 36, 0, 257, 256, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 257, 1, 0, 0, 0, 259, 260, 1, 0, 0, 0, 260, 270, 1, 0, 0, 0, 261, 263, 7, 3, 0, 0, 262, 264, 3, 57, 28, 0, 263, 262, 1, 0, 0, 0, 263, 264, 1, 0, 0, 0, 264, 266, 1, 0, 0, 0, 265, 267, 3, 73, 36, 0, 266, 265, 1, 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 266, 1, 0, 0, 0, 268, 269, 1, 0, 0, 0, 269, 271, 1, 0, 0, 0, 270, 261, 1, 0, 0, 0, 270, 271, 1, 0, 0, 0, 271, 273, 1, 0, 0, 0, 272, 225, 1, 0, 0, 0, 272, 253, 1, 0, 0, 0, 273, 60, 1, 0, 0, 0, 274, 280, 5, 34, 0, 0, 275, 279, 8, 23, 0, 0, 276, 277, 5, 92, 0, 0, 277, 279, 9, 0, 0, 0, 278, 275, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 283, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 295, 5, 34, 0, 0, 284, 290, 5, 39, 0, 0, 285, 289, 8, 24, 0, 0, 286, 287, 5, 92, 0, 0, 287, 289, 9, 0, 0, 0, 288, 285, 1, 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 292, 1, 0, 0, 0, 290, 288, 1, 0, 0, 0, 290, 291, 1, 0, 0, 0, 291, 293, 1, 0, 0, 0, 292, 290, 1, 0, 0, 0, 293, 295, 5, 39, 0, 0, 294, 274, 1, 0, 0, 0, 294, 284, 1, 0, 0, 0, 295, 62, 1, 0, 0, 0, 296, 300, 7, 25, 0, 0, 297, 299, 7, 26, 0, 0, 298, 297, 1, 0, 0, 0, 299, 302, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, 300, 301, 1, 0, 0, 0, 301, 64, 1, 0, 0, 0, 302, 300, 1, 0, 0, 0, 303, 304, 5, 91, 0, 0, 304, 305, 5, 93, 0, 0, 305, 66, 1, 0, 0, 0, 306, 307, 5, 91, 0, 0, 307, 308, 5, 42, 0, 0, 308, 309, 5, 93, 0, 0, 309, 68, 1, 0, 0, 0, 310, 317, 3, 63, 31, 0, 311, 312, 5, 46, 0, 0, 312, 316, 3, 63, 31, 0, 313, 316, 3, 65, 32, 0, 314, 316, 3, 67, 33, 0, 315, 311, 1, 0, 0, 0, 315, 313, 1, 0, 0, 0, 315, 314, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 70, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 7, 27, 0, 0, 321, 320, 1, 0, 0, 0, 322, 323, 1, 0, 0, 0, 323, 321, 1, 0, 0, 0, 323, 324, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 326, 6, 35, 0, 0, 326, 72, 1, 0, 0, 0, 327, 328, 7, 28, 0, 0, 328, 74, 1, 0, 0, 0, 329, 331, 8, 29, 0, 0, 330, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 330, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 76, 1, 0, 0, 0, 30, 0, 90, 119, 138, 160, 177, 220, 225, 230, 236, 239, 243, 248, 250, 253, 259, 263, 268, 270, 272, 278, 280, 288, 290, 294, 300, 315, 317, 323, 332, 1, 6, 0, 0] \ No newline at end of file +[4, 0, 33, 334, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 3, 5, 91, 8, 5, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 4, 13, 118, 8, 13, 11, 13, 12, 13, 119, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 4, 15, 137, 8, 15, 11, 15, 12, 15, 138, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 161, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 3, 19, 178, 8, 19, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 3, 27, 221, 8, 27, 1, 28, 1, 28, 1, 29, 3, 29, 226, 8, 29, 1, 29, 4, 29, 229, 8, 29, 11, 29, 12, 29, 230, 1, 29, 1, 29, 5, 29, 235, 8, 29, 10, 29, 12, 29, 238, 9, 29, 3, 29, 240, 8, 29, 1, 29, 1, 29, 3, 29, 244, 8, 29, 1, 29, 4, 29, 247, 8, 29, 11, 29, 12, 29, 248, 3, 29, 251, 8, 29, 1, 29, 3, 29, 254, 8, 29, 1, 29, 1, 29, 4, 29, 258, 8, 29, 11, 29, 12, 29, 259, 1, 29, 1, 29, 3, 29, 264, 8, 29, 1, 29, 4, 29, 267, 8, 29, 11, 29, 12, 29, 268, 3, 29, 271, 8, 29, 3, 29, 273, 8, 29, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 279, 8, 30, 10, 30, 12, 30, 282, 9, 30, 1, 30, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 289, 8, 30, 10, 30, 12, 30, 292, 9, 30, 1, 30, 3, 30, 295, 8, 30, 1, 31, 1, 31, 5, 31, 299, 8, 31, 10, 31, 12, 31, 302, 9, 31, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 316, 8, 34, 10, 34, 12, 34, 319, 9, 34, 1, 35, 4, 35, 322, 8, 35, 11, 35, 12, 35, 323, 1, 35, 1, 35, 1, 36, 1, 36, 1, 37, 4, 37, 331, 8, 37, 11, 37, 12, 37, 332, 0, 0, 38, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43, 22, 45, 23, 47, 24, 49, 25, 51, 26, 53, 27, 55, 28, 57, 0, 59, 29, 61, 30, 63, 0, 65, 0, 67, 0, 69, 31, 71, 32, 73, 0, 75, 33, 1, 0, 30, 2, 0, 76, 76, 108, 108, 2, 0, 73, 73, 105, 105, 2, 0, 75, 75, 107, 107, 2, 0, 69, 69, 101, 101, 2, 0, 78, 78, 110, 110, 2, 0, 79, 79, 111, 111, 2, 0, 84, 84, 116, 116, 2, 0, 9, 9, 32, 32, 2, 0, 66, 66, 98, 98, 2, 0, 87, 87, 119, 119, 2, 0, 88, 88, 120, 120, 2, 0, 83, 83, 115, 115, 2, 0, 82, 82, 114, 114, 2, 0, 71, 71, 103, 103, 2, 0, 80, 80, 112, 112, 2, 0, 67, 67, 99, 99, 2, 0, 65, 65, 97, 97, 2, 0, 68, 68, 100, 100, 2, 0, 72, 72, 104, 104, 2, 0, 89, 89, 121, 121, 2, 0, 85, 85, 117, 117, 2, 0, 70, 70, 102, 102, 2, 0, 43, 43, 45, 45, 2, 0, 34, 34, 92, 92, 2, 0, 39, 39, 92, 92, 4, 0, 36, 36, 65, 90, 95, 95, 97, 122, 6, 0, 36, 36, 45, 45, 47, 58, 65, 90, 95, 95, 97, 122, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 8, 0, 9, 10, 13, 13, 32, 34, 39, 41, 44, 44, 60, 62, 91, 91, 93, 93, 358, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 71, 1, 0, 0, 0, 0, 75, 1, 0, 0, 0, 1, 77, 1, 0, 0, 0, 3, 79, 1, 0, 0, 0, 5, 81, 1, 0, 0, 0, 7, 83, 1, 0, 0, 0, 9, 85, 1, 0, 0, 0, 11, 90, 1, 0, 0, 0, 13, 92, 1, 0, 0, 0, 15, 95, 1, 0, 0, 0, 17, 98, 1, 0, 0, 0, 19, 100, 1, 0, 0, 0, 21, 103, 1, 0, 0, 0, 23, 105, 1, 0, 0, 0, 25, 108, 1, 0, 0, 0, 27, 113, 1, 0, 0, 0, 29, 126, 1, 0, 0, 0, 31, 132, 1, 0, 0, 0, 33, 146, 1, 0, 0, 0, 35, 154, 1, 0, 0, 0, 37, 162, 1, 0, 0, 0, 39, 169, 1, 0, 0, 0, 41, 179, 1, 0, 0, 0, 43, 182, 1, 0, 0, 0, 45, 186, 1, 0, 0, 0, 47, 190, 1, 0, 0, 0, 49, 193, 1, 0, 0, 0, 51, 197, 1, 0, 0, 0, 53, 204, 1, 0, 0, 0, 55, 220, 1, 0, 0, 0, 57, 222, 1, 0, 0, 0, 59, 272, 1, 0, 0, 0, 61, 294, 1, 0, 0, 0, 63, 296, 1, 0, 0, 0, 65, 303, 1, 0, 0, 0, 67, 306, 1, 0, 0, 0, 69, 310, 1, 0, 0, 0, 71, 321, 1, 0, 0, 0, 73, 327, 1, 0, 0, 0, 75, 330, 1, 0, 0, 0, 77, 78, 5, 40, 0, 0, 78, 2, 1, 0, 0, 0, 79, 80, 5, 41, 0, 0, 80, 4, 1, 0, 0, 0, 81, 82, 5, 91, 0, 0, 82, 6, 1, 0, 0, 0, 83, 84, 5, 93, 0, 0, 84, 8, 1, 0, 0, 0, 85, 86, 5, 44, 0, 0, 86, 10, 1, 0, 0, 0, 87, 91, 5, 61, 0, 0, 88, 89, 5, 61, 0, 0, 89, 91, 5, 61, 0, 0, 90, 87, 1, 0, 0, 0, 90, 88, 1, 0, 0, 0, 91, 12, 1, 0, 0, 0, 92, 93, 5, 33, 0, 0, 93, 94, 5, 61, 0, 0, 94, 14, 1, 0, 0, 0, 95, 96, 5, 60, 0, 0, 96, 97, 5, 62, 0, 0, 97, 16, 1, 0, 0, 0, 98, 99, 5, 60, 0, 0, 99, 18, 1, 0, 0, 0, 100, 101, 5, 60, 0, 0, 101, 102, 5, 61, 0, 0, 102, 20, 1, 0, 0, 0, 103, 104, 5, 62, 0, 0, 104, 22, 1, 0, 0, 0, 105, 106, 5, 62, 0, 0, 106, 107, 5, 61, 0, 0, 107, 24, 1, 0, 0, 0, 108, 109, 7, 0, 0, 0, 109, 110, 7, 1, 0, 0, 110, 111, 7, 2, 0, 0, 111, 112, 7, 3, 0, 0, 112, 26, 1, 0, 0, 0, 113, 114, 7, 4, 0, 0, 114, 115, 7, 5, 0, 0, 115, 117, 7, 6, 0, 0, 116, 118, 7, 7, 0, 0, 117, 116, 1, 0, 0, 0, 118, 119, 1, 0, 0, 0, 119, 117, 1, 0, 0, 0, 119, 120, 1, 0, 0, 0, 120, 121, 1, 0, 0, 0, 121, 122, 7, 0, 0, 0, 122, 123, 7, 1, 0, 0, 123, 124, 7, 2, 0, 0, 124, 125, 7, 3, 0, 0, 125, 28, 1, 0, 0, 0, 126, 127, 7, 1, 0, 0, 127, 128, 7, 0, 0, 0, 128, 129, 7, 1, 0, 0, 129, 130, 7, 2, 0, 0, 130, 131, 7, 3, 0, 0, 131, 30, 1, 0, 0, 0, 132, 133, 7, 4, 0, 0, 133, 134, 7, 5, 0, 0, 134, 136, 7, 6, 0, 0, 135, 137, 7, 7, 0, 0, 136, 135, 1, 0, 0, 0, 137, 138, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, 138, 139, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 141, 7, 1, 0, 0, 141, 142, 7, 0, 0, 0, 142, 143, 7, 1, 0, 0, 143, 144, 7, 2, 0, 0, 144, 145, 7, 3, 0, 0, 145, 32, 1, 0, 0, 0, 146, 147, 7, 8, 0, 0, 147, 148, 7, 3, 0, 0, 148, 149, 7, 6, 0, 0, 149, 150, 7, 9, 0, 0, 150, 151, 7, 3, 0, 0, 151, 152, 7, 3, 0, 0, 152, 153, 7, 4, 0, 0, 153, 34, 1, 0, 0, 0, 154, 155, 7, 3, 0, 0, 155, 156, 7, 10, 0, 0, 156, 157, 7, 1, 0, 0, 157, 158, 7, 11, 0, 0, 158, 160, 7, 6, 0, 0, 159, 161, 7, 11, 0, 0, 160, 159, 1, 0, 0, 0, 160, 161, 1, 0, 0, 0, 161, 36, 1, 0, 0, 0, 162, 163, 7, 12, 0, 0, 163, 164, 7, 3, 0, 0, 164, 165, 7, 13, 0, 0, 165, 166, 7, 3, 0, 0, 166, 167, 7, 10, 0, 0, 167, 168, 7, 14, 0, 0, 168, 38, 1, 0, 0, 0, 169, 170, 7, 15, 0, 0, 170, 171, 7, 5, 0, 0, 171, 172, 7, 4, 0, 0, 172, 173, 7, 6, 0, 0, 173, 174, 7, 16, 0, 0, 174, 175, 7, 1, 0, 0, 175, 177, 7, 4, 0, 0, 176, 178, 7, 11, 0, 0, 177, 176, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 40, 1, 0, 0, 0, 179, 180, 7, 1, 0, 0, 180, 181, 7, 4, 0, 0, 181, 42, 1, 0, 0, 0, 182, 183, 7, 4, 0, 0, 183, 184, 7, 5, 0, 0, 184, 185, 7, 6, 0, 0, 185, 44, 1, 0, 0, 0, 186, 187, 7, 16, 0, 0, 187, 188, 7, 4, 0, 0, 188, 189, 7, 17, 0, 0, 189, 46, 1, 0, 0, 0, 190, 191, 7, 5, 0, 0, 191, 192, 7, 12, 0, 0, 192, 48, 1, 0, 0, 0, 193, 194, 7, 18, 0, 0, 194, 195, 7, 16, 0, 0, 195, 196, 7, 11, 0, 0, 196, 50, 1, 0, 0, 0, 197, 198, 7, 18, 0, 0, 198, 199, 7, 16, 0, 0, 199, 200, 7, 11, 0, 0, 200, 201, 7, 16, 0, 0, 201, 202, 7, 4, 0, 0, 202, 203, 7, 19, 0, 0, 203, 52, 1, 0, 0, 0, 204, 205, 7, 18, 0, 0, 205, 206, 7, 16, 0, 0, 206, 207, 7, 11, 0, 0, 207, 208, 7, 16, 0, 0, 208, 209, 7, 0, 0, 0, 209, 210, 7, 0, 0, 0, 210, 54, 1, 0, 0, 0, 211, 212, 7, 6, 0, 0, 212, 213, 7, 12, 0, 0, 213, 214, 7, 20, 0, 0, 214, 221, 7, 3, 0, 0, 215, 216, 7, 21, 0, 0, 216, 217, 7, 16, 0, 0, 217, 218, 7, 0, 0, 0, 218, 219, 7, 11, 0, 0, 219, 221, 7, 3, 0, 0, 220, 211, 1, 0, 0, 0, 220, 215, 1, 0, 0, 0, 221, 56, 1, 0, 0, 0, 222, 223, 7, 22, 0, 0, 223, 58, 1, 0, 0, 0, 224, 226, 3, 57, 28, 0, 225, 224, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 228, 1, 0, 0, 0, 227, 229, 3, 73, 36, 0, 228, 227, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 228, 1, 0, 0, 0, 230, 231, 1, 0, 0, 0, 231, 239, 1, 0, 0, 0, 232, 236, 5, 46, 0, 0, 233, 235, 3, 73, 36, 0, 234, 233, 1, 0, 0, 0, 235, 238, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, 236, 237, 1, 0, 0, 0, 237, 240, 1, 0, 0, 0, 238, 236, 1, 0, 0, 0, 239, 232, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 250, 1, 0, 0, 0, 241, 243, 7, 3, 0, 0, 242, 244, 3, 57, 28, 0, 243, 242, 1, 0, 0, 0, 243, 244, 1, 0, 0, 0, 244, 246, 1, 0, 0, 0, 245, 247, 3, 73, 36, 0, 246, 245, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 246, 1, 0, 0, 0, 248, 249, 1, 0, 0, 0, 249, 251, 1, 0, 0, 0, 250, 241, 1, 0, 0, 0, 250, 251, 1, 0, 0, 0, 251, 273, 1, 0, 0, 0, 252, 254, 3, 57, 28, 0, 253, 252, 1, 0, 0, 0, 253, 254, 1, 0, 0, 0, 254, 255, 1, 0, 0, 0, 255, 257, 5, 46, 0, 0, 256, 258, 3, 73, 36, 0, 257, 256, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 257, 1, 0, 0, 0, 259, 260, 1, 0, 0, 0, 260, 270, 1, 0, 0, 0, 261, 263, 7, 3, 0, 0, 262, 264, 3, 57, 28, 0, 263, 262, 1, 0, 0, 0, 263, 264, 1, 0, 0, 0, 264, 266, 1, 0, 0, 0, 265, 267, 3, 73, 36, 0, 266, 265, 1, 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 266, 1, 0, 0, 0, 268, 269, 1, 0, 0, 0, 269, 271, 1, 0, 0, 0, 270, 261, 1, 0, 0, 0, 270, 271, 1, 0, 0, 0, 271, 273, 1, 0, 0, 0, 272, 225, 1, 0, 0, 0, 272, 253, 1, 0, 0, 0, 273, 60, 1, 0, 0, 0, 274, 280, 5, 34, 0, 0, 275, 279, 8, 23, 0, 0, 276, 277, 5, 92, 0, 0, 277, 279, 9, 0, 0, 0, 278, 275, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 283, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 295, 5, 34, 0, 0, 284, 290, 5, 39, 0, 0, 285, 289, 8, 24, 0, 0, 286, 287, 5, 92, 0, 0, 287, 289, 9, 0, 0, 0, 288, 285, 1, 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 292, 1, 0, 0, 0, 290, 288, 1, 0, 0, 0, 290, 291, 1, 0, 0, 0, 291, 293, 1, 0, 0, 0, 292, 290, 1, 0, 0, 0, 293, 295, 5, 39, 0, 0, 294, 274, 1, 0, 0, 0, 294, 284, 1, 0, 0, 0, 295, 62, 1, 0, 0, 0, 296, 300, 7, 25, 0, 0, 297, 299, 7, 26, 0, 0, 298, 297, 1, 0, 0, 0, 299, 302, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, 300, 301, 1, 0, 0, 0, 301, 64, 1, 0, 0, 0, 302, 300, 1, 0, 0, 0, 303, 304, 5, 91, 0, 0, 304, 305, 5, 93, 0, 0, 305, 66, 1, 0, 0, 0, 306, 307, 5, 91, 0, 0, 307, 308, 5, 42, 0, 0, 308, 309, 5, 93, 0, 0, 309, 68, 1, 0, 0, 0, 310, 317, 3, 63, 31, 0, 311, 312, 5, 46, 0, 0, 312, 316, 3, 63, 31, 0, 313, 316, 3, 65, 32, 0, 314, 316, 3, 67, 33, 0, 315, 311, 1, 0, 0, 0, 315, 313, 1, 0, 0, 0, 315, 314, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 70, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 7, 27, 0, 0, 321, 320, 1, 0, 0, 0, 322, 323, 1, 0, 0, 0, 323, 321, 1, 0, 0, 0, 323, 324, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 326, 6, 35, 0, 0, 326, 72, 1, 0, 0, 0, 327, 328, 7, 28, 0, 0, 328, 74, 1, 0, 0, 0, 329, 331, 8, 29, 0, 0, 330, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 330, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 76, 1, 0, 0, 0, 30, 0, 90, 119, 138, 160, 177, 220, 225, 230, 236, 239, 243, 248, 250, 253, 259, 263, 268, 270, 272, 278, 280, 288, 290, 294, 300, 315, 317, 323, 332, 1, 6, 0, 0] \ No newline at end of file diff --git a/pkg/parser/grammar/filterquery_lexer.go b/pkg/parser/grammar/filterquery_lexer.go index c43f13c81c4f..79ae7c5a7f61 100644 --- a/pkg/parser/grammar/filterquery_lexer.go +++ b/pkg/parser/grammar/filterquery_lexer.go @@ -110,118 +110,119 @@ func filterquerylexerLexerInit() { 67, 99, 99, 2, 0, 65, 65, 97, 97, 2, 0, 68, 68, 100, 100, 2, 0, 72, 72, 104, 104, 2, 0, 89, 89, 121, 121, 2, 0, 85, 85, 117, 117, 2, 0, 70, 70, 102, 102, 2, 0, 43, 43, 45, 45, 2, 0, 34, 34, 92, 92, 2, 0, 39, 39, 92, - 92, 3, 0, 36, 36, 65, 90, 97, 122, 6, 0, 36, 36, 45, 45, 48, 58, 65, 90, - 95, 95, 97, 122, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 8, 0, 9, 10, - 13, 13, 32, 34, 39, 41, 44, 44, 60, 62, 91, 91, 93, 93, 358, 0, 1, 1, 0, - 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, - 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, - 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, - 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, - 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, - 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, - 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, - 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 71, 1, - 0, 0, 0, 0, 75, 1, 0, 0, 0, 1, 77, 1, 0, 0, 0, 3, 79, 1, 0, 0, 0, 5, 81, - 1, 0, 0, 0, 7, 83, 1, 0, 0, 0, 9, 85, 1, 0, 0, 0, 11, 90, 1, 0, 0, 0, 13, - 92, 1, 0, 0, 0, 15, 95, 1, 0, 0, 0, 17, 98, 1, 0, 0, 0, 19, 100, 1, 0, - 0, 0, 21, 103, 1, 0, 0, 0, 23, 105, 1, 0, 0, 0, 25, 108, 1, 0, 0, 0, 27, - 113, 1, 0, 0, 0, 29, 126, 1, 0, 0, 0, 31, 132, 1, 0, 0, 0, 33, 146, 1, - 0, 0, 0, 35, 154, 1, 0, 0, 0, 37, 162, 1, 0, 0, 0, 39, 169, 1, 0, 0, 0, - 41, 179, 1, 0, 0, 0, 43, 182, 1, 0, 0, 0, 45, 186, 1, 0, 0, 0, 47, 190, - 1, 0, 0, 0, 49, 193, 1, 0, 0, 0, 51, 197, 1, 0, 0, 0, 53, 204, 1, 0, 0, - 0, 55, 220, 1, 0, 0, 0, 57, 222, 1, 0, 0, 0, 59, 272, 1, 0, 0, 0, 61, 294, - 1, 0, 0, 0, 63, 296, 1, 0, 0, 0, 65, 303, 1, 0, 0, 0, 67, 306, 1, 0, 0, - 0, 69, 310, 1, 0, 0, 0, 71, 321, 1, 0, 0, 0, 73, 327, 1, 0, 0, 0, 75, 330, - 1, 0, 0, 0, 77, 78, 5, 40, 0, 0, 78, 2, 1, 0, 0, 0, 79, 80, 5, 41, 0, 0, - 80, 4, 1, 0, 0, 0, 81, 82, 5, 91, 0, 0, 82, 6, 1, 0, 0, 0, 83, 84, 5, 93, - 0, 0, 84, 8, 1, 0, 0, 0, 85, 86, 5, 44, 0, 0, 86, 10, 1, 0, 0, 0, 87, 91, - 5, 61, 0, 0, 88, 89, 5, 61, 0, 0, 89, 91, 5, 61, 0, 0, 90, 87, 1, 0, 0, - 0, 90, 88, 1, 0, 0, 0, 91, 12, 1, 0, 0, 0, 92, 93, 5, 33, 0, 0, 93, 94, - 5, 61, 0, 0, 94, 14, 1, 0, 0, 0, 95, 96, 5, 60, 0, 0, 96, 97, 5, 62, 0, - 0, 97, 16, 1, 0, 0, 0, 98, 99, 5, 60, 0, 0, 99, 18, 1, 0, 0, 0, 100, 101, - 5, 60, 0, 0, 101, 102, 5, 61, 0, 0, 102, 20, 1, 0, 0, 0, 103, 104, 5, 62, - 0, 0, 104, 22, 1, 0, 0, 0, 105, 106, 5, 62, 0, 0, 106, 107, 5, 61, 0, 0, - 107, 24, 1, 0, 0, 0, 108, 109, 7, 0, 0, 0, 109, 110, 7, 1, 0, 0, 110, 111, - 7, 2, 0, 0, 111, 112, 7, 3, 0, 0, 112, 26, 1, 0, 0, 0, 113, 114, 7, 4, - 0, 0, 114, 115, 7, 5, 0, 0, 115, 117, 7, 6, 0, 0, 116, 118, 7, 7, 0, 0, - 117, 116, 1, 0, 0, 0, 118, 119, 1, 0, 0, 0, 119, 117, 1, 0, 0, 0, 119, - 120, 1, 0, 0, 0, 120, 121, 1, 0, 0, 0, 121, 122, 7, 0, 0, 0, 122, 123, - 7, 1, 0, 0, 123, 124, 7, 2, 0, 0, 124, 125, 7, 3, 0, 0, 125, 28, 1, 0, - 0, 0, 126, 127, 7, 1, 0, 0, 127, 128, 7, 0, 0, 0, 128, 129, 7, 1, 0, 0, - 129, 130, 7, 2, 0, 0, 130, 131, 7, 3, 0, 0, 131, 30, 1, 0, 0, 0, 132, 133, - 7, 4, 0, 0, 133, 134, 7, 5, 0, 0, 134, 136, 7, 6, 0, 0, 135, 137, 7, 7, - 0, 0, 136, 135, 1, 0, 0, 0, 137, 138, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, - 138, 139, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 141, 7, 1, 0, 0, 141, - 142, 7, 0, 0, 0, 142, 143, 7, 1, 0, 0, 143, 144, 7, 2, 0, 0, 144, 145, - 7, 3, 0, 0, 145, 32, 1, 0, 0, 0, 146, 147, 7, 8, 0, 0, 147, 148, 7, 3, - 0, 0, 148, 149, 7, 6, 0, 0, 149, 150, 7, 9, 0, 0, 150, 151, 7, 3, 0, 0, - 151, 152, 7, 3, 0, 0, 152, 153, 7, 4, 0, 0, 153, 34, 1, 0, 0, 0, 154, 155, - 7, 3, 0, 0, 155, 156, 7, 10, 0, 0, 156, 157, 7, 1, 0, 0, 157, 158, 7, 11, - 0, 0, 158, 160, 7, 6, 0, 0, 159, 161, 7, 11, 0, 0, 160, 159, 1, 0, 0, 0, - 160, 161, 1, 0, 0, 0, 161, 36, 1, 0, 0, 0, 162, 163, 7, 12, 0, 0, 163, - 164, 7, 3, 0, 0, 164, 165, 7, 13, 0, 0, 165, 166, 7, 3, 0, 0, 166, 167, - 7, 10, 0, 0, 167, 168, 7, 14, 0, 0, 168, 38, 1, 0, 0, 0, 169, 170, 7, 15, - 0, 0, 170, 171, 7, 5, 0, 0, 171, 172, 7, 4, 0, 0, 172, 173, 7, 6, 0, 0, - 173, 174, 7, 16, 0, 0, 174, 175, 7, 1, 0, 0, 175, 177, 7, 4, 0, 0, 176, - 178, 7, 11, 0, 0, 177, 176, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 40, - 1, 0, 0, 0, 179, 180, 7, 1, 0, 0, 180, 181, 7, 4, 0, 0, 181, 42, 1, 0, - 0, 0, 182, 183, 7, 4, 0, 0, 183, 184, 7, 5, 0, 0, 184, 185, 7, 6, 0, 0, - 185, 44, 1, 0, 0, 0, 186, 187, 7, 16, 0, 0, 187, 188, 7, 4, 0, 0, 188, - 189, 7, 17, 0, 0, 189, 46, 1, 0, 0, 0, 190, 191, 7, 5, 0, 0, 191, 192, - 7, 12, 0, 0, 192, 48, 1, 0, 0, 0, 193, 194, 7, 18, 0, 0, 194, 195, 7, 16, - 0, 0, 195, 196, 7, 11, 0, 0, 196, 50, 1, 0, 0, 0, 197, 198, 7, 18, 0, 0, - 198, 199, 7, 16, 0, 0, 199, 200, 7, 11, 0, 0, 200, 201, 7, 16, 0, 0, 201, - 202, 7, 4, 0, 0, 202, 203, 7, 19, 0, 0, 203, 52, 1, 0, 0, 0, 204, 205, - 7, 18, 0, 0, 205, 206, 7, 16, 0, 0, 206, 207, 7, 11, 0, 0, 207, 208, 7, - 16, 0, 0, 208, 209, 7, 0, 0, 0, 209, 210, 7, 0, 0, 0, 210, 54, 1, 0, 0, - 0, 211, 212, 7, 6, 0, 0, 212, 213, 7, 12, 0, 0, 213, 214, 7, 20, 0, 0, - 214, 221, 7, 3, 0, 0, 215, 216, 7, 21, 0, 0, 216, 217, 7, 16, 0, 0, 217, - 218, 7, 0, 0, 0, 218, 219, 7, 11, 0, 0, 219, 221, 7, 3, 0, 0, 220, 211, - 1, 0, 0, 0, 220, 215, 1, 0, 0, 0, 221, 56, 1, 0, 0, 0, 222, 223, 7, 22, - 0, 0, 223, 58, 1, 0, 0, 0, 224, 226, 3, 57, 28, 0, 225, 224, 1, 0, 0, 0, - 225, 226, 1, 0, 0, 0, 226, 228, 1, 0, 0, 0, 227, 229, 3, 73, 36, 0, 228, - 227, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 228, 1, 0, 0, 0, 230, 231, - 1, 0, 0, 0, 231, 239, 1, 0, 0, 0, 232, 236, 5, 46, 0, 0, 233, 235, 3, 73, - 36, 0, 234, 233, 1, 0, 0, 0, 235, 238, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, - 236, 237, 1, 0, 0, 0, 237, 240, 1, 0, 0, 0, 238, 236, 1, 0, 0, 0, 239, - 232, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 250, 1, 0, 0, 0, 241, 243, - 7, 3, 0, 0, 242, 244, 3, 57, 28, 0, 243, 242, 1, 0, 0, 0, 243, 244, 1, - 0, 0, 0, 244, 246, 1, 0, 0, 0, 245, 247, 3, 73, 36, 0, 246, 245, 1, 0, - 0, 0, 247, 248, 1, 0, 0, 0, 248, 246, 1, 0, 0, 0, 248, 249, 1, 0, 0, 0, - 249, 251, 1, 0, 0, 0, 250, 241, 1, 0, 0, 0, 250, 251, 1, 0, 0, 0, 251, - 273, 1, 0, 0, 0, 252, 254, 3, 57, 28, 0, 253, 252, 1, 0, 0, 0, 253, 254, - 1, 0, 0, 0, 254, 255, 1, 0, 0, 0, 255, 257, 5, 46, 0, 0, 256, 258, 3, 73, - 36, 0, 257, 256, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 257, 1, 0, 0, 0, - 259, 260, 1, 0, 0, 0, 260, 270, 1, 0, 0, 0, 261, 263, 7, 3, 0, 0, 262, - 264, 3, 57, 28, 0, 263, 262, 1, 0, 0, 0, 263, 264, 1, 0, 0, 0, 264, 266, - 1, 0, 0, 0, 265, 267, 3, 73, 36, 0, 266, 265, 1, 0, 0, 0, 267, 268, 1, - 0, 0, 0, 268, 266, 1, 0, 0, 0, 268, 269, 1, 0, 0, 0, 269, 271, 1, 0, 0, - 0, 270, 261, 1, 0, 0, 0, 270, 271, 1, 0, 0, 0, 271, 273, 1, 0, 0, 0, 272, - 225, 1, 0, 0, 0, 272, 253, 1, 0, 0, 0, 273, 60, 1, 0, 0, 0, 274, 280, 5, - 34, 0, 0, 275, 279, 8, 23, 0, 0, 276, 277, 5, 92, 0, 0, 277, 279, 9, 0, - 0, 0, 278, 275, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, - 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 283, 1, 0, 0, 0, 282, - 280, 1, 0, 0, 0, 283, 295, 5, 34, 0, 0, 284, 290, 5, 39, 0, 0, 285, 289, - 8, 24, 0, 0, 286, 287, 5, 92, 0, 0, 287, 289, 9, 0, 0, 0, 288, 285, 1, - 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 292, 1, 0, 0, 0, 290, 288, 1, 0, 0, - 0, 290, 291, 1, 0, 0, 0, 291, 293, 1, 0, 0, 0, 292, 290, 1, 0, 0, 0, 293, - 295, 5, 39, 0, 0, 294, 274, 1, 0, 0, 0, 294, 284, 1, 0, 0, 0, 295, 62, - 1, 0, 0, 0, 296, 300, 7, 25, 0, 0, 297, 299, 7, 26, 0, 0, 298, 297, 1, - 0, 0, 0, 299, 302, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, 300, 301, 1, 0, 0, - 0, 301, 64, 1, 0, 0, 0, 302, 300, 1, 0, 0, 0, 303, 304, 5, 91, 0, 0, 304, - 305, 5, 93, 0, 0, 305, 66, 1, 0, 0, 0, 306, 307, 5, 91, 0, 0, 307, 308, - 5, 42, 0, 0, 308, 309, 5, 93, 0, 0, 309, 68, 1, 0, 0, 0, 310, 317, 3, 63, - 31, 0, 311, 312, 5, 46, 0, 0, 312, 316, 3, 63, 31, 0, 313, 316, 3, 65, - 32, 0, 314, 316, 3, 67, 33, 0, 315, 311, 1, 0, 0, 0, 315, 313, 1, 0, 0, - 0, 315, 314, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, - 318, 1, 0, 0, 0, 318, 70, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 7, - 27, 0, 0, 321, 320, 1, 0, 0, 0, 322, 323, 1, 0, 0, 0, 323, 321, 1, 0, 0, - 0, 323, 324, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 326, 6, 35, 0, 0, 326, - 72, 1, 0, 0, 0, 327, 328, 7, 28, 0, 0, 328, 74, 1, 0, 0, 0, 329, 331, 8, - 29, 0, 0, 330, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 330, 1, 0, 0, - 0, 332, 333, 1, 0, 0, 0, 333, 76, 1, 0, 0, 0, 30, 0, 90, 119, 138, 160, - 177, 220, 225, 230, 236, 239, 243, 248, 250, 253, 259, 263, 268, 270, 272, - 278, 280, 288, 290, 294, 300, 315, 317, 323, 332, 1, 6, 0, 0, + 92, 4, 0, 36, 36, 65, 90, 95, 95, 97, 122, 6, 0, 36, 36, 45, 45, 47, 58, + 65, 90, 95, 95, 97, 122, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 8, + 0, 9, 10, 13, 13, 32, 34, 39, 41, 44, 44, 60, 62, 91, 91, 93, 93, 358, + 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, + 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, + 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, + 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, + 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, + 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, + 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, + 0, 55, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 69, 1, 0, 0, + 0, 0, 71, 1, 0, 0, 0, 0, 75, 1, 0, 0, 0, 1, 77, 1, 0, 0, 0, 3, 79, 1, 0, + 0, 0, 5, 81, 1, 0, 0, 0, 7, 83, 1, 0, 0, 0, 9, 85, 1, 0, 0, 0, 11, 90, + 1, 0, 0, 0, 13, 92, 1, 0, 0, 0, 15, 95, 1, 0, 0, 0, 17, 98, 1, 0, 0, 0, + 19, 100, 1, 0, 0, 0, 21, 103, 1, 0, 0, 0, 23, 105, 1, 0, 0, 0, 25, 108, + 1, 0, 0, 0, 27, 113, 1, 0, 0, 0, 29, 126, 1, 0, 0, 0, 31, 132, 1, 0, 0, + 0, 33, 146, 1, 0, 0, 0, 35, 154, 1, 0, 0, 0, 37, 162, 1, 0, 0, 0, 39, 169, + 1, 0, 0, 0, 41, 179, 1, 0, 0, 0, 43, 182, 1, 0, 0, 0, 45, 186, 1, 0, 0, + 0, 47, 190, 1, 0, 0, 0, 49, 193, 1, 0, 0, 0, 51, 197, 1, 0, 0, 0, 53, 204, + 1, 0, 0, 0, 55, 220, 1, 0, 0, 0, 57, 222, 1, 0, 0, 0, 59, 272, 1, 0, 0, + 0, 61, 294, 1, 0, 0, 0, 63, 296, 1, 0, 0, 0, 65, 303, 1, 0, 0, 0, 67, 306, + 1, 0, 0, 0, 69, 310, 1, 0, 0, 0, 71, 321, 1, 0, 0, 0, 73, 327, 1, 0, 0, + 0, 75, 330, 1, 0, 0, 0, 77, 78, 5, 40, 0, 0, 78, 2, 1, 0, 0, 0, 79, 80, + 5, 41, 0, 0, 80, 4, 1, 0, 0, 0, 81, 82, 5, 91, 0, 0, 82, 6, 1, 0, 0, 0, + 83, 84, 5, 93, 0, 0, 84, 8, 1, 0, 0, 0, 85, 86, 5, 44, 0, 0, 86, 10, 1, + 0, 0, 0, 87, 91, 5, 61, 0, 0, 88, 89, 5, 61, 0, 0, 89, 91, 5, 61, 0, 0, + 90, 87, 1, 0, 0, 0, 90, 88, 1, 0, 0, 0, 91, 12, 1, 0, 0, 0, 92, 93, 5, + 33, 0, 0, 93, 94, 5, 61, 0, 0, 94, 14, 1, 0, 0, 0, 95, 96, 5, 60, 0, 0, + 96, 97, 5, 62, 0, 0, 97, 16, 1, 0, 0, 0, 98, 99, 5, 60, 0, 0, 99, 18, 1, + 0, 0, 0, 100, 101, 5, 60, 0, 0, 101, 102, 5, 61, 0, 0, 102, 20, 1, 0, 0, + 0, 103, 104, 5, 62, 0, 0, 104, 22, 1, 0, 0, 0, 105, 106, 5, 62, 0, 0, 106, + 107, 5, 61, 0, 0, 107, 24, 1, 0, 0, 0, 108, 109, 7, 0, 0, 0, 109, 110, + 7, 1, 0, 0, 110, 111, 7, 2, 0, 0, 111, 112, 7, 3, 0, 0, 112, 26, 1, 0, + 0, 0, 113, 114, 7, 4, 0, 0, 114, 115, 7, 5, 0, 0, 115, 117, 7, 6, 0, 0, + 116, 118, 7, 7, 0, 0, 117, 116, 1, 0, 0, 0, 118, 119, 1, 0, 0, 0, 119, + 117, 1, 0, 0, 0, 119, 120, 1, 0, 0, 0, 120, 121, 1, 0, 0, 0, 121, 122, + 7, 0, 0, 0, 122, 123, 7, 1, 0, 0, 123, 124, 7, 2, 0, 0, 124, 125, 7, 3, + 0, 0, 125, 28, 1, 0, 0, 0, 126, 127, 7, 1, 0, 0, 127, 128, 7, 0, 0, 0, + 128, 129, 7, 1, 0, 0, 129, 130, 7, 2, 0, 0, 130, 131, 7, 3, 0, 0, 131, + 30, 1, 0, 0, 0, 132, 133, 7, 4, 0, 0, 133, 134, 7, 5, 0, 0, 134, 136, 7, + 6, 0, 0, 135, 137, 7, 7, 0, 0, 136, 135, 1, 0, 0, 0, 137, 138, 1, 0, 0, + 0, 138, 136, 1, 0, 0, 0, 138, 139, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, + 141, 7, 1, 0, 0, 141, 142, 7, 0, 0, 0, 142, 143, 7, 1, 0, 0, 143, 144, + 7, 2, 0, 0, 144, 145, 7, 3, 0, 0, 145, 32, 1, 0, 0, 0, 146, 147, 7, 8, + 0, 0, 147, 148, 7, 3, 0, 0, 148, 149, 7, 6, 0, 0, 149, 150, 7, 9, 0, 0, + 150, 151, 7, 3, 0, 0, 151, 152, 7, 3, 0, 0, 152, 153, 7, 4, 0, 0, 153, + 34, 1, 0, 0, 0, 154, 155, 7, 3, 0, 0, 155, 156, 7, 10, 0, 0, 156, 157, + 7, 1, 0, 0, 157, 158, 7, 11, 0, 0, 158, 160, 7, 6, 0, 0, 159, 161, 7, 11, + 0, 0, 160, 159, 1, 0, 0, 0, 160, 161, 1, 0, 0, 0, 161, 36, 1, 0, 0, 0, + 162, 163, 7, 12, 0, 0, 163, 164, 7, 3, 0, 0, 164, 165, 7, 13, 0, 0, 165, + 166, 7, 3, 0, 0, 166, 167, 7, 10, 0, 0, 167, 168, 7, 14, 0, 0, 168, 38, + 1, 0, 0, 0, 169, 170, 7, 15, 0, 0, 170, 171, 7, 5, 0, 0, 171, 172, 7, 4, + 0, 0, 172, 173, 7, 6, 0, 0, 173, 174, 7, 16, 0, 0, 174, 175, 7, 1, 0, 0, + 175, 177, 7, 4, 0, 0, 176, 178, 7, 11, 0, 0, 177, 176, 1, 0, 0, 0, 177, + 178, 1, 0, 0, 0, 178, 40, 1, 0, 0, 0, 179, 180, 7, 1, 0, 0, 180, 181, 7, + 4, 0, 0, 181, 42, 1, 0, 0, 0, 182, 183, 7, 4, 0, 0, 183, 184, 7, 5, 0, + 0, 184, 185, 7, 6, 0, 0, 185, 44, 1, 0, 0, 0, 186, 187, 7, 16, 0, 0, 187, + 188, 7, 4, 0, 0, 188, 189, 7, 17, 0, 0, 189, 46, 1, 0, 0, 0, 190, 191, + 7, 5, 0, 0, 191, 192, 7, 12, 0, 0, 192, 48, 1, 0, 0, 0, 193, 194, 7, 18, + 0, 0, 194, 195, 7, 16, 0, 0, 195, 196, 7, 11, 0, 0, 196, 50, 1, 0, 0, 0, + 197, 198, 7, 18, 0, 0, 198, 199, 7, 16, 0, 0, 199, 200, 7, 11, 0, 0, 200, + 201, 7, 16, 0, 0, 201, 202, 7, 4, 0, 0, 202, 203, 7, 19, 0, 0, 203, 52, + 1, 0, 0, 0, 204, 205, 7, 18, 0, 0, 205, 206, 7, 16, 0, 0, 206, 207, 7, + 11, 0, 0, 207, 208, 7, 16, 0, 0, 208, 209, 7, 0, 0, 0, 209, 210, 7, 0, + 0, 0, 210, 54, 1, 0, 0, 0, 211, 212, 7, 6, 0, 0, 212, 213, 7, 12, 0, 0, + 213, 214, 7, 20, 0, 0, 214, 221, 7, 3, 0, 0, 215, 216, 7, 21, 0, 0, 216, + 217, 7, 16, 0, 0, 217, 218, 7, 0, 0, 0, 218, 219, 7, 11, 0, 0, 219, 221, + 7, 3, 0, 0, 220, 211, 1, 0, 0, 0, 220, 215, 1, 0, 0, 0, 221, 56, 1, 0, + 0, 0, 222, 223, 7, 22, 0, 0, 223, 58, 1, 0, 0, 0, 224, 226, 3, 57, 28, + 0, 225, 224, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 228, 1, 0, 0, 0, 227, + 229, 3, 73, 36, 0, 228, 227, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 228, + 1, 0, 0, 0, 230, 231, 1, 0, 0, 0, 231, 239, 1, 0, 0, 0, 232, 236, 5, 46, + 0, 0, 233, 235, 3, 73, 36, 0, 234, 233, 1, 0, 0, 0, 235, 238, 1, 0, 0, + 0, 236, 234, 1, 0, 0, 0, 236, 237, 1, 0, 0, 0, 237, 240, 1, 0, 0, 0, 238, + 236, 1, 0, 0, 0, 239, 232, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 250, + 1, 0, 0, 0, 241, 243, 7, 3, 0, 0, 242, 244, 3, 57, 28, 0, 243, 242, 1, + 0, 0, 0, 243, 244, 1, 0, 0, 0, 244, 246, 1, 0, 0, 0, 245, 247, 3, 73, 36, + 0, 246, 245, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 246, 1, 0, 0, 0, 248, + 249, 1, 0, 0, 0, 249, 251, 1, 0, 0, 0, 250, 241, 1, 0, 0, 0, 250, 251, + 1, 0, 0, 0, 251, 273, 1, 0, 0, 0, 252, 254, 3, 57, 28, 0, 253, 252, 1, + 0, 0, 0, 253, 254, 1, 0, 0, 0, 254, 255, 1, 0, 0, 0, 255, 257, 5, 46, 0, + 0, 256, 258, 3, 73, 36, 0, 257, 256, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, + 259, 257, 1, 0, 0, 0, 259, 260, 1, 0, 0, 0, 260, 270, 1, 0, 0, 0, 261, + 263, 7, 3, 0, 0, 262, 264, 3, 57, 28, 0, 263, 262, 1, 0, 0, 0, 263, 264, + 1, 0, 0, 0, 264, 266, 1, 0, 0, 0, 265, 267, 3, 73, 36, 0, 266, 265, 1, + 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 266, 1, 0, 0, 0, 268, 269, 1, 0, 0, + 0, 269, 271, 1, 0, 0, 0, 270, 261, 1, 0, 0, 0, 270, 271, 1, 0, 0, 0, 271, + 273, 1, 0, 0, 0, 272, 225, 1, 0, 0, 0, 272, 253, 1, 0, 0, 0, 273, 60, 1, + 0, 0, 0, 274, 280, 5, 34, 0, 0, 275, 279, 8, 23, 0, 0, 276, 277, 5, 92, + 0, 0, 277, 279, 9, 0, 0, 0, 278, 275, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, + 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, + 283, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 295, 5, 34, 0, 0, 284, 290, + 5, 39, 0, 0, 285, 289, 8, 24, 0, 0, 286, 287, 5, 92, 0, 0, 287, 289, 9, + 0, 0, 0, 288, 285, 1, 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 292, 1, 0, 0, + 0, 290, 288, 1, 0, 0, 0, 290, 291, 1, 0, 0, 0, 291, 293, 1, 0, 0, 0, 292, + 290, 1, 0, 0, 0, 293, 295, 5, 39, 0, 0, 294, 274, 1, 0, 0, 0, 294, 284, + 1, 0, 0, 0, 295, 62, 1, 0, 0, 0, 296, 300, 7, 25, 0, 0, 297, 299, 7, 26, + 0, 0, 298, 297, 1, 0, 0, 0, 299, 302, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, + 300, 301, 1, 0, 0, 0, 301, 64, 1, 0, 0, 0, 302, 300, 1, 0, 0, 0, 303, 304, + 5, 91, 0, 0, 304, 305, 5, 93, 0, 0, 305, 66, 1, 0, 0, 0, 306, 307, 5, 91, + 0, 0, 307, 308, 5, 42, 0, 0, 308, 309, 5, 93, 0, 0, 309, 68, 1, 0, 0, 0, + 310, 317, 3, 63, 31, 0, 311, 312, 5, 46, 0, 0, 312, 316, 3, 63, 31, 0, + 313, 316, 3, 65, 32, 0, 314, 316, 3, 67, 33, 0, 315, 311, 1, 0, 0, 0, 315, + 313, 1, 0, 0, 0, 315, 314, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, + 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 70, 1, 0, 0, 0, 319, 317, 1, 0, + 0, 0, 320, 322, 7, 27, 0, 0, 321, 320, 1, 0, 0, 0, 322, 323, 1, 0, 0, 0, + 323, 321, 1, 0, 0, 0, 323, 324, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, + 326, 6, 35, 0, 0, 326, 72, 1, 0, 0, 0, 327, 328, 7, 28, 0, 0, 328, 74, + 1, 0, 0, 0, 329, 331, 8, 29, 0, 0, 330, 329, 1, 0, 0, 0, 331, 332, 1, 0, + 0, 0, 332, 330, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 76, 1, 0, 0, 0, + 30, 0, 90, 119, 138, 160, 177, 220, 225, 230, 236, 239, 243, 248, 250, + 253, 259, 263, 268, 270, 272, 278, 280, 288, 290, 294, 300, 315, 317, 323, + 332, 1, 6, 0, 0, } deserializer := antlr.NewATNDeserializer(nil) staticData.atn = deserializer.Deserialize(staticData.serializedATN) diff --git a/pkg/querier/builder_query.go b/pkg/querier/builder_query.go index 5123b2e25bab..756dbd318b7b 100644 --- a/pkg/querier/builder_query.go +++ b/pkg/querier/builder_query.go @@ -89,6 +89,12 @@ func (q *builderQuery[T]) Fingerprint() string { // Add filter if present if q.spec.Filter != nil && q.spec.Filter.Expression != "" { parts = append(parts, fmt.Sprintf("filter=%s", q.spec.Filter.Expression)) + + for name, item := range q.variables { + if strings.Contains(q.spec.Filter.Expression, "$"+name) { + parts = append(parts, fmt.Sprintf("%s=%s", name, fmt.Sprint(item.Value))) + } + } } // Add group by keys @@ -210,6 +216,15 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string, return nil, errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "Query timed out"). WithAdditional("Try refining your search by adding relevant resource attributes filtering") } + + if !errors.Is(err, context.Canceled) { + return nil, errors.Newf( + errors.TypeInternal, + errors.CodeInternal, + "Something went wrong on our end. It's not you, it's us. Our team is notified about it. Reach out to support if issue persists.", + ) + } + return nil, err } defer rows.Close() diff --git a/pkg/querier/consume.go b/pkg/querier/consume.go index a39c57a73978..3b1ab29efcd6 100644 --- a/pkg/querier/consume.go +++ b/pkg/querier/consume.go @@ -5,6 +5,7 @@ import ( "math" "reflect" "regexp" + "slices" "sort" "strconv" "strings" @@ -17,6 +18,10 @@ import ( var ( aggRe = regexp.MustCompile(`^__result_(\d+)$`) + // legacyReservedColumnTargetAliases identifies result value from a user + // written clickhouse query. The column alias indcate which value is + // to be considered as final result (or target) + legacyReservedColumnTargetAliases = []string{"__result", "__value", "result", "res", "value"} ) // consume reads every row and shapes it into the payload expected for the @@ -131,6 +136,9 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt } else if numericColsCount == 1 { // classic single-value query fallbackValue = val fallbackSeen = true + } else if slices.Contains(legacyReservedColumnTargetAliases, name) { + fallbackValue = val + fallbackSeen = true } else { // numeric label lblVals = append(lblVals, fmt.Sprint(val)) @@ -150,6 +158,9 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt } else if numericColsCount == 1 { // classic single-value query fallbackValue = val fallbackSeen = true + } else if slices.Contains(legacyReservedColumnTargetAliases, name) { + fallbackValue = val + fallbackSeen = true } else { // numeric label lblVals = append(lblVals, fmt.Sprint(val)) diff --git a/pkg/querier/postprocess.go b/pkg/querier/postprocess.go index d001d96ea87d..924e64173c9e 100644 --- a/pkg/querier/postprocess.go +++ b/pkg/querier/postprocess.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/SigNoz/govaluate" + "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" ) @@ -44,6 +45,73 @@ func getQueryName(spec any) string { return getqueryInfo(spec).Name } +func StepIntervalForQuery(req *qbtypes.QueryRangeRequest, name string) int64 { + stepsMap := make(map[string]int64) + for _, query := range req.CompositeQuery.Queries { + switch spec := query.Spec.(type) { + case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]: + stepsMap[spec.Name] = int64(spec.StepInterval.Seconds()) + case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]: + stepsMap[spec.Name] = int64(spec.StepInterval.Seconds()) + case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]: + stepsMap[spec.Name] = int64(spec.StepInterval.Seconds()) + case qbtypes.PromQuery: + stepsMap[spec.Name] = int64(spec.Step.Seconds()) + } + } + + if step, ok := stepsMap[name]; ok { + return step + } + + exprStr := "" + + for _, query := range req.CompositeQuery.Queries { + switch spec := query.Spec.(type) { + case qbtypes.QueryBuilderFormula: + if spec.Name == name { + exprStr = spec.Expression + } + } + } + + expression, _ := govaluate.NewEvaluableExpressionWithFunctions(exprStr, qbtypes.EvalFuncs()) + + steps := []int64{} + + for _, v := range expression.Vars() { + steps = append(steps, stepsMap[v]) + } + + return querybuilder.LCMList(steps) +} + +func NumAggregationForQuery(req *qbtypes.QueryRangeRequest, name string) int64 { + numAgg := 0 + for _, query := range req.CompositeQuery.Queries { + switch spec := query.Spec.(type) { + case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]: + if spec.Name == name { + numAgg += 1 + } + case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]: + if spec.Name == name { + numAgg += 1 + } + case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]: + if spec.Name == name { + numAgg += 1 + } + case qbtypes.QueryBuilderFormula: + if spec.Name == name { + numAgg += 1 + } + } + } + + return int64(numAgg) +} + func (q *querier) postProcessResults(ctx context.Context, results map[string]any, req *qbtypes.QueryRangeRequest) (map[string]any, error) { // Convert results to typed format for processing typedResults := make(map[string]*qbtypes.Result) @@ -81,6 +149,18 @@ func (q *querier) postProcessResults(ctx context.Context, results map[string]any // Apply table formatting for UI if requested if req.FormatOptions != nil && req.FormatOptions.FormatTableResultForUI && req.RequestType == qbtypes.RequestTypeScalar { + + // merge result only needed for non-CH query + if len(req.CompositeQuery.Queries) == 1 { + if req.CompositeQuery.Queries[0].Type == qbtypes.QueryTypeClickHouseSQL { + retResult := map[string]any{} + for name, v := range typedResults { + retResult[name] = v.Value + } + return retResult, nil + } + } + // Format results as a table - this merges all queries into a single table tableResult := q.formatScalarResultsAsTable(typedResults, req) @@ -96,6 +176,36 @@ func (q *querier) postProcessResults(ctx context.Context, results map[string]any return tableResult, nil } + if req.RequestType == qbtypes.RequestTypeTimeSeries && req.FormatOptions != nil && req.FormatOptions.FillGaps { + for name := range typedResults { + funcs := []qbtypes.Function{{Name: qbtypes.FunctionNameFillZero}} + funcs = q.prepareFillZeroArgsWithStep(funcs, req, StepIntervalForQuery(req, name)) + // empty time series if it doesn't exist + tsData, ok := typedResults[name].Value.(*qbtypes.TimeSeriesData) + if !ok { + tsData = &qbtypes.TimeSeriesData{} + } + + if len(tsData.Aggregations) == 0 { + numAgg := NumAggregationForQuery(req, name) + tsData.Aggregations = make([]*qbtypes.AggregationBucket, numAgg) + for idx := range numAgg { + tsData.Aggregations[idx] = &qbtypes.AggregationBucket{ + Index: int(idx), + Series: []*qbtypes.TimeSeries{ + { + Labels: make([]*qbtypes.Label, 0), + Values: make([]*qbtypes.TimeSeriesValue, 0), + }, + }, + } + } + } + + typedResults[name] = q.applyFunctions(typedResults[name], funcs) + } + } + // Convert back to map[string]any finalResults := make(map[string]any) for name, result := range typedResults { @@ -131,6 +241,19 @@ func postProcessMetricQuery( req *qbtypes.QueryRangeRequest, ) *qbtypes.Result { + config := query.Aggregations[0] + spaceAggOrderBy := fmt.Sprintf("%s(%s)", config.SpaceAggregation.StringValue(), config.MetricName) + timeAggOrderBy := fmt.Sprintf("%s(%s)", config.TimeAggregation.StringValue(), config.MetricName) + timeSpaceAggOrderBy := fmt.Sprintf("%s(%s(%s))", config.SpaceAggregation.StringValue(), config.TimeAggregation.StringValue(), config.MetricName) + + for idx := range query.Order { + if query.Order[idx].Key.Name == spaceAggOrderBy || + query.Order[idx].Key.Name == timeAggOrderBy || + query.Order[idx].Key.Name == timeSpaceAggOrderBy { + query.Order[idx].Key.Name = qbtypes.DefaultOrderByKey + } + } + if query.Limit > 0 { result = q.applySeriesLimit(result, query.Limit, query.Order) } @@ -224,6 +347,13 @@ func (q *querier) applyFormulas(ctx context.Context, results map[string]*qbtypes // Process each formula for name, formula := range formulaQueries { + + for idx := range formula.Order { + if formula.Order[idx].Key.Name == formula.Name || formula.Order[idx].Key.Name == formula.Expression { + formula.Order[idx].Key.Name = qbtypes.DefaultOrderByKey + } + } + // Check if we're dealing with time series or scalar data if req.RequestType == qbtypes.RequestTypeTimeSeries { result := q.processTimeSeriesFormula(ctx, results, formula, req) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 3a473d1b816f..c9c5457a69d6 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -162,6 +162,17 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype Duration: time.Second * time.Duration(querybuilder.MinAllowedStepIntervalForMetric(req.Start, req.End)), } } + + req.CompositeQuery.Queries[idx].Spec = spec + } + } else if query.Type == qbtypes.QueryTypePromQL { + switch spec := query.Spec.(type) { + case qbtypes.PromQuery: + if spec.Step.Seconds() == 0 { + spec.Step = qbtypes.Step{ + Duration: time.Second * time.Duration(querybuilder.RecommendedStepIntervalForMetric(req.Start, req.End)), + } + } req.CompositeQuery.Queries[idx].Spec = spec } } diff --git a/pkg/querybuilder/fallback_expr.go b/pkg/querybuilder/fallback_expr.go index ca69bf16a39f..425e305699fe 100644 --- a/pkg/querybuilder/fallback_expr.go +++ b/pkg/querybuilder/fallback_expr.go @@ -56,6 +56,16 @@ func CollisionHandledFinalExpr( // the key didn't have the right context to be added to the query // we try to use the context we know of keysForField := keys[field.Name] + + if len(keysForField) == 0 { + // check if the key exists with {fieldContext}.{key} + // because the context could be legitimate prefix in user data, example `metric.max` + keyWithContext := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name) + if len(keys[keyWithContext]) > 0 { + keysForField = keys[keyWithContext] + } + } + if len(keysForField) == 0 { // - the context is not provided // - there are not keys for the field @@ -68,7 +78,7 @@ func CollisionHandledFinalExpr( return "", nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, correction) } else { // not even a close match, return an error - return "", nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field %s not found", field.Name) + return "", nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name) } } else { for _, key := range keysForField { @@ -90,6 +100,10 @@ func CollisionHandledFinalExpr( stmts = append(stmts, colName) } + for idx := range stmts { + stmts[idx] = sqlbuilder.Escape(stmts[idx]) + } + multiIfStmt := fmt.Sprintf("multiIf(%s, NULL)", strings.Join(stmts, ", ")) return multiIfStmt, allArgs, nil diff --git a/pkg/querybuilder/resourcefilter/statement_builder.go b/pkg/querybuilder/resourcefilter/statement_builder.go index 1deb32eee3d8..3a3cb7fe283d 100644 --- a/pkg/querybuilder/resourcefilter/statement_builder.go +++ b/pkg/querybuilder/resourcefilter/statement_builder.go @@ -155,6 +155,8 @@ func (b *resourceFilterStatementBuilder[T]) addConditions( JsonKeyToKey: b.jsonKeyToKey, SkipFullTextFilter: true, SkipFunctionCalls: true, + // there is no need for "key" not found error for resource filtering + IgnoreNotFoundKeys: true, Variables: variables, }) diff --git a/pkg/querybuilder/time.go b/pkg/querybuilder/time.go index f3e50db150da..d18d2ac0d91c 100644 --- a/pkg/querybuilder/time.go +++ b/pkg/querybuilder/time.go @@ -3,6 +3,10 @@ package querybuilder import ( "fmt" "math" + "time" + + "github.com/SigNoz/signoz/pkg/types/metrictypes" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" ) const ( @@ -69,7 +73,16 @@ func RecommendedStepIntervalForMetric(start, end uint64) uint64 { } // return the nearest lower multiple of 60 - return step - step%60 + recommended := step - step%60 + + // if the time range is greater than 1 day, and less than 1 week set the step interval to be multiple of 5 minutes + // if the time range is greater than 1 week, set the step interval to be multiple of 30 mins + if end-start >= uint64(24*time.Hour.Nanoseconds()) && end-start < uint64(7*24*time.Hour.Nanoseconds()) { + recommended = uint64(math.Round(float64(recommended)/300)) * 300 + } else if end-start >= uint64(7*24*time.Hour.Nanoseconds()) { + recommended = uint64(math.Round(float64(recommended)/1800)) * 1800 + } + return recommended } func MinAllowedStepIntervalForMetric(start, end uint64) uint64 { @@ -84,7 +97,64 @@ func MinAllowedStepIntervalForMetric(start, end uint64) uint64 { } // return the nearest lower multiple of 60 - return step - step%60 + minAllowed := step - step%60 + + // if the time range is greater than 1 day, and less than 1 week set the step interval to be multiple of 5 minutes + // if the time range is greater than 1 week, set the step interval to be multiple of 30 mins + if end-start >= uint64(24*time.Hour.Nanoseconds()) && end-start < uint64(7*24*time.Hour.Nanoseconds()) { + minAllowed = uint64(math.Round(float64(minAllowed)/300)) * 300 + } else if end-start >= uint64(7*24*time.Hour.Nanoseconds()) { + minAllowed = uint64(math.Round(float64(minAllowed)/1800)) * 1800 + } + return minAllowed +} + +func AdjustedMetricTimeRange(start, end, step uint64, mq qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) (uint64, uint64) { + // align the start to the step interval + start = start - (start % (step * 1000)) + // if the query is a rate query, we adjust the start time by one more step + // so that we can calculate the rate for the first data point + hasRunningDiff := false + for _, fn := range mq.Functions { + if fn.Name == qbtypes.FunctionNameRunningDiff { + hasRunningDiff = true + break + } + } + if (mq.Aggregations[0].TimeAggregation == metrictypes.TimeAggregationRate || mq.Aggregations[0].TimeAggregation == metrictypes.TimeAggregationIncrease) && + mq.Aggregations[0].Temporality != metrictypes.Delta { + start -= step * 1000 + } + if hasRunningDiff { + start -= step * 1000 + } + // align the end to the nearest minute + adjustStep := uint64(math.Min(float64(step), 60)) + end = end - (end % (adjustStep * 1000)) + return start, end +} + +func GCD(a, b int64) int64 { + for b != 0 { + a, b = b, a%b + } + return a +} + +func LCM(a, b int64) int64 { + return (a * b) / GCD(a, b) +} + +// LCMList computes the LCM of a list of int64 numbers. +func LCMList(nums []int64) int64 { + if len(nums) == 0 { + return 1 + } + result := nums[0] + for _, num := range nums[1:] { + result = LCM(result, num) + } + return result } func AssignReservedVars(vars map[string]any, start, end uint64) { diff --git a/pkg/querybuilder/where_clause_visitor.go b/pkg/querybuilder/where_clause_visitor.go index 2f05ec3b93ec..936803cd6276 100644 --- a/pkg/querybuilder/where_clause_visitor.go +++ b/pkg/querybuilder/where_clause_visitor.go @@ -30,6 +30,7 @@ type filterExpressionVisitor struct { skipResourceFilter bool skipFullTextFilter bool skipFunctionCalls bool + ignoreNotFoundKeys bool variables map[string]qbtypes.VariableItem keysWithWarnings map[string]bool @@ -46,6 +47,7 @@ type FilterExprVisitorOpts struct { SkipResourceFilter bool SkipFullTextFilter bool SkipFunctionCalls bool + IgnoreNotFoundKeys bool Variables map[string]qbtypes.VariableItem } @@ -62,6 +64,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis skipResourceFilter: opts.SkipResourceFilter, skipFullTextFilter: opts.SkipFullTextFilter, skipFunctionCalls: opts.SkipFunctionCalls, + ignoreNotFoundKeys: opts.IgnoreNotFoundKeys, variables: opts.Variables, keysWithWarnings: make(map[string]bool), } @@ -292,6 +295,15 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext) any { keys := v.Visit(ctx.Key()).([]*telemetrytypes.TelemetryFieldKey) + // if key is missing and can be ignored, the condition is ignored + if len(keys) == 0 && v.ignoreNotFoundKeys { + // Why do we return "true"? to prevent from create a empty tuple + // example, if the condition is (x AND (y OR z)) + // if we find ourselves ignoring all, then it creates and invalid + // condition (()) which throws invalid tuples error + return "true" + } + // this is used to skip the resource filtering on main table if // the query may use the resources table sub-query filter if v.skipResourceFilter { @@ -302,6 +314,13 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext } } keys = filteredKeys + if len(keys) == 0 { + // Why do we return "true"? to prevent from create a empty tuple + // example, if the condition is (resource.service.name='api' AND (env='prod' OR env='production')) + // if we find ourselves skipping all, then it creates and invalid + // condition (()) which throws invalid tuples error + return "true" + } } // Handle EXISTS specially @@ -429,7 +448,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext var varItem qbtypes.VariableItem varItem, ok = v.variables[var_] // if not present, try without `$` prefix - if !ok { + if !ok && len(var_) > 0 { varItem, ok = v.variables[var_[1:]] } @@ -680,6 +699,19 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any { fieldKeysForName := v.fieldKeys[keyName] + // if the context is explicitly provided, filter out the remaining + // example, resource.attr = 'value', then we don't want to search on + // anything other than the resource attributes + if fieldKey.FieldContext != telemetrytypes.FieldContextUnspecified { + filteredKeys := []*telemetrytypes.TelemetryFieldKey{} + for _, item := range fieldKeysForName { + if item.FieldContext == fieldKey.FieldContext { + filteredKeys = append(filteredKeys, item) + } + } + fieldKeysForName = filteredKeys + } + // for the body json search, we need to add search on the body field even // if there is a field with the same name as attribute/resource attribute // Since it will ORed with the fieldKeysForName, it will not result empty @@ -691,9 +723,16 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any { } if len(fieldKeysForName) == 0 { + // check if the key exists with {fieldContext}.{key} + // because the context could be legitimate prefix in user data, example `span.div_num = 20` + keyWithContext := fmt.Sprintf("%s.%s", fieldKey.FieldContext.StringValue(), fieldKey.Name) + if len(v.fieldKeys[keyWithContext]) > 0 { + return v.fieldKeys[keyWithContext] + } + if strings.HasPrefix(fieldKey.Name, v.jsonBodyPrefix) && v.jsonBodyPrefix != "" && keyName == "" { v.errors = append(v.errors, "missing key for body json search - expected key of the form `body.key` (ex: `body.status`)") - } else { + } else if !v.ignoreNotFoundKeys { // TODO(srikanthccv): do we want to return an error here? // should we infer the type and auto-magically build a key for expression? v.errors = append(v.errors, fmt.Sprintf("key `%s` not found", fieldKey.Name)) @@ -718,8 +757,13 @@ func trimQuotes(txt string) string { if len(txt) >= 2 { if (txt[0] == '"' && txt[len(txt)-1] == '"') || (txt[0] == '\'' && txt[len(txt)-1] == '\'') { - return txt[1 : len(txt)-1] + txt = txt[1 : len(txt)-1] } } + + // unescape so clickhouse-go can escape it + // https://github.com/ClickHouse/clickhouse-go/blob/6c5ddb38dd2edc841a3b927711b841014759bede/bind.go#L278 + txt = strings.ReplaceAll(txt, `\\`, `\`) + txt = strings.ReplaceAll(txt, `\'`, `'`) return txt } diff --git a/pkg/telemetrylogs/condition_builder_test.go b/pkg/telemetrylogs/condition_builder_test.go index b8c4f7aebb2e..5e8dceff4fd8 100644 --- a/pkg/telemetrylogs/condition_builder_test.go +++ b/pkg/telemetrylogs/condition_builder_test.go @@ -44,7 +44,7 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorGreaterThan, value: float64(100), - expectedSQL: "(attributes_number['request.duration'] > ? AND mapContains(attributes_number, 'request.duration') = ?)", + expectedSQL: "(toFloat64(attributes_number['request.duration']) > ? AND mapContains(attributes_number, 'request.duration') = ?)", expectedArgs: []any{float64(100), true}, expectedError: nil, }, @@ -57,7 +57,7 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorLessThan, value: float64(1024), - expectedSQL: "(attributes_number['request.size'] < ? AND mapContains(attributes_number, 'request.size') = ?)", + expectedSQL: "(toFloat64(attributes_number['request.size']) < ? AND mapContains(attributes_number, 'request.size') = ?)", expectedArgs: []any{float64(1024), true}, expectedError: nil, }, diff --git a/pkg/telemetrylogs/field_mapper.go b/pkg/telemetrylogs/field_mapper.go index 78ff7d4cc36c..db76ecf43792 100644 --- a/pkg/telemetrylogs/field_mapper.go +++ b/pkg/telemetrylogs/field_mapper.go @@ -9,6 +9,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" "golang.org/x/exp/maps" ) @@ -159,7 +160,7 @@ func (m *fieldMapper) ColumnExpressionFor( // is it a static field? if _, ok := logsV2Columns[field.Name]; ok { // if it is, attach the column name directly - field.FieldContext = telemetrytypes.FieldContextSpan + field.FieldContext = telemetrytypes.FieldContextLog colName, _ = m.FieldFor(ctx, field) } else { // - the context is not provided @@ -173,7 +174,7 @@ func (m *fieldMapper) ColumnExpressionFor( return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, correction) } else { // not even a close match, return an error - return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field %s not found", field.Name) + return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name) } } } else if len(keysForField) == 1 { @@ -190,5 +191,5 @@ func (m *fieldMapper) ColumnExpressionFor( } } - return fmt.Sprintf("%s AS `%s`", colName, field.Name), nil + return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil } diff --git a/pkg/telemetrylogs/filter_compiler.go b/pkg/telemetrylogs/filter_compiler.go deleted file mode 100644 index 69dc90bd5297..000000000000 --- a/pkg/telemetrylogs/filter_compiler.go +++ /dev/null @@ -1,55 +0,0 @@ -package telemetrylogs - -import ( - "context" - - "github.com/SigNoz/signoz/pkg/querybuilder" - qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" - "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "github.com/huandu/go-sqlbuilder" -) - -type FilterCompilerOpts struct { - FieldMapper qbtypes.FieldMapper - ConditionBuilder qbtypes.ConditionBuilder - MetadataStore telemetrytypes.MetadataStore - FullTextColumn *telemetrytypes.TelemetryFieldKey - JsonBodyPrefix string - JsonKeyToKey qbtypes.JsonKeyToFieldFunc - SkipResourceFilter bool -} - -type filterCompiler struct { - opts FilterCompilerOpts -} - -func NewFilterCompiler(opts FilterCompilerOpts) *filterCompiler { - return &filterCompiler{ - opts: opts, - } -} - -func (c *filterCompiler) Compile(ctx context.Context, expr string) (*sqlbuilder.WhereClause, []string, error) { - selectors := querybuilder.QueryStringToKeysSelectors(expr) - - keys, err := c.opts.MetadataStore.GetKeysMulti(ctx, selectors) - if err != nil { - return nil, nil, err - } - - filterWhereClause, warnings, err := querybuilder.PrepareWhereClause(expr, querybuilder.FilterExprVisitorOpts{ - FieldMapper: c.opts.FieldMapper, - ConditionBuilder: c.opts.ConditionBuilder, - FieldKeys: keys, - FullTextColumn: c.opts.FullTextColumn, - JsonBodyPrefix: c.opts.JsonBodyPrefix, - JsonKeyToKey: c.opts.JsonKeyToKey, - SkipResourceFilter: c.opts.SkipResourceFilter, - }) - - if err != nil { - return nil, nil, err - } - - return filterWhereClause, warnings, nil -} diff --git a/pkg/telemetrylogs/filter_expr_logs_test.go b/pkg/telemetrylogs/filter_expr_logs_test.go index 4da4026d38fd..6b9d5c64539d 100644 --- a/pkg/telemetrylogs/filter_expr_logs_test.go +++ b/pkg/telemetrylogs/filter_expr_logs_test.go @@ -396,7 +396,7 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with conditions", query: "\"connection timeout\" duration>30", shouldPass: true, - expectedQuery: "WHERE (match(body, ?) AND (attributes_number['duration'] > ? AND mapContains(attributes_number, 'duration') = ?))", + expectedQuery: "WHERE (match(body, ?) AND (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?))", expectedArgs: []any{"connection timeout", float64(30), true}, expectedErrorContains: "", }, @@ -422,7 +422,7 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with parentheses", query: "error (status.code=500 OR status.code=503)", shouldPass: true, - expectedQuery: "WHERE (match(body, ?) AND (((attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?) OR (attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?))))", + expectedQuery: "WHERE (match(body, ?) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))))", expectedArgs: []any{"error", float64(500), true, float64(503), true}, expectedErrorContains: "", }, @@ -430,7 +430,7 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with parentheses", query: "(status.code=500 OR status.code=503) error", shouldPass: true, - expectedQuery: "WHERE ((((attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?) OR (attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?))) AND match(body, ?))", + expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))) AND match(body, ?))", expectedArgs: []any{float64(500), true, float64(503), true, "error"}, expectedErrorContains: "", }, @@ -438,7 +438,7 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with parentheses", query: "error AND (status.code=500 OR status.code=503)", shouldPass: true, - expectedQuery: "WHERE (match(body, ?) AND (((attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?) OR (attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?))))", + expectedQuery: "WHERE (match(body, ?) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))))", expectedArgs: []any{"error", float64(500), true, float64(503), true}, expectedErrorContains: "", }, @@ -446,7 +446,7 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with parentheses", query: "(status.code=500 OR status.code=503) AND error", shouldPass: true, - expectedQuery: "WHERE ((((attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?) OR (attributes_number['status.code'] = ? AND mapContains(attributes_number, 'status.code') = ?))) AND match(body, ?))", + expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))) AND match(body, ?))", expectedArgs: []any{float64(500), true, float64(503), true, "error"}, expectedErrorContains: "", }, @@ -754,7 +754,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Basic equality", query: "status=200", shouldPass: true, - expectedQuery: "WHERE (attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)", expectedArgs: []any{float64(200), true}, expectedErrorContains: "", }, @@ -762,7 +762,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Basic equality", query: "code=400", shouldPass: true, - expectedQuery: "WHERE (attributes_number['code'] = ? AND mapContains(attributes_number, 'code') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['code']) = ? AND mapContains(attributes_number, 'code') = ?)", expectedArgs: []any{float64(400), true}, expectedErrorContains: "", }, @@ -794,7 +794,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Basic equality", query: "count=0", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] = ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) = ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(0), true}, expectedErrorContains: "", }, @@ -812,7 +812,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Not equals", query: "status!=200", shouldPass: true, - expectedQuery: "WHERE attributes_number['status'] <> ?", + expectedQuery: "WHERE toFloat64(attributes_number['status']) <> ?", expectedArgs: []any{float64(200)}, expectedErrorContains: "", }, @@ -820,7 +820,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Not equals", query: "status<>200", shouldPass: true, - expectedQuery: "WHERE attributes_number['status'] <> ?", + expectedQuery: "WHERE toFloat64(attributes_number['status']) <> ?", expectedArgs: []any{float64(200)}, expectedErrorContains: "", }, @@ -828,7 +828,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Not equals", query: "code!=400", shouldPass: true, - expectedQuery: "WHERE attributes_number['code'] <> ?", + expectedQuery: "WHERE toFloat64(attributes_number['code']) <> ?", expectedArgs: []any{float64(400)}, expectedErrorContains: "", }, @@ -854,7 +854,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Less than", query: "count<10", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] < ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(10), true}, expectedErrorContains: "", }, @@ -862,7 +862,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Less than", query: "duration<1000", shouldPass: true, - expectedQuery: "WHERE (attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?)", expectedArgs: []any{float64(1000), true}, expectedErrorContains: "", }, @@ -872,7 +872,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Less than or equal", query: "count<=10", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] <= ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) <= ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(10), true}, expectedErrorContains: "", }, @@ -880,7 +880,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Less than or equal", query: "duration<=1000", shouldPass: true, - expectedQuery: "WHERE (attributes_number['duration'] <= ? AND mapContains(attributes_number, 'duration') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) <= ? AND mapContains(attributes_number, 'duration') = ?)", expectedArgs: []any{float64(1000), true}, expectedErrorContains: "", }, @@ -890,7 +890,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Greater than", query: "count>10", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(10), true}, expectedErrorContains: "", }, @@ -898,7 +898,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Greater than", query: "duration>1000", shouldPass: true, - expectedQuery: "WHERE (attributes_number['duration'] > ? AND mapContains(attributes_number, 'duration') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?)", expectedArgs: []any{float64(1000), true}, expectedErrorContains: "", }, @@ -908,7 +908,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Greater than or equal", query: "count>=10", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] >= ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) >= ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(10), true}, expectedErrorContains: "", }, @@ -916,7 +916,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Greater than or equal", query: "duration>=1000", shouldPass: true, - expectedQuery: "WHERE (attributes_number['duration'] >= ? AND mapContains(attributes_number, 'duration') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) >= ? AND mapContains(attributes_number, 'duration') = ?)", expectedArgs: []any{float64(1000), true}, expectedErrorContains: "", }, @@ -1062,7 +1062,7 @@ func TestFilterExprLogs(t *testing.T) { category: "BETWEEN operator", query: "count BETWEEN 1 AND 10", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] BETWEEN ? AND ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) BETWEEN ? AND ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(1), float64(10), true}, expectedErrorContains: "", }, @@ -1070,7 +1070,7 @@ func TestFilterExprLogs(t *testing.T) { category: "BETWEEN operator", query: "duration BETWEEN 100 AND 1000", shouldPass: true, - expectedQuery: "WHERE (attributes_number['duration'] BETWEEN ? AND ? AND mapContains(attributes_number, 'duration') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration') = ?)", expectedArgs: []any{float64(100), float64(1000), true}, expectedErrorContains: "", }, @@ -1078,7 +1078,7 @@ func TestFilterExprLogs(t *testing.T) { category: "BETWEEN operator", query: "amount BETWEEN 0.1 AND 9.9", shouldPass: true, - expectedQuery: "WHERE (attributes_number['amount'] BETWEEN ? AND ? AND mapContains(attributes_number, 'amount') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['amount']) BETWEEN ? AND ? AND mapContains(attributes_number, 'amount') = ?)", expectedArgs: []any{0.1, 9.9, true}, expectedErrorContains: "", }, @@ -1088,7 +1088,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT BETWEEN operator", query: "count NOT BETWEEN 1 AND 10", shouldPass: true, - expectedQuery: "WHERE attributes_number['count'] NOT BETWEEN ? AND ?", + expectedQuery: "WHERE toFloat64(attributes_number['count']) NOT BETWEEN ? AND ?", expectedArgs: []any{float64(1), float64(10)}, expectedErrorContains: "", }, @@ -1096,7 +1096,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT BETWEEN operator", query: "duration NOT BETWEEN 100 AND 1000", shouldPass: true, - expectedQuery: "WHERE attributes_number['duration'] NOT BETWEEN ? AND ?", + expectedQuery: "WHERE toFloat64(attributes_number['duration']) NOT BETWEEN ? AND ?", expectedArgs: []any{float64(100), float64(1000)}, expectedErrorContains: "", }, @@ -1104,7 +1104,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT BETWEEN operator", query: "amount NOT BETWEEN 0.1 AND 9.9", shouldPass: true, - expectedQuery: "WHERE attributes_number['amount'] NOT BETWEEN ? AND ?", + expectedQuery: "WHERE toFloat64(attributes_number['amount']) NOT BETWEEN ? AND ?", expectedArgs: []any{0.1, 9.9}, expectedErrorContains: "", }, @@ -1114,7 +1114,7 @@ func TestFilterExprLogs(t *testing.T) { category: "IN operator (parentheses)", query: "status IN (200, 201, 202)", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? OR attributes_number['status'] = ? OR attributes_number['status'] = ?) AND mapContains(attributes_number, 'status') = ?)", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status') = ?)", expectedArgs: []any{float64(200), float64(201), float64(202), true}, expectedErrorContains: "", }, @@ -1122,7 +1122,7 @@ func TestFilterExprLogs(t *testing.T) { category: "IN operator (parentheses)", query: "error.code IN (404, 500, 503)", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['error.code'] = ? OR attributes_number['error.code'] = ? OR attributes_number['error.code'] = ?) AND mapContains(attributes_number, 'error.code') = ?)", + expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code') = ?)", expectedArgs: []any{float64(404), float64(500), float64(503), true}, expectedErrorContains: "", }, @@ -1148,7 +1148,7 @@ func TestFilterExprLogs(t *testing.T) { category: "IN operator (brackets)", query: "status IN [200, 201, 202]", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? OR attributes_number['status'] = ? OR attributes_number['status'] = ?) AND mapContains(attributes_number, 'status') = ?)", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status') = ?)", expectedArgs: []any{float64(200), float64(201), float64(202), true}, expectedErrorContains: "", }, @@ -1156,7 +1156,7 @@ func TestFilterExprLogs(t *testing.T) { category: "IN operator (brackets)", query: "error.code IN [404, 500, 503]", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['error.code'] = ? OR attributes_number['error.code'] = ? OR attributes_number['error.code'] = ?) AND mapContains(attributes_number, 'error.code') = ?)", + expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code') = ?)", expectedArgs: []any{float64(404), float64(500), float64(503), true}, expectedErrorContains: "", }, @@ -1182,7 +1182,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT IN operator (parentheses)", query: "status NOT IN (400, 500)", shouldPass: true, - expectedQuery: "WHERE (attributes_number['status'] <> ? AND attributes_number['status'] <> ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['status']) <> ? AND toFloat64(attributes_number['status']) <> ?)", expectedArgs: []any{float64(400), float64(500)}, expectedErrorContains: "", }, @@ -1190,7 +1190,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT IN operator (parentheses)", query: "error.code NOT IN (401, 403)", shouldPass: true, - expectedQuery: "WHERE (attributes_number['error.code'] <> ? AND attributes_number['error.code'] <> ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['error.code']) <> ? AND toFloat64(attributes_number['error.code']) <> ?)", expectedArgs: []any{float64(401), float64(403)}, expectedErrorContains: "", }, @@ -1216,7 +1216,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT IN operator (brackets)", query: "status NOT IN [400, 500]", shouldPass: true, - expectedQuery: "WHERE (attributes_number['status'] <> ? AND attributes_number['status'] <> ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['status']) <> ? AND toFloat64(attributes_number['status']) <> ?)", expectedArgs: []any{float64(400), float64(500)}, expectedErrorContains: "", }, @@ -1224,7 +1224,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT IN operator (brackets)", query: "error.code NOT IN [401, 403]", shouldPass: true, - expectedQuery: "WHERE (attributes_number['error.code'] <> ? AND attributes_number['error.code'] <> ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['error.code']) <> ? AND toFloat64(attributes_number['error.code']) <> ?)", expectedArgs: []any{float64(401), float64(403)}, expectedErrorContains: "", }, @@ -1335,7 +1335,7 @@ func TestFilterExprLogs(t *testing.T) { query: "email REGEXP \"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}$\"", shouldPass: true, expectedQuery: "WHERE (match(attributes_string['email'], ?) AND mapContains(attributes_string, 'email') = ?)", - expectedArgs: []any{"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}$", true}, + expectedArgs: []any{"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", true}, expectedErrorContains: "", }, { @@ -1343,7 +1343,7 @@ func TestFilterExprLogs(t *testing.T) { query: "version REGEXP \"^v\\\\d+\\\\.\\\\d+\\\\.\\\\d+$\"", shouldPass: true, expectedQuery: "WHERE (match(attributes_string['version'], ?) AND mapContains(attributes_string, 'version') = ?)", - expectedArgs: []any{"^v\\\\d+\\\\.\\\\d+\\\\.\\\\d+$", true}, + expectedArgs: []any{"^v\\d+\\.\\d+\\.\\d+$", true}, expectedErrorContains: "", }, { @@ -1351,7 +1351,7 @@ func TestFilterExprLogs(t *testing.T) { query: "path REGEXP \"^/api/v\\\\d+/users/\\\\d+$\"", shouldPass: true, expectedQuery: "WHERE (match(attributes_string['path'], ?) AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"^/api/v\\\\d+/users/\\\\d+$", true}, + expectedArgs: []any{"^/api/v\\d+/users/\\d+$", true}, expectedErrorContains: "", }, { @@ -1377,7 +1377,7 @@ func TestFilterExprLogs(t *testing.T) { query: "email NOT REGEXP \"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}$\"", shouldPass: true, expectedQuery: "WHERE NOT match(attributes_string['email'], ?)", - expectedArgs: []any{"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}$"}, + expectedArgs: []any{"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"}, expectedErrorContains: "", }, { @@ -1385,15 +1385,15 @@ func TestFilterExprLogs(t *testing.T) { query: "version NOT REGEXP \"^v\\\\d+\\\\.\\\\d+\\\\.\\\\d+$\"", shouldPass: true, expectedQuery: "WHERE NOT match(attributes_string['version'], ?)", - expectedArgs: []any{"^v\\\\d+\\\\.\\\\d+\\\\.\\\\d+$"}, + expectedArgs: []any{"^v\\d+\\.\\d+\\.\\d+$"}, expectedErrorContains: "", }, { category: "NOT REGEXP operator", - query: "path NOT REGEXP \"^/api/v\\\\d+/users/\\\\d+$\"", + query: "path NOT REGEXP \"^/api/v\\d+/users/\\d+$\"", shouldPass: true, expectedQuery: "WHERE NOT match(attributes_string['path'], ?)", - expectedArgs: []any{"^/api/v\\\\d+/users/\\\\d+$"}, + expectedArgs: []any{"^/api/v\\d+/users/\\d+$"}, expectedErrorContains: "", }, @@ -1464,7 +1464,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit AND", query: "status=200 AND service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", expectedArgs: []any{float64(200), true, "api", true}, expectedErrorContains: "", }, @@ -1472,7 +1472,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit AND", query: "count>0 AND duration<1000", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))", expectedArgs: []any{float64(0), true, float64(1000), true}, expectedErrorContains: "", }, @@ -1490,7 +1490,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit OR", query: "status=200 OR status=201", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) OR (attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", expectedArgs: []any{float64(200), true, float64(201), true}, expectedErrorContains: "", }, @@ -1506,7 +1506,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit OR", query: "count<10 OR count>100", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['count'] < ? AND mapContains(attributes_number, 'count') = ?) OR (attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", expectedArgs: []any{float64(10), true, float64(100), true}, expectedErrorContains: "", }, @@ -1516,7 +1516,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT with expressions", query: "NOT status=200", shouldPass: true, - expectedQuery: "WHERE NOT ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?))", + expectedQuery: "WHERE NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", expectedArgs: []any{float64(200), true}, expectedErrorContains: "", }, @@ -1532,7 +1532,7 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT with expressions", query: "NOT count>10", shouldPass: true, - expectedQuery: "WHERE NOT ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?))", + expectedQuery: "WHERE NOT ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", expectedArgs: []any{float64(10), true}, expectedErrorContains: "", }, @@ -1542,7 +1542,7 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + OR combinations", query: "status=200 AND (service.name=\"api\" OR service.name=\"web\")", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) OR (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) OR (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))))", expectedArgs: []any{float64(200), true, "api", true, "web", true}, expectedErrorContains: "", }, @@ -1550,7 +1550,7 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + OR combinations", query: "(count>10 AND count<100) OR (duration>1000 AND duration<5000)", shouldPass: true, - expectedQuery: "WHERE ((((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_number['count'] < ? AND mapContains(attributes_number, 'count') = ?))) OR (((attributes_number['duration'] > ? AND mapContains(attributes_number, 'duration') = ?) AND (attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?))))", + expectedQuery: "WHERE ((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?))) OR (((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))))", expectedArgs: []any{float64(10), true, float64(100), true, float64(1000), true, float64(5000), true}, expectedErrorContains: "", }, @@ -1568,7 +1568,7 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + NOT combinations", query: "status=200 AND NOT service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND NOT ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND NOT ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))", expectedArgs: []any{float64(200), true, "api", true}, expectedErrorContains: "", }, @@ -1576,7 +1576,7 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + NOT combinations", query: "count>0 AND NOT error.code EXISTS", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND NOT (mapContains(attributes_number, 'error.code') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND NOT (mapContains(attributes_number, 'error.code') = ?))", expectedArgs: []any{float64(0), true, true}, expectedErrorContains: "", }, @@ -1586,7 +1586,7 @@ func TestFilterExprLogs(t *testing.T) { category: "OR + NOT combinations", query: "NOT status=200 OR NOT service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE (NOT ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)) OR NOT ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))", + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)) OR NOT ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))", expectedArgs: []any{float64(200), true, "api", true}, expectedErrorContains: "", }, @@ -1594,7 +1594,7 @@ func TestFilterExprLogs(t *testing.T) { category: "OR + NOT combinations", query: "NOT count>0 OR NOT error.code EXISTS", shouldPass: true, - expectedQuery: "WHERE (NOT ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?)) OR NOT (mapContains(attributes_number, 'error.code') = ?))", + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?)) OR NOT (mapContains(attributes_number, 'error.code') = ?))", expectedArgs: []any{float64(0), true, true}, expectedErrorContains: "", }, @@ -1604,7 +1604,7 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + OR + NOT combinations", query: "status=200 AND (service.name=\"api\" OR NOT duration>1000)", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) OR NOT ((attributes_number['duration'] > ? AND mapContains(attributes_number, 'duration') = ?)))))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) OR NOT ((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?)))))", expectedArgs: []any{float64(200), true, "api", true, float64(1000), true}, expectedErrorContains: "", }, @@ -1620,7 +1620,7 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + OR + NOT combinations", query: "NOT (status=200 AND service.name=\"api\") OR count>0", shouldPass: true, - expectedQuery: "WHERE (NOT ((((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))) OR (attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?))", + expectedQuery: "WHERE (NOT ((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", expectedArgs: []any{float64(200), true, "api", true, float64(0), true}, expectedErrorContains: "", }, @@ -1630,7 +1630,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Implicit AND", query: "status=200 service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", expectedArgs: []any{float64(200), true, "api", true}, expectedErrorContains: "", }, @@ -1638,7 +1638,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Implicit AND", query: "count>0 duration<1000", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))", expectedArgs: []any{float64(0), true, float64(1000), true}, expectedErrorContains: "", }, @@ -1656,7 +1656,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Mixed implicit/explicit AND", query: "status=200 AND service.name=\"api\" duration<1000", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) AND (attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))", expectedArgs: []any{float64(200), true, "api", true, float64(1000), true}, expectedErrorContains: "", }, @@ -1664,7 +1664,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Mixed implicit/explicit AND", query: "count>0 level=\"ERROR\" AND message CONTAINS \"error\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) AND (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) AND (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))", expectedArgs: []any{float64(0), true, "ERROR", true, "%error%", true}, expectedErrorContains: "", }, @@ -1674,7 +1674,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Simple grouping", query: "(status=200)", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", expectedArgs: []any{float64(200), true}, expectedErrorContains: "", }, @@ -1690,7 +1690,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Simple grouping", query: "(count>0)", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", expectedArgs: []any{float64(0), true}, expectedErrorContains: "", }, @@ -1700,7 +1700,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Nested grouping", query: "((status=200))", shouldPass: true, - expectedQuery: "WHERE (((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)))", + expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)))", expectedArgs: []any{float64(200), true}, expectedErrorContains: "", }, @@ -1716,7 +1716,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Nested grouping", query: "((count>0) AND (duration<1000))", shouldPass: true, - expectedQuery: "WHERE ((((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?)) AND ((attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?))))", + expectedQuery: "WHERE ((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?)) AND ((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))))", expectedArgs: []any{float64(0), true, float64(1000), true}, expectedErrorContains: "", }, @@ -1726,7 +1726,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Complex nested grouping", query: "(status=200 AND (service.name=\"api\" OR service.name=\"web\"))", shouldPass: true, - expectedQuery: "WHERE (((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) OR (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))))", + expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) OR (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))))", expectedArgs: []any{float64(200), true, "api", true, "web", true}, expectedErrorContains: "", }, @@ -1734,7 +1734,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Complex nested grouping", query: "((count>0 AND count<100) OR (duration>1000 AND duration<5000))", shouldPass: true, - expectedQuery: "WHERE (((((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_number['count'] < ? AND mapContains(attributes_number, 'count') = ?))) OR (((attributes_number['duration'] > ? AND mapContains(attributes_number, 'duration') = ?) AND (attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?)))))", + expectedQuery: "WHERE (((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?))) OR (((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?)))))", expectedArgs: []any{float64(0), true, float64(100), true, float64(1000), true, float64(5000), true}, expectedErrorContains: "", }, @@ -1752,7 +1752,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Deep nesting", query: "(((status=200 OR status=201) AND service.name=\"api\") OR ((status=202 OR status=203) AND service.name=\"web\"))", shouldPass: true, - expectedQuery: "WHERE (((((((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) OR (attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?))) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))) OR (((((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) OR (attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?))) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))))", + expectedQuery: "WHERE (((((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))) OR (((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))))", expectedArgs: []any{float64(200), true, float64(201), true, "api", true, float64(202), true, float64(203), true, "web", true}, expectedErrorContains: "", }, @@ -1760,7 +1760,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Deep nesting", query: "(count>0 AND ((duration<1000 AND service.name=\"api\") OR (duration<500 AND service.name=\"web\")))", shouldPass: true, - expectedQuery: "WHERE (((attributes_number['count'] > ? AND mapContains(attributes_number, 'count') = ?) AND (((((attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))) OR (((attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))))))", + expectedQuery: "WHERE (((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))) OR (((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))))))", expectedArgs: []any{float64(0), true, float64(1000), true, "api", true, float64(500), true, "web", true}, expectedErrorContains: "", }, @@ -1795,7 +1795,7 @@ func TestFilterExprLogs(t *testing.T) { query: "message='This is a \\'quoted\\' message'", shouldPass: true, expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"This is a \\'quoted\\' message", true}, + expectedArgs: []any{"This is a 'quoted' message", true}, expectedErrorContains: "", }, @@ -1804,7 +1804,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Numeric values", query: "status=200", shouldPass: true, - expectedQuery: "WHERE (attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)", expectedArgs: []any{float64(200), true}, expectedErrorContains: "", }, @@ -1812,7 +1812,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Numeric values", query: "count=0", shouldPass: true, - expectedQuery: "WHERE (attributes_number['count'] = ? AND mapContains(attributes_number, 'count') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['count']) = ? AND mapContains(attributes_number, 'count') = ?)", expectedArgs: []any{float64(0), true}, expectedErrorContains: "", }, @@ -1820,7 +1820,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Numeric values", query: "duration=1000.5", shouldPass: true, - expectedQuery: "WHERE (attributes_number['duration'] = ? AND mapContains(attributes_number, 'duration') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) = ? AND mapContains(attributes_number, 'duration') = ?)", expectedArgs: []any{float64(1000.5), true}, expectedErrorContains: "", }, @@ -1828,7 +1828,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Numeric values", query: "amount=-10.25", shouldPass: true, - expectedQuery: "WHERE (attributes_number['amount'] = ? AND mapContains(attributes_number, 'amount') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['amount']) = ? AND mapContains(attributes_number, 'amount') = ?)", expectedArgs: []any{float64(-10.25), true}, expectedErrorContains: "", }, @@ -1915,7 +1915,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Nested object paths", query: "metadata.dimensions.width>1000", shouldPass: true, - expectedQuery: "WHERE (attributes_number['metadata.dimensions.width'] > ? AND mapContains(attributes_number, 'metadata.dimensions.width') = ?)", + expectedQuery: "WHERE (toFloat64(attributes_number['metadata.dimensions.width']) > ? AND mapContains(attributes_number, 'metadata.dimensions.width') = ?)", expectedArgs: []any{float64(1000), true}, expectedErrorContains: "", }, @@ -1938,28 +1938,28 @@ func TestFilterExprLogs(t *testing.T) { category: "Operator precedence", query: "NOT status=200 AND service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE (NOT ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", expectedArgs: []any{float64(200), true, "api", true}, // Should be (NOT status=200) AND service.name="api" }, { category: "Operator precedence", query: "status=200 AND service.name=\"api\" OR service.name=\"web\"", shouldPass: true, - expectedQuery: "WHERE (((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)) OR (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", + expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)) OR (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", expectedArgs: []any{float64(200), true, "api", true, "web", true}, // Should be (status=200 AND service.name="api") OR service.name="web" }, { category: "Operator precedence", query: "NOT status=200 OR NOT service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE (NOT ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)) OR NOT ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))", + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)) OR NOT ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?)))", expectedArgs: []any{float64(200), true, "api", true}, // Should be (NOT status=200) OR (NOT service.name="api") }, { category: "Operator precedence", query: "status=200 OR service.name=\"api\" AND level=\"ERROR\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) OR ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?)))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?)))", expectedArgs: []any{float64(200), true, "api", true, "ERROR", true}, // Should be status=200 OR (service.name="api" AND level="ERROR") }, @@ -1984,7 +1984,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Whitespace patterns", query: "status=200 AND service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", expectedArgs: []any{float64(200), true, "api", true}, // Multiple spaces }, @@ -2137,7 +2137,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Common filters", query: "(first_name LIKE \"John%\" OR last_name LIKE \"Smith%\") AND age>=18", shouldPass: true, - expectedQuery: "WHERE ((((attributes_string['first_name'] LIKE ? AND mapContains(attributes_string, 'first_name') = ?) OR (attributes_string['last_name'] LIKE ? AND mapContains(attributes_string, 'last_name') = ?))) AND (attributes_number['age'] >= ? AND mapContains(attributes_number, 'age') = ?))", + expectedQuery: "WHERE ((((attributes_string['first_name'] LIKE ? AND mapContains(attributes_string, 'first_name') = ?) OR (attributes_string['last_name'] LIKE ? AND mapContains(attributes_string, 'last_name') = ?))) AND (toFloat64(attributes_number['age']) >= ? AND mapContains(attributes_number, 'age') = ?))", expectedArgs: []any{"John%", true, "Smith%", true, float64(18), true}, }, { @@ -2154,7 +2154,7 @@ func TestFilterExprLogs(t *testing.T) { category: "More common filters", query: "service.name=\"api\" AND (status>=500 OR duration>1000) AND NOT message CONTAINS \"expected\"", shouldPass: true, - expectedQuery: "WHERE ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) AND (((attributes_number['status'] >= ? AND mapContains(attributes_number, 'status') = ?) OR (attributes_number['duration'] > ? AND mapContains(attributes_number, 'duration') = ?))) AND NOT ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)))", + expectedQuery: "WHERE ((resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?) AND (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?))) AND NOT ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)))", expectedArgs: []any{"api", true, float64(500), true, float64(1000), true, "%expected%", true}, }, @@ -2205,14 +2205,14 @@ func TestFilterExprLogs(t *testing.T) { query: "path=\"C:\\\\Program Files\\\\Application\"", shouldPass: true, expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"C:\\\\Program Files\\\\Application", true}, + expectedArgs: []any{"C:\\Program Files\\Application", true}, }, { category: "Escaped values", query: "path=\"^prefix\\\\.suffix$\\\\d+\\\\w+\"", shouldPass: true, expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"^prefix\\\\.suffix$\\\\d+\\\\w+", true}, + expectedArgs: []any{"^prefix\\.suffix$\\d+\\w+", true}, }, // Inconsistent/unusual whitespace @@ -2220,7 +2220,7 @@ func TestFilterExprLogs(t *testing.T) { category: "Unusual whitespace", query: "status = 200 AND service.name = \"api\"", shouldPass: true, - expectedQuery: "WHERE ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (resources_string['service.name'] = ? AND mapContains(resources_string, 'service.name') = ?))", expectedArgs: []any{float64(200), true, "api", true}, }, { @@ -2281,7 +2281,7 @@ func TestFilterExprLogs(t *testing.T) { ) `, shouldPass: true, - expectedQuery: "WHERE ((((((((attributes_number['status'] >= ? AND mapContains(attributes_number, 'status') = ?) AND (attributes_number['status'] < ? AND mapContains(attributes_number, 'status') = ?))) OR (((attributes_number['status'] >= ? AND mapContains(attributes_number, 'status') = ?) AND (attributes_number['status'] < ? AND mapContains(attributes_number, 'status') = ?) AND NOT ((attributes_number['status'] = ? AND mapContains(attributes_number, 'status') = ?)))))) AND ((((resources_string['service.name'] = ? OR resources_string['service.name'] = ? OR resources_string['service.name'] = ?) AND mapContains(resources_string, 'service.name') = ?) OR (((resources_string['service.type'] = ? AND mapContains(resources_string, 'service.type') = ?) AND NOT ((resources_string['service.deprecated'] = ? AND mapContains(resources_string, 'service.deprecated') = ?)))))))) AND (((((attributes_number['duration'] < ? AND mapContains(attributes_number, 'duration') = ?) OR ((attributes_number['duration'] BETWEEN ? AND ? AND mapContains(attributes_number, 'duration') = ?)))) AND ((resources_string['environment'] <> ? OR (((resources_string['environment'] = ? AND mapContains(resources_string, 'environment') = ?) AND (attributes_bool['is_automated_test'] = ? AND mapContains(attributes_bool, 'is_automated_test') = ?))))))) AND NOT ((((((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))) AND (attributes_string['severity'] = ? AND mapContains(attributes_string, 'severity') = ?)))))", + expectedQuery: "WHERE ((((((((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status') = ?) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status') = ?))) OR (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status') = ?) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status') = ?) AND NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)))))) AND ((((resources_string['service.name'] = ? OR resources_string['service.name'] = ? OR resources_string['service.name'] = ?) AND mapContains(resources_string, 'service.name') = ?) OR (((resources_string['service.type'] = ? AND mapContains(resources_string, 'service.type') = ?) AND NOT ((resources_string['service.deprecated'] = ? AND mapContains(resources_string, 'service.deprecated') = ?)))))))) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?) OR ((toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration') = ?)))) AND ((resources_string['environment'] <> ? OR (((resources_string['environment'] = ? AND mapContains(resources_string, 'environment') = ?) AND (attributes_bool['is_automated_test'] = ? AND mapContains(attributes_bool, 'is_automated_test') = ?))))))) AND NOT ((((((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))) AND (attributes_string['severity'] = ? AND mapContains(attributes_string, 'severity') = ?)))))", expectedArgs: []any{ float64(200), true, float64(300), true, float64(400), true, float64(500), true, float64(404), true, "api", "web", "auth", true, diff --git a/pkg/telemetrylogs/statement_builder.go b/pkg/telemetrylogs/statement_builder.go index 0ea14b154060..a4a55649c3d2 100644 --- a/pkg/telemetrylogs/statement_builder.go +++ b/pkg/telemetrylogs/statement_builder.go @@ -73,6 +73,8 @@ func (b *logQueryStatementBuilder) Build( return nil, err } + b.adjustKeys(ctx, keys, query) + // Create SQL builder q := sqlbuilder.NewSelectBuilder() @@ -124,6 +126,77 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) [] return keySelectors } +func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) { + // for group by / order by, if there is a key + // that exactly matches the name of intrinsic field but has + // a field context or data type that doesn't match the field context or data type of the + // intrinsic field, + // and there is no additional key present in the data with the incoming key match, + // then override the given context with + // intrinsic field context and data type + // Why does that happen? Because we have a lot of dashboards created by users and shared over web + // that has incorrect context or data type populated so we fix it + // note: this override happens only when there is no match; if there is a match, + // we can't make decision on behalf of users so we let it use unmodified + + // example: {"key": "severity_text","type": "tag","dataType": "string"} + // This is sent as "tag", when it's not, this was earlier managed with + // `isColumn`, which we don't have in v5 (because it's not a user concern whether it's mat col or not) + // Such requests as-is look for attributes, the following code exists to handle them + checkMatch := func(k *telemetrytypes.TelemetryFieldKey) { + var overallMatch bool + + findMatch := func(staticKeys map[string]telemetrytypes.TelemetryFieldKey) bool { + // for a given key `k`, iterate over the metadata keys `keys` + // and see if there is any exact match + match := false + for _, mapKey := range keys[k.Name] { + if mapKey.FieldContext == k.FieldContext && mapKey.FieldDataType == k.FieldDataType { + match = true + } + } + // we don't have exact match, then it's doesn't exist in attribute or resource attribute + // use the intrinsic/calculated field + if !match { + b.logger.InfoContext(ctx, "overriding the field context and data type", "key", k.Name) + k.FieldContext = staticKeys[k.Name].FieldContext + k.FieldDataType = staticKeys[k.Name].FieldDataType + } + return match + } + + if _, ok := IntrinsicFields[k.Name]; ok { + overallMatch = overallMatch || findMatch(IntrinsicFields) + } + + if !overallMatch { + // check if all the key for the given field have been materialized, if so + // set the key to materialized + materilized := true + for _, key := range keys[k.Name] { + materilized = materilized && key.Materialized + } + k.Materialized = materilized + } + } + + for idx := range query.GroupBy { + checkMatch(&query.GroupBy[idx].TelemetryFieldKey) + } + for idx := range query.Order { + checkMatch(&query.Order[idx].Key.TelemetryFieldKey) + } + + keys["id"] = []*telemetrytypes.TelemetryFieldKey{ + { + Name: "id", + Signal: telemetrytypes.SignalLogs, + FieldContext: telemetrytypes.FieldContextLog, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + } +} + // buildListQuery builds a query for list panel type func (b *logQueryStatementBuilder) buildListQuery( ctx context.Context, @@ -229,7 +302,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( } colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.TelemetryFieldKey.Name) allGroupByArgs = append(allGroupByArgs, args...) - sb.SelectMore(sqlbuilder.Escape(colExpr)) + sb.SelectMore(colExpr) fieldNames = append(fieldNames, fmt.Sprintf("`%s`", gb.TelemetryFieldKey.Name)) } @@ -349,7 +422,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery( } colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.TelemetryFieldKey.Name) allGroupByArgs = append(allGroupByArgs, args...) - sb.SelectMore(sqlbuilder.Escape(colExpr)) + sb.SelectMore(colExpr) } // for scalar queries, the rate would be end-start diff --git a/pkg/telemetrylogs/stmt_builder_test.go b/pkg/telemetrylogs/stmt_builder_test.go index 26e38dac7469..c12170f99c73 100644 --- a/pkg/telemetrylogs/stmt_builder_test.go +++ b/pkg/telemetrylogs/stmt_builder_test.go @@ -36,7 +36,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation ) } -func TestStatementBuilder(t *testing.T) { +func TestStatementBuilderTimeSeries(t *testing.T) { cases := []struct { name string requestType qbtypes.RequestType @@ -45,7 +45,7 @@ func TestStatementBuilder(t *testing.T) { expectedErr error }{ { - name: "test", + name: "Time series with limit", requestType: qbtypes.RequestTypeTimeSeries, query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ Signal: telemetrytypes.SignalLogs, @@ -68,13 +68,13 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", Args: []any{"cartservice", "%service.name%", "%service.name%cartservice%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, { - name: "test", + name: "Time series with limit + custom order by", requestType: qbtypes.RequestTypeTimeSeries, query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ Signal: telemetrytypes.SignalLogs, @@ -107,7 +107,7 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", Args: []any{"cartservice", "%service.name%", "%service.name%cartservice%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, @@ -152,3 +152,96 @@ func TestStatementBuilder(t *testing.T) { }) } } + +func TestStatementBuilderListQuery(t *testing.T) { + cases := []struct { + name string + requestType qbtypes.RequestType + query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation] + expected qbtypes.Statement + expectedErr error + }{ + { + name: "default list", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'cartservice'", + }, + Limit: 10, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"cartservice", "%service.name%", "%service.name%cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + { + name: "list query with mat col order by", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'cartservice'", + }, + Limit: 10, + Order: []qbtypes.OrderBy{ + { + Key: qbtypes.OrderByKey{ + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "materialized.key.name", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, + Direction: qbtypes.OrderDirectionDesc, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?", + Args: []any{"cartservice", "%service.name%", "%service.name%cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + } + + fm := NewFieldMapper() + cb := NewConditionBuilder(fm) + mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() + + aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + + resourceFilterStmtBuilder := resourceFilterStmtBuilder() + + statementBuilder := NewLogQueryStatementBuilder( + instrumentationtest.New().ToProviderSettings(), + mockMetadataStore, + fm, + cb, + resourceFilterStmtBuilder, + aggExprRewriter, + DefaultFullTextColumn, + BodyJSONStringSearchPrefix, + GetBodyJSONKey, + ) + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil) + + if c.expectedErr != nil { + require.Error(t, err) + require.Contains(t, err.Error(), c.expectedErr.Error()) + } else { + require.NoError(t, err) + require.Equal(t, c.expected.Query, q.Query) + require.Equal(t, c.expected.Args, q.Args) + require.Equal(t, c.expected.Warnings, q.Warnings) + } + }) + } +} diff --git a/pkg/telemetrymetadata/field_mapper.go b/pkg/telemetrymetadata/field_mapper.go index b1eb5ad49830..a7564cbe7c58 100644 --- a/pkg/telemetrymetadata/field_mapper.go +++ b/pkg/telemetrymetadata/field_mapper.go @@ -9,6 +9,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" "golang.org/x/exp/maps" ) @@ -95,7 +96,7 @@ func (m *fieldMapper) ColumnExpressionFor( return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, correction) } else { // not even a close match, return an error - return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field %s not found", field.Name) + return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name) } } } else if len(keysForField) == 1 { @@ -112,5 +113,5 @@ func (m *fieldMapper) ColumnExpressionFor( } } - return fmt.Sprintf("%s AS `%s`", colName, field.Name), nil + return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil } diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index 7375a96b4a90..4fd86e29f146 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -159,7 +159,12 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector dataTypes = append(dataTypes, fieldKeySelector.FieldDataType) } // now look at the field context - if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified { + // we don't write most of intrinsic fields to tag attributes table + // for this reason we don't want to apply tag_type if the field context + // if not attribute or resource attribute + if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified && + (fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute || + fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) { fieldKeyConds = append(fieldKeyConds, sb.E("tag_type", fieldKeySelector.FieldContext.TagType())) } @@ -349,7 +354,12 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors } // now look at the field context - if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified { + // we don't write most of intrinsic fields to tag attributes table + // for this reason we don't want to apply tag_type if the field context + // if not attribute or resource attribute + if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified && + (fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute || + fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) { fieldKeyConds = append(fieldKeyConds, sb.E("tag_type", fieldKeySelector.FieldContext.TagType())) } diff --git a/pkg/telemetrymetadata/metadata_test.go b/pkg/telemetrymetadata/metadata_test.go index e02eb61fa4df..078f74891529 100644 --- a/pkg/telemetrymetadata/metadata_test.go +++ b/pkg/telemetrymetadata/metadata_test.go @@ -60,7 +60,7 @@ func TestGetKeys(t *testing.T) { query := `SELECT.*` mock.ExpectQuery(query). - WithArgs("%http.method%", telemetrytypes.FieldContextSpan.TagType(), telemetrytypes.FieldDataTypeString.TagDataType(), 10). + WithArgs("%http.method%", telemetrytypes.FieldDataTypeString.TagDataType(), 10). WillReturnRows(cmock.NewRows([]cmock.ColumnType{ {Name: "tag_key", Type: "String"}, {Name: "tag_type", Type: "String"}, diff --git a/pkg/telemetrymetrics/condition_builder.go b/pkg/telemetrymetrics/condition_builder.go index 28074c7c8c4a..bacb90705ff5 100644 --- a/pkg/telemetrymetrics/condition_builder.go +++ b/pkg/telemetrymetrics/condition_builder.go @@ -93,23 +93,13 @@ func (c *conditionBuilder) conditionFor( if !ok { return "", qbtypes.ErrInValues } - // instead of using IN, we use `=` + `OR` to make use of index - conditions := []string{} - for _, value := range values { - conditions = append(conditions, sb.E(tblFieldName, value)) - } - return sb.Or(conditions...), nil + return sb.In(tblFieldName, values), nil case qbtypes.FilterOperatorNotIn: values, ok := value.([]any) if !ok { return "", qbtypes.ErrInValues } - // instead of using NOT IN, we use `!=` + `AND` to make use of index - conditions := []string{} - for _, value := range values { - conditions = append(conditions, sb.NE(tblFieldName, value)) - } - return sb.And(conditions...), nil + return sb.NotIn(tblFieldName, values), nil // exists and not exists // in the UI based query builder, `exists` and `not exists` are used for diff --git a/pkg/telemetrymetrics/condition_builder_test.go b/pkg/telemetrymetrics/condition_builder_test.go index 9d6ae2e53eee..b15ebb28914e 100644 --- a/pkg/telemetrymetrics/condition_builder_test.go +++ b/pkg/telemetrymetrics/condition_builder_test.go @@ -118,8 +118,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorIn, value: []any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"}, - expectedSQL: "(metric_name = ? OR metric_name = ? OR metric_name = ?)", - expectedArgs: []any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"}, + expectedSQL: "metric_name IN (?)", + expectedArgs: []any{[]any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"}}, expectedError: nil, }, { @@ -141,8 +141,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorNotIn, value: []any{"debug", "info", "trace"}, - expectedSQL: "(metric_name <> ? AND metric_name <> ? AND metric_name <> ?)", - expectedArgs: []any{"debug", "info", "trace"}, + expectedSQL: "metric_name NOT IN (?)", + expectedArgs: []any{[]any{"debug", "info", "trace"}}, expectedError: nil, }, { diff --git a/pkg/telemetrymetrics/field_mapper.go b/pkg/telemetrymetrics/field_mapper.go index b80706da4850..b13dbcd327dd 100644 --- a/pkg/telemetrymetrics/field_mapper.go +++ b/pkg/telemetrymetrics/field_mapper.go @@ -8,6 +8,7 @@ import ( schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" ) var ( @@ -100,5 +101,5 @@ func (m *fieldMapper) ColumnExpressionFor( return "", err } - return fmt.Sprintf("%s AS `%s`", colName, field.Name), nil + return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil } diff --git a/pkg/telemetrymetrics/statement_builder.go b/pkg/telemetrymetrics/statement_builder.go index b1228160e49c..e8ffeabb3a5c 100644 --- a/pkg/telemetrymetrics/statement_builder.go +++ b/pkg/telemetrymetrics/statement_builder.go @@ -11,6 +11,7 @@ import ( qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/huandu/go-sqlbuilder" + "golang.org/x/exp/slices" ) const ( @@ -84,6 +85,8 @@ func (b *metricQueryStatementBuilder) Build( return nil, err } + start, end = querybuilder.AdjustedMetricTimeRange(start, end, uint64(query.StepInterval.Seconds()), query) + return b.buildPipelineStatement(ctx, start, end, query, keys, variables) } @@ -149,7 +152,7 @@ func (b *metricQueryStatementBuilder) buildPipelineStatement( origSpaceAgg := query.Aggregations[0].SpaceAggregation origTimeAgg := query.Aggregations[0].TimeAggregation - origGroupBy := query.GroupBy + origGroupBy := slices.Clone(query.GroupBy) if query.Aggregations[0].SpaceAggregation.IsPercentile() && query.Aggregations[0].Type != metrictypes.ExpHistogramType { @@ -162,8 +165,20 @@ func (b *metricQueryStatementBuilder) buildPipelineStatement( } } - // we need to add le in the group by if it doesn't exist - if !leExists { + if leExists { + // if the user themselves adds `le`, then we remove it from the original group by + // this is to avoid preparing a query that returns `nan`s, see following query + // SELECT + // ts, + // le, + // histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.99) AS value + // FROM __spatial_aggregation_cte + // GROUP BY + // le, + // ts + + origGroupBy = slices.DeleteFunc(origGroupBy, func(k qbtypes.GroupByKey) bool { return k.Name == "le" }) + } else { query.GroupBy = append(query.GroupBy, qbtypes.GroupByKey{ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "le"}, }) diff --git a/pkg/telemetrymetrics/stmt_builder_test.go b/pkg/telemetrymetrics/stmt_builder_test.go index 669eea7002c7..2a87facdedba 100644 --- a/pkg/telemetrymetrics/stmt_builder_test.go +++ b/pkg/telemetrymetrics/stmt_builder_test.go @@ -49,8 +49,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947419000))) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947419000))) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte", - Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983448000), "cumulative", false, "cartservice", "signoz_calls_total", uint64(1747947419000), uint64(1747983448000), 0}, + Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947360000))) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947360000))) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte", + Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", false, "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0}, }, expectedErr: nil, }, @@ -83,7 +83,7 @@ func TestStatementBuilder(t *testing.T) { }, expected: qbtypes.Statement{ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte", - Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983448000), "delta", false, "cartservice", "signoz_calls_total", uint64(1747947419000), uint64(1747983448000)}, + Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "delta", false, "cartservice", "signoz_calls_total", uint64(1747947390000), uint64(1747983420000)}, }, expectedErr: nil, }, @@ -115,7 +115,7 @@ func TestStatementBuilder(t *testing.T) { }, expected: qbtypes.Statement{ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts", - Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983448000), "delta", false, "cartservice", "signoz_latency", uint64(1747947419000), uint64(1747983448000)}, + Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", false, "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)}, }, expectedErr: nil, }, @@ -148,7 +148,7 @@ func TestStatementBuilder(t *testing.T) { }, expected: qbtypes.Statement{ Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint, `host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `host.name`) SELECT * FROM __spatial_aggregation_cte", - Args: []any{"system.memory.usage", uint64(1747936800000), uint64(1747983448000), "unspecified", false, "big-data-node-1", "system.memory.usage", uint64(1747947419000), uint64(1747983448000), 0}, + Args: []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", false, "big-data-node-1", "system.memory.usage", uint64(1747947390000), uint64(1747983420000), 0}, }, expectedErr: nil, }, @@ -176,8 +176,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, `le`, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947419000))) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947419000))) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts", - Args: []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983448000), "cumulative", false, "http_server_duration_bucket", uint64(1747947419000), uint64(1747983448000), 0}, + Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, `le`, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947390000))) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDateTime(fromUnixTimestamp64Milli(1747947390000))) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts", + Args: []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983420000), "cumulative", false, "http_server_duration_bucket", uint64(1747947390000), uint64(1747983420000), 0}, }, expectedErr: nil, }, diff --git a/pkg/telemetrytraces/condition_builder.go b/pkg/telemetrytraces/condition_builder.go index 47e38e20f4e1..1256a1d7ee30 100644 --- a/pkg/telemetrytraces/condition_builder.go +++ b/pkg/telemetrytraces/condition_builder.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "slices" + "strconv" "strings" + "time" schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" "github.com/SigNoz/signoz/pkg/errors" @@ -43,7 +45,24 @@ func (c *conditionBuilder) conditionFor( return "", err } - tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName) + // TODO(srikanthccv): maybe extend this to every possible attribute + if key.Name == "duration_nano" || key.Name == "durationNano" { // QoL improvement + if strDuration, ok := value.(string); ok { + duration, err := time.ParseDuration(strDuration) + if err == nil { + value = duration.Nanoseconds() + } else { + duration, err := strconv.ParseFloat(strDuration, 64) + if err == nil { + value = duration + } else { + return "", errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "invalid duration value: %s", strDuration) + } + } + } + } else { + tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName) + } // regular operators switch operator { diff --git a/pkg/telemetrytraces/condition_builder_test.go b/pkg/telemetrytraces/condition_builder_test.go index a990648639b7..b6b45b1a73ba 100644 --- a/pkg/telemetrytraces/condition_builder_test.go +++ b/pkg/telemetrytraces/condition_builder_test.go @@ -44,7 +44,7 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorGreaterThan, value: float64(100), - expectedSQL: "(attributes_number['request.duration'] > ? AND mapContains(attributes_number, 'request.duration') = ?)", + expectedSQL: "(toFloat64(attributes_number['request.duration']) > ? AND mapContains(attributes_number, 'request.duration') = ?)", expectedArgs: []any{float64(100), true}, expectedError: nil, }, @@ -57,7 +57,7 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorLessThan, value: float64(1024), - expectedSQL: "(attributes_number['request.size'] < ? AND mapContains(attributes_number, 'request.size') = ?)", + expectedSQL: "(toFloat64(attributes_number['request.size']) < ? AND mapContains(attributes_number, 'request.size') = ?)", expectedArgs: []any{float64(1024), true}, expectedError: nil, }, diff --git a/pkg/telemetrytraces/const.go b/pkg/telemetrytraces/const.go index 1b8f4c4de1c9..473b255e1d70 100644 --- a/pkg/telemetrytraces/const.go +++ b/pkg/telemetrytraces/const.go @@ -108,7 +108,7 @@ var ( Name: "spanKind", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, - FieldDataType: telemetrytypes.FieldDataTypeNumber, + FieldDataType: telemetrytypes.FieldDataTypeString, }, "durationNano": { Name: "durationNano", @@ -142,7 +142,7 @@ var ( Description: "Derived response status code from the HTTP/RPC status code attributes. Learn more [here](https://signoz.io/docs/traces-management/guides/derived-fields-spans/#response_status_code)", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, - FieldDataType: telemetrytypes.FieldDataTypeNumber, + FieldDataType: telemetrytypes.FieldDataTypeString, }, "external_http_url": { Name: "external_http_url", @@ -205,7 +205,7 @@ var ( Description: "Whether the span is remote. Learn more [here](https://signoz.io/docs/traces-management/guides/derived-fields-spans/#is_remote)", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, - FieldDataType: telemetrytypes.FieldDataTypeBool, + FieldDataType: telemetrytypes.FieldDataTypeString, }, } @@ -214,7 +214,7 @@ var ( Name: "responseStatusCode", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, - FieldDataType: telemetrytypes.FieldDataTypeNumber, + FieldDataType: telemetrytypes.FieldDataTypeString, }, "externalHttpUrl": { Name: "externalHttpUrl", @@ -268,7 +268,61 @@ var ( Name: "isRemote", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, - FieldDataType: telemetrytypes.FieldDataTypeBool, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "serviceName": { + Name: "serviceName", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "httpRoute": { + Name: "httpRoute", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "msgSystem": { + Name: "msgSystem", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "msgOperation": { + Name: "msgOperation", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "dbSystem": { + Name: "dbSystem", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "rpcSystem": { + Name: "rpcSystem", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "rpcService": { + Name: "rpcService", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "rpcMethod": { + Name: "rpcMethod", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + "peerService": { + Name: "peerService", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, }, } SpanSearchScopeRoot = "isroot" diff --git a/pkg/telemetrytraces/field_mapper.go b/pkg/telemetrytraces/field_mapper.go index 90873c55e6ba..0d18bf2514b0 100644 --- a/pkg/telemetrytraces/field_mapper.go +++ b/pkg/telemetrytraces/field_mapper.go @@ -9,6 +9,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" "golang.org/x/exp/maps" ) @@ -119,6 +120,41 @@ var ( "attribute_string_rpc$$method_exists": {Name: "attribute_string_rpc$$method_exists", Type: schema.ColumnTypeBool}, "attribute_string_peer$$service_exists": {Name: "attribute_string_peer$$service_exists", Type: schema.ColumnTypeBool}, } + + // TODO(srikanthccv): remove this mapping + oldToNew = map[string]string{ + // deprecated intrinsic -> new intrinsic + "traceID": "trace_id", + "spanID": "span_id", + "parentSpanID": "parent_span_id", + "spanKind": "kind_string", + "durationNano": "duration_nano", + "statusCode": "status_code", + "statusMessage": "status_message", + "statusCodeString": "status_code_string", + + // deprecated derived -> new derived / materialized + "references": "links", + "responseStatusCode": "response_status_code", + "externalHttpUrl": "external_http_url", + "httpUrl": "http_url", + "externalHttpMethod": "external_http_method", + "httpMethod": "http_method", + "httpHost": "http_host", + "dbName": "db_name", + "dbOperation": "db_operation", + "hasError": "has_error", + "isRemote": "is_remote", + "serviceName": "resource_string_service$$name", + "httpRoute": "attribute_string_http$$route", + "msgSystem": "attribute_string_messaging$$system", + "msgOperation": "attribute_string_messaging$$operation", + "dbSystem": "attribute_string_db$$system", + "rpcSystem": "attribute_string_rpc$$system", + "rpcService": "attribute_string_rpc$$service", + "rpcMethod": "attribute_string_rpc$$method", + "peerService": "attribute_string_peer$$service", + } ) type defaultFieldMapper struct{} @@ -155,6 +191,16 @@ func (m *defaultFieldMapper) getColumn( // The actual SQL will be generated in the condition builder return &schema.Column{Name: key.Name, Type: schema.ColumnTypeBool}, nil } + + // TODO(srikanthccv): remove this when it's safe to remove + // issue with CH aliasing + if _, ok := CalculatedFieldsDeprecated[key.Name]; ok { + return indexV3Columns[oldToNew[key.Name]], nil + } + if _, ok := IntrinsicFieldsDeprecated[key.Name]; ok { + return indexV3Columns[oldToNew[key.Name]], nil + } + if col, ok := indexV3Columns[key.Name]; ok { return col, nil } @@ -262,7 +308,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor( return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "%s", correction) } else { // not even a close match, return an error - return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field %s not found", field.Name) + return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name) } } } else if len(keysForField) == 1 { @@ -279,5 +325,5 @@ func (m *defaultFieldMapper) ColumnExpressionFor( } } - return fmt.Sprintf("%s AS `%s`", colName, field.Name), nil + return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil } diff --git a/pkg/telemetrytraces/statement_builder.go b/pkg/telemetrytraces/statement_builder.go index 3ce6352c1b40..d231bdfd1800 100644 --- a/pkg/telemetrytraces/statement_builder.go +++ b/pkg/telemetrytraces/statement_builder.go @@ -74,6 +74,8 @@ func (b *traceQueryStatementBuilder) Build( return nil, err } + b.adjustKeys(ctx, keys, query) + // Check if filter contains trace_id(s) and optimize time range if needed if query.Filter != nil && query.Filter.Expression != "" && b.telemetryStore != nil { traceIDs, found := ExtractTraceIDsFromFilter(query.Filter.Expression) @@ -128,19 +130,17 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]) for idx := range query.SelectFields { keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{ - Name: query.SelectFields[idx].Name, - Signal: telemetrytypes.SignalTraces, - FieldContext: query.SelectFields[idx].FieldContext, - FieldDataType: query.SelectFields[idx].FieldDataType, + Name: query.SelectFields[idx].Name, + Signal: telemetrytypes.SignalTraces, + FieldContext: query.SelectFields[idx].FieldContext, }) } for idx := range query.Order { keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{ - Name: query.Order[idx].Key.Name, - Signal: telemetrytypes.SignalTraces, - FieldContext: query.Order[idx].Key.FieldContext, - FieldDataType: query.Order[idx].Key.FieldDataType, + Name: query.Order[idx].Key.Name, + Signal: telemetrytypes.SignalTraces, + FieldContext: query.Order[idx].Key.FieldContext, }) } @@ -151,6 +151,100 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]) return keySelectors } +func (b *traceQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]) { + // for group by / order by / selected fields, if there is a key + // that exactly matches the name of intrinsic / calculated field but has + // a field context or data type that doesn't match the field context or data type of the + // intrinsic field, + // and there is no additional key present in the data with the incoming key match, + // then override the given context with + // intrinsic / calculated field context and data type + // Why does that happen? Because we have a lot of assets created by users and shared over web + // that has incorrect context or data type populated so we fix it + // note: this override happens only when there is no match; if there is a match, + // we can't make decision on behalf of users so we let it use unmodified + + // example: {"key": "httpRoute","type": "tag","dataType": "string"} + // This is sent as "tag", when it's not, this was earlier managed with + // `isColumn`, which we don't have in v5 (because it's not a user concern whether it's mat col or not) + // Such requests as-is look for attributes, the following code exists to handle them + checkMatch := func(k *telemetrytypes.TelemetryFieldKey) { + var overallMatch bool + + findMatch := func(staticKeys map[string]telemetrytypes.TelemetryFieldKey) bool { + // for a given key `k`, iterate over the metadata keys `keys` + // and see if there is any exact match + match := false + for _, mapKey := range keys[k.Name] { + if mapKey.FieldContext == k.FieldContext && mapKey.FieldDataType == k.FieldDataType { + match = true + } + } + // we don't have exact match, then it's doesn't exist in attribute or resource attribute + // use the intrinsic/calculated field + if !match { + b.logger.InfoContext(ctx, "overriding the field context and data type", "key", k.Name) + k.FieldContext = staticKeys[k.Name].FieldContext + k.FieldDataType = staticKeys[k.Name].FieldDataType + } + return match + } + + if _, ok := IntrinsicFields[k.Name]; ok { + overallMatch = overallMatch || findMatch(IntrinsicFields) + } + if _, ok := CalculatedFields[k.Name]; ok { + overallMatch = overallMatch || findMatch(CalculatedFields) + } + if _, ok := IntrinsicFieldsDeprecated[k.Name]; ok { + overallMatch = overallMatch || findMatch(IntrinsicFieldsDeprecated) + } + if _, ok := CalculatedFieldsDeprecated[k.Name]; ok { + overallMatch = overallMatch || findMatch(CalculatedFieldsDeprecated) + } + + if !overallMatch { + // check if all the key for the given field have been materialized, if so + // set the key to materialized + materilized := true + for _, key := range keys[k.Name] { + materilized = materilized && key.Materialized + } + k.Materialized = materilized + } + } + + for idx := range query.GroupBy { + checkMatch(&query.GroupBy[idx].TelemetryFieldKey) + } + for idx := range query.Order { + checkMatch(&query.Order[idx].Key.TelemetryFieldKey) + } + for idx := range query.SelectFields { + checkMatch(&query.SelectFields[idx]) + } + + // add deprecated fields only during statement building + // why? + // 1. to not fail filter expression that use deprecated cols + // 2. this could have been moved to metadata fetching itself, however, that + // would mean, they also show up in suggestions we we don't want to do + for fieldKeyName, fieldKey := range IntrinsicFieldsDeprecated { + if _, ok := keys[fieldKeyName]; !ok { + keys[fieldKeyName] = []*telemetrytypes.TelemetryFieldKey{&fieldKey} + } else { + keys[fieldKeyName] = append(keys[fieldKeyName], &fieldKey) + } + } + for fieldKeyName, fieldKey := range CalculatedFieldsDeprecated { + if _, ok := keys[fieldKeyName]; !ok { + keys[fieldKeyName] = []*telemetrytypes.TelemetryFieldKey{&fieldKey} + } else { + keys[fieldKeyName] = append(keys[fieldKeyName], &fieldKey) + } + } +} + // buildListQuery builds a query for list panel type func (b *traceQueryStatementBuilder) buildListQuery( ctx context.Context, @@ -176,7 +270,11 @@ func (b *traceQueryStatementBuilder) buildListQuery( selectedFields := query.SelectFields if len(selectedFields) == 0 { - selectedFields = maps.Values(DefaultFields) + sortedKeys := maps.Keys(DefaultFields) + slices.Sort(sortedKeys) + for _, key := range sortedKeys { + selectedFields = append(selectedFields, DefaultFields[key]) + } } selectFieldKeys := []string{} @@ -196,7 +294,7 @@ func (b *traceQueryStatementBuilder) buildListQuery( if err != nil { return nil, err } - sb.SelectMore(sqlbuilder.Escape(colExpr)) + sb.SelectMore(colExpr) } // From table @@ -277,7 +375,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery( } colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.TelemetryFieldKey.Name) allGroupByArgs = append(allGroupByArgs, args...) - sb.SelectMore(sqlbuilder.Escape(colExpr)) + sb.SelectMore(colExpr) fieldNames = append(fieldNames, fmt.Sprintf("`%s`", gb.TelemetryFieldKey.Name)) } @@ -394,7 +492,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery( } colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.TelemetryFieldKey.Name) allGroupByArgs = append(allGroupByArgs, args...) - sb.SelectMore(sqlbuilder.Escape(colExpr)) + sb.SelectMore(colExpr) } // for scalar queries, the rate would be end-start diff --git a/pkg/telemetrytraces/stmt_builder_test.go b/pkg/telemetrytraces/stmt_builder_test.go index c89c14c93d45..ef1a5cdb9e00 100644 --- a/pkg/telemetrytraces/stmt_builder_test.go +++ b/pkg/telemetrytraces/stmt_builder_test.go @@ -59,11 +59,368 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, + { + name: "legacy httpRoute in group by", + requestType: qbtypes.RequestTypeTimeSeries, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.TraceAggregation{ + { + Expression: "count()", + }, + }, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + GroupBy: []qbtypes.GroupByKey{ + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "httpRoute", + FieldDataType: telemetrytypes.FieldDataTypeString, + FieldContext: telemetrytypes.FieldContextAttribute, + }, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(attribute_string_http$$route <> ?, attribute_string_http$$route, NULL)) AS `httpRoute`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `httpRoute` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(attribute_string_http$$route <> ?, attribute_string_http$$route, NULL)) AS `httpRoute`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`httpRoute`) GLOBAL IN (SELECT `httpRoute` FROM __limit_cte) GROUP BY ts, `httpRoute`", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), "", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + }, + expectedErr: nil, + }, + { + name: "legacy fields in search and group by", + requestType: qbtypes.RequestTypeTimeSeries, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.TraceAggregation{ + { + Expression: "count()", + }, + }, + Filter: &qbtypes.Filter{ + Expression: "serviceName = $service.name AND httpMethod EXISTS AND spanKind = 'Server'", + }, + Limit: 10, + GroupBy: []qbtypes.GroupByKey{ + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "httpRoute", + FieldDataType: telemetrytypes.FieldDataTypeString, + FieldContext: telemetrytypes.FieldContextAttribute, + }, + }, + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "httpMethod", + FieldDataType: telemetrytypes.FieldDataTypeString, + FieldContext: telemetrytypes.FieldContextAttribute, + }, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (true AND true AND true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(attribute_string_http$$route <> ?, attribute_string_http$$route, NULL)) AS `httpRoute`, toString(multiIf(http_method <> ?, http_method, NULL)) AS `httpMethod`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((resource_string_service$$name = ? AND resource_string_service$$name <> ?) AND http_method <> ? AND kind_string = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `httpRoute`, `httpMethod` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(attribute_string_http$$route <> ?, attribute_string_http$$route, NULL)) AS `httpRoute`, toString(multiIf(http_method <> ?, http_method, NULL)) AS `httpMethod`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((resource_string_service$$name = ? AND resource_string_service$$name <> ?) AND http_method <> ? AND kind_string = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`httpRoute`, `httpMethod`) GLOBAL IN (SELECT `httpRoute`, `httpMethod` FROM __limit_cte) GROUP BY ts, `httpRoute`, `httpMethod`", + Args: []any{uint64(1747945619), uint64(1747983448), "", "", "redis-manual", "", "", "Server", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "", "", "redis-manual", "", "", "Server", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + }, + expectedErr: nil, + }, + { + name: "context as key prefix test", + requestType: qbtypes.RequestTypeTimeSeries, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.TraceAggregation{ + { + Expression: "sum(metric.max_count)", + }, + }, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + GroupBy: []qbtypes.GroupByKey{ + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "service.name", + }, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, sum(multiIf(mapContains(attributes_number, 'metric.max_count') = ?, toFloat64(attributes_number['metric.max_count']), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, sum(multiIf(mapContains(attributes_number, 'metric.max_count') = ?, toFloat64(attributes_number['metric.max_count']), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), true, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + }, + expectedErr: nil, + }, + { + name: "mat number key in aggregation test", + requestType: qbtypes.RequestTypeTimeSeries, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.TraceAggregation{ + { + Expression: "sum(cart.items_count)", + }, + }, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + GroupBy: []qbtypes.GroupByKey{ + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "service.name", + }, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists` = ?, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists` = ?, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), true, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + }, + expectedErr: nil, + }, + { + name: "Legacy column with incorrect field context test", + requestType: qbtypes.RequestTypeTimeSeries, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.TraceAggregation{ + { + Expression: "count()", + }, + }, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + GroupBy: []qbtypes.GroupByKey{ + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "responseStatusCode", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(response_status_code <> ?, response_status_code, NULL)) AS `responseStatusCode`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(response_status_code <> ?, response_status_code, NULL)) AS `responseStatusCode`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`responseStatusCode`) GLOBAL IN (SELECT `responseStatusCode` FROM __limit_cte) GROUP BY ts, `responseStatusCode`", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), "", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + }, + expectedErr: nil, + }, + { + name: "Legacy column in aggregation and incorrect field context test", + requestType: qbtypes.RequestTypeTimeSeries, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.TraceAggregation{ + { + Expression: "p90(durationNano)", + }, + }, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + GroupBy: []qbtypes.GroupByKey{ + { + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "responseStatusCode", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(response_status_code <> ?, response_status_code, NULL)) AS `responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> ?, duration_nano, NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(response_status_code <> ?, response_status_code, NULL)) AS `responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> ?, duration_nano, NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`responseStatusCode`) GLOBAL IN (SELECT `responseStatusCode` FROM __limit_cte) GROUP BY ts, `responseStatusCode`", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), "", 0, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "", 0, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + }, + expectedErr: nil, + }, + } + + fm := NewFieldMapper() + cb := NewConditionBuilder(fm) + mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() + aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + + resourceFilterStmtBuilder := resourceFilterStmtBuilder() + + statementBuilder := NewTraceQueryStatementBuilder( + instrumentationtest.New().ToProviderSettings(), + mockMetadataStore, + fm, + cb, + resourceFilterStmtBuilder, + aggExprRewriter, + nil, + ) + + vars := map[string]qbtypes.VariableItem{ + "service.name": { + Value: "redis-manual", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, vars) + + if c.expectedErr != nil { + require.Error(t, err) + require.Contains(t, err.Error(), c.expectedErr.Error()) + } else { + require.NoError(t, err) + require.Equal(t, c.expected.Query, q.Query) + require.Equal(t, c.expected.Args, q.Args) + require.Equal(t, c.expected.Warnings, q.Warnings) + } + }) + } +} + +func TestStatementBuilderListQuery(t *testing.T) { + cases := []struct { + name string + requestType qbtypes.RequestType + query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation] + expected qbtypes.Statement + expectedErr error + }{ + { + name: "List query with mat selected fields", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + SelectFields: []telemetrytypes.TelemetryFieldKey{ + { + Name: "name", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + { + Name: "service.name", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextResource, + FieldDataType: telemetrytypes.FieldDataTypeString, + Materialized: true, + }, + { + Name: "duration_nano", + Signal: telemetrytypes.SignalTraces, + FieldContext: telemetrytypes.FieldContextSpan, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + { + Name: "cart.items_count", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeFloat64, + }, + }, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT name AS `name`, resources_string['service.name'] AS `service.name`, duration_nano AS `duration_nano`, `attribute_number_cart$$items_count` AS `cart.items_count`, timestamp AS `timestamp`, span_id AS `span_id`, trace_id AS `trace_id` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + { + name: "List query with default fields and attribute order by", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Order: []qbtypes.OrderBy{ + { + Key: qbtypes.OrderByKey{ + TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, + Direction: qbtypes.OrderDirectionDesc, + }, + }, + Limit: 10, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT duration_nano AS `duration_nano`, name AS `name`, response_status_code AS `response_status_code`, `resource_string_service$$name` AS `service.name`, span_id AS `span_id`, timestamp AS `timestamp`, trace_id AS `trace_id` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY attributes_string['user.id'] AS `user.id` desc LIMIT ?", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + { + name: "List query with legacy fields", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + SelectFields: []telemetrytypes.TelemetryFieldKey{ + { + Name: "name", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + { + Name: "serviceName", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + { + Name: "durationNano", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + { + Name: "httpMethod", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + { + Name: "responseStatusCode", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, + Limit: 10, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT name AS `name`, resource_string_service$$name AS `serviceName`, duration_nano AS `durationNano`, http_method AS `httpMethod`, response_status_code AS `responseStatusCode`, timestamp AS `timestamp`, span_id AS `span_id`, trace_id AS `trace_id` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"redis-manual", "%service.name%", "%service.name%redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + }, + expectedErr: nil, + }, } fm := NewFieldMapper() diff --git a/pkg/telemetrytraces/test_data.go b/pkg/telemetrytraces/test_data.go index 926fc61aae76..6c25ae8c83d6 100644 --- a/pkg/telemetrytraces/test_data.go +++ b/pkg/telemetrytraces/test_data.go @@ -34,6 +34,28 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey { FieldDataType: telemetrytypes.FieldDataTypeString, }, }, + "metric.max_count": { + { + Name: "metric.max_count", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeFloat64, + }, + }, + "cart.items_count": { + { + Name: "cart.items_count", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeFloat64, + Materialized: true, + }, + }, + "user.id": { + { + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, } for _, keys := range keysMap { for _, key := range keys { diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/functions.go b/pkg/types/querybuildertypes/querybuildertypesv5/functions.go index c4592e351cd7..837c4a473cf2 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/functions.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/functions.go @@ -376,8 +376,8 @@ func funcFillZero(result *TimeSeries, start, end, step int64) *TimeSeries { return result } - alignedStart := (start / step) * step - alignedEnd := ((end + step - 1) / step) * step + alignedStart := start - (start % (step * 1000)) + alignedEnd := end existingValues := make(map[int64]*TimeSeriesValue) for _, v := range result.Values { @@ -386,7 +386,7 @@ func funcFillZero(result *TimeSeries, start, end, step int64) *TimeSeries { filledValues := make([]*TimeSeriesValue, 0) - for ts := alignedStart; ts <= alignedEnd; ts += step { + for ts := alignedStart; ts <= alignedEnd; ts += step * 1000 { if val, exists := existingValues[ts]; exists { filledValues = append(filledValues, val) } else { diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/functions_test.go b/pkg/types/querybuildertypes/querybuildertypesv5/functions_test.go index f16a30a595cc..3b62bdc0a729 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/functions_test.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/functions_test.go @@ -698,7 +698,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 3000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 1.0}, @@ -717,7 +717,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 3000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 1.0}, @@ -737,7 +737,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 6000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 1.0}, @@ -761,7 +761,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 6000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 1.0}, @@ -780,7 +780,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 3000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 0}, @@ -798,7 +798,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 3000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 1.0}, @@ -820,7 +820,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 1000, end: 4000, - step: 1000, + step: 1, expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 1000, Value: 1.0}, @@ -841,7 +841,7 @@ func TestFuncFillZero(t *testing.T) { }, start: 50000, // Not aligned to 60s end: 250000, // Not aligned to 60s - step: 60000, // 60 seconds + step: 60, // 60 seconds expected: &TimeSeries{ Values: []*TimeSeriesValue{ {Timestamp: 0, Value: 0}, // Aligned start @@ -849,7 +849,6 @@ func TestFuncFillZero(t *testing.T) { {Timestamp: 120000, Value: 2.0}, {Timestamp: 180000, Value: 0}, // Filled gap {Timestamp: 240000, Value: 4.0}, - {Timestamp: 300000, Value: 0}, // Aligned end }, }, }, @@ -891,7 +890,7 @@ func TestApplyFunction_FillZero(t *testing.T) { Args: []FunctionArg{ {Value: 1000.0}, // start {Value: 4000.0}, // end - {Value: 1000.0}, // step + {Value: 1.0}, // step }, } diff --git a/pkg/types/telemetrytypes/field.go b/pkg/types/telemetrytypes/field.go index 261a44ead5fc..38928adcc952 100644 --- a/pkg/types/telemetrytypes/field.go +++ b/pkg/types/telemetrytypes/field.go @@ -36,6 +36,9 @@ func (f TelemetryFieldKey) String() string { if f.FieldDataType != FieldDataTypeUnspecified { sb.WriteString(fmt.Sprintf(",type=%s", f.FieldDataType.StringValue())) } + if f.Materialized { + sb.WriteString(",materialized") + } return sb.String() } @@ -163,11 +166,26 @@ func DataTypeCollisionHandledFieldName(key *TelemetryFieldKey, value any, tblFie case FieldDataTypeFloat64, FieldDataTypeInt64, FieldDataTypeNumber: switch v := value.(type) { + // why? ; CH returns an error for a simple check + // attributes_number['http.status_code'] = 200 but not for attributes_number['http.status_code'] >= 200 + // DB::Exception: Bad get: has UInt64, requested Float64. + // How is it working in v4? v4 prepares the full query with values in query string + // When we format the float it becomes attributes_number['http.status_code'] = 200.000 + // Which CH gladly accepts and doesn't throw error + // However, when passed as query args, the default formatter + // https://github.com/ClickHouse/clickhouse-go/blob/757e102f6d8c6059d564ce98795b4ce2a101b1a5/bind.go#L393 + // is used which prepares the + // final query as attributes_number['http.status_code'] = 200 giving this error + // This following is one way to workaround it + case float32, float64: + tblFieldName = castFloatHack(tblFieldName) case string: // try to convert the number attribute to string tblFieldName = castString(tblFieldName) // numeric col vs string literal case []any: - if hasString(v) { + if allFloats(v) { + tblFieldName = castFloatHack(tblFieldName) + } else if hasString(v) { tblFieldName, value = castString(tblFieldName), toStrings(v) } } @@ -185,8 +203,9 @@ func DataTypeCollisionHandledFieldName(key *TelemetryFieldKey, value any, tblFie return tblFieldName, value } -func castFloat(col string) string { return fmt.Sprintf("toFloat64OrNull(%s)", col) } -func castString(col string) string { return fmt.Sprintf("toString(%s)", col) } +func castFloat(col string) string { return fmt.Sprintf("toFloat64OrNull(%s)", col) } +func castFloatHack(col string) string { return fmt.Sprintf("toFloat64(%s)", col) } +func castString(col string) string { return fmt.Sprintf("toString(%s)", col) } func allFloats(in []any) bool { for _, x := range in { diff --git a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go index 41e7065689c8..63bc0b4e7bca 100644 --- a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go +++ b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go @@ -179,7 +179,10 @@ func matchesKey(selector *telemetrytypes.FieldKeySelector, key *telemetrytypes.T } // Check field context + // check for the context filter only for attribute and resource attribute if selector.FieldContext != telemetrytypes.FieldContextUnspecified && + (selector.FieldContext == telemetrytypes.FieldContextAttribute || + selector.FieldContext == telemetrytypes.FieldContextResource) && selector.FieldContext != key.FieldContext { return false }