diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 5828bf95660d..26f69087b164 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -59,7 +59,7 @@ func NewAPIHandler(opts APIHandlerOptions, signoz *signoz.SigNoz) (*APIHandler, LicensingAPI: httplicensing.NewLicensingAPI(signoz.Licensing), FieldsAPI: fields.NewAPI(signoz.Instrumentation.ToProviderSettings(), signoz.TelemetryStore), Signoz: signoz, - QuerierAPI: querierAPI.NewAPI(signoz.Instrumentation.ToProviderSettings(), signoz.Querier), + QuerierAPI: querierAPI.NewAPI(signoz.Instrumentation.ToProviderSettings(), signoz.Querier, signoz.Analytics), }) if err != nil { diff --git a/ee/query-service/rules/anomaly.go b/ee/query-service/rules/anomaly.go index c7ab34805fec..16b2eeb060ff 100644 --- a/ee/query-service/rules/anomaly.go +++ b/ee/query-service/rules/anomaly.go @@ -198,15 +198,17 @@ func (r *AnomalyRule) prepareQueryRangeV5(ts time.Time) (*qbtypes.QueryRangeRequ startTs, endTs := r.Timestamps(ts) start, end := startTs.UnixMilli(), endTs.UnixMilli() - return &qbtypes.QueryRangeRequest{ + req := &qbtypes.QueryRangeRequest{ Start: uint64(start), End: uint64(end), RequestType: qbtypes.RequestTypeTimeSeries, CompositeQuery: qbtypes.CompositeQuery{ - Queries: r.Condition().CompositeQuery.Queries, + Queries: make([]qbtypes.QueryEnvelope, 0), }, NoCache: true, - }, nil + } + copy(r.Condition().CompositeQuery.Queries, req.CompositeQuery.Queries) + return req, nil } func (r *AnomalyRule) GetSelectedQuery() string { @@ -263,7 +265,7 @@ func (r *AnomalyRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUID, anomalies, err := r.providerV2.GetAnomalies(ctx, orgID, &anomalyV2.AnomaliesRequest{ Params: *params, - Seasonality: anomalyV2.Seasonality{valuer.NewString(r.seasonality.String())}, + Seasonality: anomalyV2.Seasonality{String: valuer.NewString(r.seasonality.String())}, }) if err != nil { return nil, err @@ -303,8 +305,10 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (interface{}, erro var err error if r.version == "v5" { + zap.L().Info("running v5 query") res, err = r.buildAndRunQueryV5(ctx, r.OrgID(), ts) } else { + zap.L().Info("running v4 query") res, err = r.buildAndRunQuery(ctx, r.OrgID(), ts) } if err != nil { diff --git a/pkg/contextlinks/alert_link_visitor.go b/pkg/contextlinks/alert_link_visitor.go new file mode 100644 index 000000000000..cac8bba29f20 --- /dev/null +++ b/pkg/contextlinks/alert_link_visitor.go @@ -0,0 +1,448 @@ +package contextlinks + +import ( + "fmt" + "slices" + "strings" + + parser "github.com/SigNoz/signoz/pkg/parser/grammar" + "github.com/antlr4-go/antlr/v4" + "golang.org/x/exp/maps" + + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" +) + +type WhereClauseRewriter struct { + parser.BaseFilterQueryVisitor + labels map[string]string + groupByItems []qbtypes.GroupByKey + groupBySet map[string]struct{} + keysSeen map[string]struct{} + rewritten strings.Builder +} + +// PrepareFilterExpression prepares the where clause for the query +// `labels` contains the key value pairs of the labels from the result of the query +// We "visit" the where clause and make necessary changes to existing query +// There are two cases: +// 1. The label is present in the where clause +// 2. The label is not present in the where clause +// +// Example for case 2: +// Latency by service.name without any filter +// In this case, for each service with latency > threshold we send a notification +// The expectation is that clicking on the related traces for service A, will +// take us to the traces page with the filter service.name=A +// So for all the missing labels in the where clause, we add them as key = value +// +// Example for case 1: +// Severity text IN (WARN, ERROR) +// In this case, the Severity text will appear in the `labels` if it were part of the group +// by clause, in which case we replace it with the actual value for the notification +// i.e Severity text = WARN +// If the Severity text is not part of the group by clause, then we add it as it is +func PrepareFilterExpression(labels map[string]string, whereClause string, groupByItems []qbtypes.GroupByKey) string { + if whereClause == "" && len(labels) == 0 { + return "" + } + + groupBySet := make(map[string]struct{}) + for _, item := range groupByItems { + groupBySet[item.Name] = struct{}{} + } + + input := antlr.NewInputStream(whereClause) + lexer := parser.NewFilterQueryLexer(input) + stream := antlr.NewCommonTokenStream(lexer, 0) + parser := parser.NewFilterQueryParser(stream) + + tree := parser.Query() + + rewriter := &WhereClauseRewriter{ + labels: labels, + groupByItems: groupByItems, + groupBySet: groupBySet, + keysSeen: map[string]struct{}{}, + } + + // visit the tree to rewrite the where clause + rewriter.Visit(tree) + rewrittenClause := strings.TrimSpace(rewriter.rewritten.String()) + + // sorted key for deterministic order + sortedKeys := maps.Keys(labels) + slices.Sort(sortedKeys) + + // case 2: add missing labels from the labels map + missingLabels := []string{} + for _, key := range sortedKeys { + if !rewriter.isKeyInWhereClause(key) { + // escape the value if it contains special characters or spaces + escapedValue := escapeValueIfNeeded(labels[key]) + missingLabels = append(missingLabels, fmt.Sprintf("%s=%s", key, escapedValue)) + } + } + + // combine + if len(missingLabels) > 0 { + if rewrittenClause != "" { + rewrittenClause = fmt.Sprintf("(%s) AND %s", rewrittenClause, strings.Join(missingLabels, " AND ")) + } else { + rewrittenClause = strings.Join(missingLabels, " AND ") + } + } + + return rewrittenClause +} + +// Visit implements the visitor for the query rule +func (r *WhereClauseRewriter) Visit(tree antlr.ParseTree) interface{} { + return tree.Accept(r) +} + +// VisitQuery visits the query node +func (r *WhereClauseRewriter) VisitQuery(ctx *parser.QueryContext) interface{} { + if ctx.Expression() != nil { + ctx.Expression().Accept(r) + } + return nil +} + +// VisitExpression visits the expression node +func (r *WhereClauseRewriter) VisitExpression(ctx *parser.ExpressionContext) interface{} { + if ctx.OrExpression() != nil { + ctx.OrExpression().Accept(r) + } + return nil +} + +// VisitOrExpression visits OR expressions +func (r *WhereClauseRewriter) VisitOrExpression(ctx *parser.OrExpressionContext) interface{} { + for i, andExpr := range ctx.AllAndExpression() { + if i > 0 { + r.rewritten.WriteString(" OR ") + } + andExpr.Accept(r) + } + return nil +} + +// VisitAndExpression visits AND expressions +func (r *WhereClauseRewriter) VisitAndExpression(ctx *parser.AndExpressionContext) interface{} { + unaryExprs := ctx.AllUnaryExpression() + for i, unaryExpr := range unaryExprs { + if i > 0 { + // Check if there's an explicit AND + if i-1 < len(ctx.AllAND()) && ctx.AND(i-1) != nil { + r.rewritten.WriteString(" AND ") + } else { + // implicit + r.rewritten.WriteString(" AND ") + } + } + unaryExpr.Accept(r) + } + return nil +} + +// VisitUnaryExpression visits unary expressions (with optional NOT) +func (r *WhereClauseRewriter) VisitUnaryExpression(ctx *parser.UnaryExpressionContext) interface{} { + if ctx.NOT() != nil { + r.rewritten.WriteString("NOT ") + } + if ctx.Primary() != nil { + ctx.Primary().Accept(r) + } + return nil +} + +// VisitPrimary visits primary expressions +func (r *WhereClauseRewriter) VisitPrimary(ctx *parser.PrimaryContext) interface{} { + if ctx.LPAREN() != nil && ctx.RPAREN() != nil { + r.rewritten.WriteString("(") + if ctx.OrExpression() != nil { + ctx.OrExpression().Accept(r) + } + r.rewritten.WriteString(")") + } else if ctx.Comparison() != nil { + ctx.Comparison().Accept(r) + } else if ctx.FunctionCall() != nil { + ctx.FunctionCall().Accept(r) + } else if ctx.FullText() != nil { + ctx.FullText().Accept(r) + } else if ctx.Key() != nil { + ctx.Key().Accept(r) + } else if ctx.Value() != nil { + ctx.Value().Accept(r) + } + return nil +} + +// VisitComparison visits comparison expressions +func (r *WhereClauseRewriter) VisitComparison(ctx *parser.ComparisonContext) interface{} { + if ctx.Key() == nil { + return nil + } + + key := ctx.Key().GetText() + r.keysSeen[key] = struct{}{} + + // Check if this key is in the labels and was part of group by + if value, exists := r.labels[key]; exists { + if _, partOfGroup := r.groupBySet[key]; partOfGroup { + // Case 1: Replace with actual value + escapedValue := escapeValueIfNeeded(value) + r.rewritten.WriteString(fmt.Sprintf("%s=%s", key, escapedValue)) + return nil + } + } + + // Otherwise, keep the original comparison + r.rewritten.WriteString(key) + + if ctx.EQUALS() != nil { + r.rewritten.WriteString("=") + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.NOT_EQUALS() != nil || ctx.NEQ() != nil { + if ctx.NOT_EQUALS() != nil { + r.rewritten.WriteString("!=") + } else { + r.rewritten.WriteString("<>") + } + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.LT() != nil { + r.rewritten.WriteString("<") + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.LE() != nil { + r.rewritten.WriteString("<=") + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.GT() != nil { + r.rewritten.WriteString(">") + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.GE() != nil { + r.rewritten.WriteString(">=") + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.LIKE() != nil || ctx.ILIKE() != nil { + if ctx.LIKE() != nil { + r.rewritten.WriteString(" LIKE ") + } else { + r.rewritten.WriteString(" ILIKE ") + } + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.NOT_LIKE() != nil || ctx.NOT_ILIKE() != nil { + if ctx.NOT_LIKE() != nil { + r.rewritten.WriteString(" NOT LIKE ") + } else { + r.rewritten.WriteString(" NOT ILIKE ") + } + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.BETWEEN() != nil { + if ctx.NOT() != nil { + r.rewritten.WriteString(" NOT BETWEEN ") + } else { + r.rewritten.WriteString(" BETWEEN ") + } + if len(ctx.AllValue()) >= 2 { + r.rewritten.WriteString(ctx.Value(0).GetText()) + r.rewritten.WriteString(" AND ") + r.rewritten.WriteString(ctx.Value(1).GetText()) + } + } else if ctx.InClause() != nil { + r.rewritten.WriteString(" ") + ctx.InClause().Accept(r) + } else if ctx.NotInClause() != nil { + r.rewritten.WriteString(" ") + ctx.NotInClause().Accept(r) + } else if ctx.EXISTS() != nil { + if ctx.NOT() != nil { + r.rewritten.WriteString(" NOT EXISTS") + } else { + r.rewritten.WriteString(" EXISTS") + } + } else if ctx.REGEXP() != nil { + if ctx.NOT() != nil { + r.rewritten.WriteString(" NOT REGEXP ") + } else { + r.rewritten.WriteString(" REGEXP ") + } + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } else if ctx.CONTAINS() != nil { + if ctx.NOT() != nil { + r.rewritten.WriteString(" NOT CONTAINS ") + } else { + r.rewritten.WriteString(" CONTAINS ") + } + if ctx.Value(0) != nil { + r.rewritten.WriteString(ctx.Value(0).GetText()) + } + } + + return nil +} + +// VisitInClause visits IN clauses +func (r *WhereClauseRewriter) VisitInClause(ctx *parser.InClauseContext) interface{} { + r.rewritten.WriteString("IN ") + if ctx.LPAREN() != nil { + r.rewritten.WriteString("(") + if ctx.ValueList() != nil { + ctx.ValueList().Accept(r) + } + r.rewritten.WriteString(")") + } else if ctx.LBRACK() != nil { + r.rewritten.WriteString("[") + if ctx.ValueList() != nil { + ctx.ValueList().Accept(r) + } + r.rewritten.WriteString("]") + } else if ctx.Value() != nil { + r.rewritten.WriteString(ctx.Value().GetText()) + } + return nil +} + +// VisitNotInClause visits NOT IN clauses +func (r *WhereClauseRewriter) VisitNotInClause(ctx *parser.NotInClauseContext) interface{} { + r.rewritten.WriteString("NOT IN ") + if ctx.LPAREN() != nil { + r.rewritten.WriteString("(") + if ctx.ValueList() != nil { + ctx.ValueList().Accept(r) + } + r.rewritten.WriteString(")") + } else if ctx.LBRACK() != nil { + r.rewritten.WriteString("[") + if ctx.ValueList() != nil { + ctx.ValueList().Accept(r) + } + r.rewritten.WriteString("]") + } else if ctx.Value() != nil { + r.rewritten.WriteString(ctx.Value().GetText()) + } + return nil +} + +// VisitValueList visits value lists +func (r *WhereClauseRewriter) VisitValueList(ctx *parser.ValueListContext) interface{} { + values := ctx.AllValue() + for i, val := range values { + if i > 0 { + r.rewritten.WriteString(", ") + } + r.rewritten.WriteString(val.GetText()) + } + return nil +} + +// VisitFullText visits full text expressions +func (r *WhereClauseRewriter) VisitFullText(ctx *parser.FullTextContext) interface{} { + r.rewritten.WriteString(ctx.GetText()) + return nil +} + +// VisitFunctionCall visits function calls +func (r *WhereClauseRewriter) VisitFunctionCall(ctx *parser.FunctionCallContext) interface{} { + // Write function name + if ctx.HAS() != nil { + r.rewritten.WriteString("has") + } else if ctx.HASANY() != nil { + r.rewritten.WriteString("hasany") + } else if ctx.HASALL() != nil { + r.rewritten.WriteString("hasall") + } + + r.rewritten.WriteString("(") + if ctx.FunctionParamList() != nil { + ctx.FunctionParamList().Accept(r) + } + r.rewritten.WriteString(")") + return nil +} + +// VisitFunctionParamList visits function parameter lists +func (r *WhereClauseRewriter) VisitFunctionParamList(ctx *parser.FunctionParamListContext) interface{} { + params := ctx.AllFunctionParam() + for i, param := range params { + if i > 0 { + r.rewritten.WriteString(", ") + } + param.Accept(r) + } + return nil +} + +// VisitFunctionParam visits function parameters +func (r *WhereClauseRewriter) VisitFunctionParam(ctx *parser.FunctionParamContext) interface{} { + if ctx.Key() != nil { + ctx.Key().Accept(r) + } else if ctx.Value() != nil { + ctx.Value().Accept(r) + } else if ctx.Array() != nil { + ctx.Array().Accept(r) + } + return nil +} + +// VisitArray visits array expressions +func (r *WhereClauseRewriter) VisitArray(ctx *parser.ArrayContext) interface{} { + r.rewritten.WriteString("[") + if ctx.ValueList() != nil { + ctx.ValueList().Accept(r) + } + r.rewritten.WriteString("]") + return nil +} + +// VisitValue visits value expressions +func (r *WhereClauseRewriter) VisitValue(ctx *parser.ValueContext) interface{} { + r.rewritten.WriteString(ctx.GetText()) + return nil +} + +// VisitKey visits key expressions +func (r *WhereClauseRewriter) VisitKey(ctx *parser.KeyContext) interface{} { + r.keysSeen[ctx.GetText()] = struct{}{} + r.rewritten.WriteString(ctx.GetText()) + return nil +} + +func (r *WhereClauseRewriter) isKeyInWhereClause(key string) bool { + _, ok := r.keysSeen[key] + return ok +} + +// escapeValueIfNeeded adds single quotes to string values and escapes single quotes within them +// Numeric and boolean values are returned as-is +func escapeValueIfNeeded(value string) string { + // Check if it's a number + if _, err := fmt.Sscanf(value, "%f", new(float64)); err == nil { + return value + } + + // Check if it's a boolean + if strings.ToLower(value) == "true" || strings.ToLower(value) == "false" { + return value + } + + // For all other values (strings), escape single quotes and wrap in single quotes + escaped := strings.ReplaceAll(value, "'", "\\'") + return fmt.Sprintf("'%s'", escaped) +} diff --git a/pkg/contextlinks/alert_link_visitor_test.go b/pkg/contextlinks/alert_link_visitor_test.go new file mode 100644 index 000000000000..d1efe3db6c18 --- /dev/null +++ b/pkg/contextlinks/alert_link_visitor_test.go @@ -0,0 +1,260 @@ +package contextlinks + +import ( + "testing" + + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/stretchr/testify/assert" +) + +func TestPrepareFiltersV5(t *testing.T) { + tests := []struct { + name string + labels map[string]string + whereClause string + groupByItems []qbtypes.GroupByKey + expected string + description string + }{ + { + name: "empty_inputs", + labels: map[string]string{}, + whereClause: "", + groupByItems: []qbtypes.GroupByKey{}, + expected: "", + description: "Should return empty string for empty inputs", + }, + { + name: "no_label_replacement", + labels: map[string]string{}, + whereClause: "service.name = 'serviceB'", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + }, + expected: "service.name='serviceB'", + description: "No change", + }, + { + name: "in_clause_replacement", + labels: map[string]string{ + "severity_text": "WARN", + }, + whereClause: "severity_text IN ('WARN', 'ERROR')", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "severity_text"}}, + }, + expected: "severity_text='WARN'", + description: "Should replace IN clause with actual value when key is in group by", + }, + { + name: "missing_label_addition", // case 2 + labels: map[string]string{ + "service.name": "serviceA", + }, + whereClause: "status_code > 400", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + }, + expected: "(status_code>400) AND service.name='serviceA'", + description: "Should add missing labels from labels map", + }, + { + name: "multiple_missing_labels", + labels: map[string]string{ + "service.name": "serviceA", + "region": "us-east-1", + }, + whereClause: "status_code > 400", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "region"}}, + }, + expected: "(status_code>400) AND region='us-east-1' AND service.name='serviceA'", + description: "Should add all missing labels", + }, + { + name: "complex_where_clause", + labels: map[string]string{ + "service.name": "serviceA", + }, + whereClause: "(status_code > 400 OR status_code < 200) AND method = 'GET'", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + }, + expected: "((status_code>400 OR status_code<200) AND method='GET') AND service.name='serviceA'", + description: "Should preserve complex boolean logic and add missing labels", + }, + { + name: "label_not_in_group_by", + labels: map[string]string{ + "service.name": "serviceA", + }, + whereClause: "service.name = 'serviceB'", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "region"}}, // service.name not in group by + }, + expected: "service.name='serviceB'", + description: "Should not replace label if not in group by items", + }, + { + name: "special_characters_in_values", + labels: map[string]string{ + "message": "Error: Connection failed", + "path": "/api/v1/users", + }, + whereClause: "", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "message"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "path"}}, + }, + expected: "message='Error: Connection failed' AND path='/api/v1/users'", + description: "Should quote values with special characters", + }, + { + name: "numeric_and_boolean_values", + labels: map[string]string{ + "count": "42", + "isEnabled": "true", + }, + whereClause: "", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "count"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "isEnabled"}}, + }, + expected: "count=42 AND isEnabled=true", + description: "Should not quote numeric and boolean values", + }, + + { + name: "like_operator", + labels: map[string]string{ + "path": "/api/users", + }, + whereClause: "path LIKE '/api%'", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "path"}}, + }, + expected: "path='/api/users'", + description: "Should replace LIKE comparisons when key is in group by", + }, + + { + name: "not_operators", + labels: map[string]string{ + "status": "active", + }, + whereClause: "status NOT IN ('deleted', 'archived')", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "status"}}, + }, + expected: "status='active'", + description: "Should replace NOT IN clause when key is in group by", + }, + + { + name: "between_operator", + labels: map[string]string{ + "response_time": "250", + }, + whereClause: "response_time BETWEEN 100 AND 500", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "response_time"}}, + }, + expected: "response_time=250", + description: "Should replace BETWEEN clause when key is in group by", + }, + + { + name: "function_calls", + labels: map[string]string{ + "service.name": "serviceA", + }, + whereClause: "has(tags, 'production')", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + }, + expected: "(has(tags, 'production')) AND service.name='serviceA'", + description: "Should preserve function calls and add missing labels", + }, + { + name: "already_quoted_values", + labels: map[string]string{ + "message": "\"Error message\"", + "tag": "'production'", + }, + whereClause: "", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "message"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "tag"}}, + }, + expected: "message='\"Error message\"' AND tag='\\'production\\''", + description: "Should not double-quote already quoted values", + }, + + { + name: "mixed_replacement_and_addition", + labels: map[string]string{ + "service.name": "serviceA", + "severity_text": "ERROR", + "region": "us-west-2", + }, + whereClause: "severity_text IN ('WARN', 'ERROR') AND status_code > 400", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "severity_text"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "region"}}, + }, + expected: "(severity_text='ERROR' AND status_code>400) AND region='us-west-2' AND service.name='serviceA'", + description: "Should both replace existing labels and add missing ones", + }, + + { + name: "implicit_and_handling", + labels: map[string]string{ + "env": "production", + }, + whereClause: "status_code=200 method='GET'", // implicit AND + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "env"}}, + }, + expected: "(status_code=200 AND method='GET') AND env='production'", + description: "Should handle implicit AND between expressions", + }, + + { + name: "exists_operator", + labels: map[string]string{ + "service.name": "serviceA", + }, + whereClause: "error_details EXISTS", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + }, + expected: "(error_details EXISTS) AND service.name='serviceA'", + description: "Should preserve EXISTS operator", + }, + + { + name: "empty_where_clause_with_labels", + labels: map[string]string{ + "service.name": "serviceA", + "region": "us-east-1", + }, + whereClause: "", + groupByItems: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}, + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "region"}}, + }, + expected: "region='us-east-1' AND service.name='serviceA'", + description: "Should create where clause from labels when original is empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := PrepareFilterExpression(tt.labels, tt.whereClause, tt.groupByItems) + assert.Equal(t, tt.expected, result, tt.description) + }) + } +} diff --git a/pkg/query-service/contextlinks/links.go b/pkg/contextlinks/links.go similarity index 54% rename from pkg/query-service/contextlinks/links.go rename to pkg/contextlinks/links.go index 7023554958d7..30e2c583c0b6 100644 --- a/pkg/query-service/contextlinks/links.go +++ b/pkg/contextlinks/links.go @@ -14,13 +14,13 @@ import ( func PrepareLinksToTraces(start, end time.Time, filterItems []v3.FilterItem) string { // Traces list view expects time in nanoseconds - tr := v3.URLShareableTimeRange{ + tr := URLShareableTimeRange{ Start: start.UnixNano(), End: end.UnixNano(), PageSize: 100, } - options := v3.URLShareableOptions{ + options := URLShareableOptions{ MaxLines: 2, Format: "list", SelectColumns: tracesV3.TracesListViewDefaultSelectedColumns, @@ -29,32 +29,34 @@ func PrepareLinksToTraces(start, end time.Time, filterItems []v3.FilterItem) str period, _ := json.Marshal(tr) urlEncodedTimeRange := url.QueryEscape(string(period)) - builderQuery := v3.BuilderQuery{ - DataSource: v3.DataSourceTraces, - QueryName: "A", - AggregateOperator: v3.AggregateOperatorNoOp, - AggregateAttribute: v3.AttributeKey{}, - Filters: &v3.FilterSet{ - Items: filterItems, - Operator: "AND", - }, - Expression: "A", - Disabled: false, - Having: []v3.Having{}, - StepInterval: 60, - OrderBy: []v3.OrderBy{ - { - ColumnName: "timestamp", - Order: "desc", + linkQuery := LinkQuery{ + BuilderQuery: v3.BuilderQuery{ + DataSource: v3.DataSourceTraces, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + AggregateAttribute: v3.AttributeKey{}, + Filters: &v3.FilterSet{ + Items: filterItems, + Operator: "AND", + }, + Expression: "A", + Disabled: false, + Having: []v3.Having{}, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, }, }, } - urlData := v3.URLShareableCompositeQuery{ + urlData := URLShareableCompositeQuery{ QueryType: string(v3.QueryTypeBuilder), - Builder: v3.URLShareableBuilderQuery{ - QueryData: []v3.BuilderQuery{ - builderQuery, + Builder: URLShareableBuilderQuery{ + QueryData: []LinkQuery{ + linkQuery, }, QueryFormulas: make([]string, 0), }, @@ -72,13 +74,13 @@ func PrepareLinksToTraces(start, end time.Time, filterItems []v3.FilterItem) str func PrepareLinksToLogs(start, end time.Time, filterItems []v3.FilterItem) string { // Logs list view expects time in milliseconds - tr := v3.URLShareableTimeRange{ + tr := URLShareableTimeRange{ Start: start.UnixMilli(), End: end.UnixMilli(), PageSize: 100, } - options := v3.URLShareableOptions{ + options := URLShareableOptions{ MaxLines: 2, Format: "list", SelectColumns: []v3.AttributeKey{}, @@ -87,32 +89,34 @@ func PrepareLinksToLogs(start, end time.Time, filterItems []v3.FilterItem) strin period, _ := json.Marshal(tr) urlEncodedTimeRange := url.QueryEscape(string(period)) - builderQuery := v3.BuilderQuery{ - DataSource: v3.DataSourceLogs, - QueryName: "A", - AggregateOperator: v3.AggregateOperatorNoOp, - AggregateAttribute: v3.AttributeKey{}, - Filters: &v3.FilterSet{ - Items: filterItems, - Operator: "AND", - }, - Expression: "A", - Disabled: false, - Having: []v3.Having{}, - StepInterval: 60, - OrderBy: []v3.OrderBy{ - { - ColumnName: "timestamp", - Order: "desc", + linkQuery := LinkQuery{ + BuilderQuery: v3.BuilderQuery{ + DataSource: v3.DataSourceLogs, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + AggregateAttribute: v3.AttributeKey{}, + Filters: &v3.FilterSet{ + Items: filterItems, + Operator: "AND", + }, + Expression: "A", + Disabled: false, + Having: []v3.Having{}, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, }, }, } - urlData := v3.URLShareableCompositeQuery{ + urlData := URLShareableCompositeQuery{ QueryType: string(v3.QueryTypeBuilder), - Builder: v3.URLShareableBuilderQuery{ - QueryData: []v3.BuilderQuery{ - builderQuery, + Builder: URLShareableBuilderQuery{ + QueryData: []LinkQuery{ + linkQuery, }, QueryFormulas: make([]string, 0), }, @@ -220,3 +224,111 @@ func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, return filterItems } + +func PrepareLinksToTracesV5(start, end time.Time, whereClause string) string { + + // Traces list view expects time in nanoseconds + tr := URLShareableTimeRange{ + Start: start.UnixNano(), + End: end.UnixNano(), + PageSize: 100, + } + + options := URLShareableOptions{ + MaxLines: 2, + Format: "list", + SelectColumns: tracesV3.TracesListViewDefaultSelectedColumns, + } + + period, _ := json.Marshal(tr) + urlEncodedTimeRange := url.QueryEscape(string(period)) + + linkQuery := LinkQuery{ + BuilderQuery: v3.BuilderQuery{ + DataSource: v3.DataSourceTraces, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + AggregateAttribute: v3.AttributeKey{}, + Expression: "A", + Disabled: false, + Having: []v3.Having{}, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + }, + Filter: &FilterExpression{Expression: whereClause}, + } + + urlData := URLShareableCompositeQuery{ + QueryType: string(v3.QueryTypeBuilder), + Builder: URLShareableBuilderQuery{ + QueryData: []LinkQuery{ + linkQuery, + }, + QueryFormulas: make([]string, 0), + }, + } + + data, _ := json.Marshal(urlData) + compositeQuery := url.QueryEscape(url.QueryEscape(string(data))) + + optionsData, _ := json.Marshal(options) + urlEncodedOptions := url.QueryEscape(string(optionsData)) + + return fmt.Sprintf("compositeQuery=%s&timeRange=%s&startTime=%d&endTime=%d&options=%s", compositeQuery, urlEncodedTimeRange, tr.Start, tr.End, urlEncodedOptions) +} + +func PrepareLinksToLogsV5(start, end time.Time, whereClause string) string { + + // Logs list view expects time in milliseconds + tr := URLShareableTimeRange{ + Start: start.UnixMilli(), + End: end.UnixMilli(), + PageSize: 100, + } + + options := URLShareableOptions{ + MaxLines: 2, + Format: "list", + SelectColumns: []v3.AttributeKey{}, + } + + period, _ := json.Marshal(tr) + urlEncodedTimeRange := url.QueryEscape(string(period)) + + linkQuery := LinkQuery{ + BuilderQuery: v3.BuilderQuery{ + DataSource: v3.DataSourceLogs, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + AggregateAttribute: v3.AttributeKey{}, + Expression: "A", + Disabled: false, + Having: []v3.Having{}, + StepInterval: 60, + }, + Filter: &FilterExpression{Expression: whereClause}, + } + + urlData := URLShareableCompositeQuery{ + QueryType: string(v3.QueryTypeBuilder), + Builder: URLShareableBuilderQuery{ + QueryData: []LinkQuery{ + linkQuery, + }, + QueryFormulas: make([]string, 0), + }, + } + + data, _ := json.Marshal(urlData) + compositeQuery := url.QueryEscape(url.QueryEscape(string(data))) + + optionsData, _ := json.Marshal(options) + urlEncodedOptions := url.QueryEscape(string(optionsData)) + + return fmt.Sprintf("compositeQuery=%s&timeRange=%s&startTime=%d&endTime=%d&options=%s", compositeQuery, urlEncodedTimeRange, tr.Start, tr.End, urlEncodedOptions) +} diff --git a/pkg/contextlinks/types.go b/pkg/contextlinks/types.go new file mode 100644 index 000000000000..bfc1ab6abbe0 --- /dev/null +++ b/pkg/contextlinks/types.go @@ -0,0 +1,40 @@ +package contextlinks + +import v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" + +// TODO(srikanthccv): Fix the URL management +type URLShareableTimeRange struct { + Start int64 `json:"start"` + End int64 `json:"end"` + PageSize int64 `json:"pageSize"` +} + +type FilterExpression struct { + Expression string `json:"expression,omitempty"` +} + +type Aggregation struct { + Expression string `json:"expression,omitempty"` +} + +type LinkQuery struct { + v3.BuilderQuery + Filter *FilterExpression `json:"filter,omitempty"` + Aggregations []*Aggregation `json:"aggregations,omitempty"` +} + +type URLShareableBuilderQuery struct { + QueryData []LinkQuery `json:"queryData"` + QueryFormulas []string `json:"queryFormulas"` +} + +type URLShareableCompositeQuery struct { + QueryType string `json:"queryType"` + Builder URLShareableBuilderQuery `json:"builder"` +} + +type URLShareableOptions struct { + MaxLines int `json:"maxLines"` + Format string `json:"format"` + SelectColumns []v3.AttributeKey `json:"selectColumns"` +} diff --git a/pkg/querier/api.go b/pkg/querier/api.go index 05a26581ad2d..d4aa6e376752 100644 --- a/pkg/querier/api.go +++ b/pkg/querier/api.go @@ -1,10 +1,13 @@ package querier import ( + "context" "encoding/json" "net/http" + "regexp" "runtime/debug" + "github.com/SigNoz/signoz/pkg/analytics" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/http/render" @@ -14,12 +17,13 @@ import ( ) type API struct { - set factory.ProviderSettings - querier Querier + set factory.ProviderSettings + analytics analytics.Analytics + querier Querier } -func NewAPI(set factory.ProviderSettings, querier Querier) *API { - return &API{set: set, querier: querier} +func NewAPI(set factory.ProviderSettings, querier Querier, analytics analytics.Analytics) *API { + return &API{set: set, querier: querier, analytics: analytics} } func (a *API) QueryRange(rw http.ResponseWriter, req *http.Request) { @@ -76,5 +80,91 @@ func (a *API) QueryRange(rw http.ResponseWriter, req *http.Request) { return } + a.logEvent(req.Context(), req.Header.Get("Referer"), queryRangeResponse.QBEvent) + render.Success(rw, http.StatusOK, queryRangeResponse) } + +func (a *API) logEvent(ctx context.Context, referrer string, event *qbtypes.QBEvent) { + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + a.set.Logger.DebugContext(ctx, "couldn't get claims from context") + return + } + + if !(event.LogsUsed || event.MetricsUsed || event.TracesUsed) { + a.set.Logger.DebugContext(ctx, "no data source in request, dubious?") + return + } + + properties := map[string]any{ + "version": event.Version, + "logs_used": event.LogsUsed, + "traces_used": event.TracesUsed, + "metrics_used": event.MetricsUsed, + "filter_applied": event.FilterApplied, + "group_by_applied": event.GroupByApplied, + "query_type": event.QueryType, + "panel_type": event.PanelType, + "number_of_queries": event.NumberOfQueries, + } + + if referrer == "" { + a.set.Logger.DebugContext(ctx, "no referrer, we don't ball non-UI requests") + return + } + + properties["referrer"] = referrer + + logsExplorerMatched, _ := regexp.MatchString(`/logs/logs-explorer(?:\?.*)?$`, referrer) + traceExplorerMatched, _ := regexp.MatchString(`/traces-explorer(?:\?.*)?$`, referrer) + metricsExplorerMatched, _ := regexp.MatchString(`/metrics-explorer/explorer(?:\?.*)?$`, referrer) + dashboardMatched, _ := regexp.MatchString(`/dashboard/[a-zA-Z0-9\-]+/(new|edit)(?:\?.*)?$`, referrer) + alertMatched, _ := regexp.MatchString(`/alerts/(new|edit)(?:\?.*)?$`, referrer) + + switch { + case dashboardMatched: + properties["module_name"] = "dashboard" + case alertMatched: + properties["module_name"] = "rule" + case metricsExplorerMatched: + properties["module_name"] = "metrics-explorer" + case logsExplorerMatched: + properties["module_name"] = "logs-explorer" + case traceExplorerMatched: + properties["module_name"] = "traces-explorer" + default: + a.set.Logger.DebugContext(ctx, "nothing matches referrer", "referrer", referrer) + return + } + + if dashboardMatched { + if dashboardIDRegex, err := regexp.Compile(`/dashboard/([a-f0-9\-]+)/`); err == nil { + if matches := dashboardIDRegex.FindStringSubmatch(referrer); len(matches) > 1 { + properties["dashboard_id"] = matches[1] + } + } + + if widgetIDRegex, err := regexp.Compile(`widgetId=([a-f0-9\-]+)`); err == nil { + if matches := widgetIDRegex.FindStringSubmatch(referrer); len(matches) > 1 { + properties["widget_id"] = matches[1] + } + } + } + + if alertMatched { + if alertIDRegex, err := regexp.Compile(`ruleId=(\d+)`); err == nil { + if matches := alertIDRegex.FindStringSubmatch(referrer); len(matches) > 1 { + properties["rule_id"] = matches[1] + } + } + } + + a.set.Logger.DebugContext(ctx, "sending analytics events", "analytics.event.properties", properties) + + if !event.HasData { + a.analytics.TrackUser(ctx, claims.OrgID, claims.UserID, "Telemetry Query Returned Empty", properties) + return + } + a.analytics.TrackUser(ctx, claims.OrgID, claims.UserID, "Telemetry Query Returned Results", properties) +} diff --git a/pkg/querier/consume.go b/pkg/querier/consume.go index 3b1ab29efcd6..63ad7292687f 100644 --- a/pkg/querier/consume.go +++ b/pkg/querier/consume.go @@ -42,7 +42,7 @@ func consume(rows driver.Rows, kind qbtypes.RequestType, queryWindow *qbtypes.Ti payload, err = readAsTimeSeries(rows, queryWindow, step, queryName) case qbtypes.RequestTypeScalar: payload, err = readAsScalar(rows, queryName) - case qbtypes.RequestTypeRaw: + case qbtypes.RequestTypeRaw, qbtypes.RequestTypeTrace: payload, err = readAsRaw(rows, queryName) // TODO: add support for other request types } diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index c9c5457a69d6..ce6a7a770259 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -6,6 +6,7 @@ import ( "log/slog" "slices" "strconv" + "strings" "sync" "time" @@ -111,10 +112,16 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype if tmplVars == nil { tmplVars = make(map[string]qbtypes.VariableItem) } + event := &qbtypes.QBEvent{ + Version: "v5", + NumberOfQueries: len(req.CompositeQuery.Queries), + PanelType: req.RequestType.StringValue(), + } // First pass: collect all metric names that need temporality metricNames := make([]string, 0) for idx, query := range req.CompositeQuery.Queries { + event.QueryType = query.Type.StringValue() if query.Type == qbtypes.QueryTypeBuilder { if spec, ok := query.Spec.(qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]); ok { for _, agg := range spec.Aggregations { @@ -128,6 +135,9 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype // allowed, we override it. switch spec := query.Spec.(type) { case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]: + event.TracesUsed = true + event.FilterApplied = spec.Filter != nil && spec.Filter.Expression != "" + event.GroupByApplied = len(spec.GroupBy) > 0 if spec.StepInterval.Seconds() == 0 { spec.StepInterval = qbtypes.Step{ Duration: time.Second * time.Duration(querybuilder.RecommendedStepInterval(req.Start, req.End)), @@ -140,6 +150,9 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype } req.CompositeQuery.Queries[idx].Spec = spec case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]: + event.LogsUsed = true + event.FilterApplied = spec.Filter != nil && spec.Filter.Expression != "" + event.GroupByApplied = len(spec.GroupBy) > 0 if spec.StepInterval.Seconds() == 0 { spec.StepInterval = qbtypes.Step{ Duration: time.Second * time.Duration(querybuilder.RecommendedStepInterval(req.Start, req.End)), @@ -152,6 +165,9 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype } req.CompositeQuery.Queries[idx].Spec = spec case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]: + event.MetricsUsed = true + event.FilterApplied = spec.Filter != nil && spec.Filter.Expression != "" + event.GroupByApplied = len(spec.GroupBy) > 0 if spec.StepInterval.Seconds() == 0 { spec.StepInterval = qbtypes.Step{ Duration: time.Second * time.Duration(querybuilder.RecommendedStepIntervalForMetric(req.Start, req.End)), @@ -166,6 +182,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype req.CompositeQuery.Queries[idx].Spec = spec } } else if query.Type == qbtypes.QueryTypePromQL { + event.MetricsUsed = true switch spec := query.Spec.(type) { case qbtypes.PromQuery: if spec.Step.Seconds() == 0 { @@ -175,6 +192,15 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype } req.CompositeQuery.Queries[idx].Spec = spec } + } else if query.Type == qbtypes.QueryTypeClickHouseSQL { + switch spec := query.Spec.(type) { + case qbtypes.ClickHouseQuery: + if strings.TrimSpace(spec.Query) != "" { + event.MetricsUsed = strings.Contains(spec.Query, "signoz_metrics") + event.LogsUsed = strings.Contains(spec.Query, "signoz_logs") + event.TracesUsed = strings.Contains(spec.Query, "signoz_traces") + } + } } } @@ -247,14 +273,56 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype } } } - return q.run(ctx, orgID, queries, req, steps) + qbResp, qbErr := q.run(ctx, orgID, queries, req, steps, event) + if qbResp != nil { + qbResp.QBEvent = event + } + return qbResp, qbErr } -func (q *querier) run(ctx context.Context, orgID valuer.UUID, qs map[string]qbtypes.Query, req *qbtypes.QueryRangeRequest, steps map[string]qbtypes.Step) (*qbtypes.QueryRangeResponse, error) { +func (q *querier) run( + ctx context.Context, + orgID valuer.UUID, + qs map[string]qbtypes.Query, + req *qbtypes.QueryRangeRequest, + steps map[string]qbtypes.Step, + qbEvent *qbtypes.QBEvent, +) (*qbtypes.QueryRangeResponse, error) { results := make(map[string]any) warnings := make([]string, 0) stats := qbtypes.ExecStats{} + hasData := func(result *qbtypes.Result) bool { + if result == nil || result.Value == nil { + return false + } + switch result.Type { + case qbtypes.RequestTypeScalar: + if val, ok := result.Value.(*qbtypes.ScalarData); ok { + return len(val.Data) != 0 + } + case qbtypes.RequestTypeRaw: + if val, ok := result.Value.(*qbtypes.RawData); ok { + return len(val.Rows) != 0 + } + case qbtypes.RequestTypeTimeSeries: + if val, ok := result.Value.(*qbtypes.TimeSeriesData); ok { + if len(val.Aggregations) != 0 { + anyNonEmpty := false + for _, aggBucket := range val.Aggregations { + if len(aggBucket.Series) != 0 { + anyNonEmpty = true + break + } + } + return anyNonEmpty + } + return false + } + } + return false + } + for name, query := range qs { // Skip cache if NoCache is set, or if cache is not available if req.NoCache || q.bucketCache == nil || query.Fingerprint() == "" { @@ -264,6 +332,7 @@ func (q *querier) run(ctx context.Context, orgID valuer.UUID, qs map[string]qbty q.logger.InfoContext(ctx, "no bucket cache or fingerprint, executing query", "fingerprint", query.Fingerprint()) } result, err := query.Execute(ctx) + qbEvent.HasData = qbEvent.HasData || hasData(result) if err != nil { return nil, err } @@ -274,6 +343,7 @@ func (q *querier) run(ctx context.Context, orgID valuer.UUID, qs map[string]qbty stats.DurationMS += result.Stats.DurationMS } else { result, err := q.executeWithCache(ctx, orgID, query, steps[name], req.NoCache) + qbEvent.HasData = qbEvent.HasData || hasData(result) if err != nil { return nil, err } diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 73cd2dbbff96..17e1baf63220 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -38,6 +38,7 @@ import ( jsoniter "github.com/json-iterator/go" _ "github.com/mattn/go-sqlite3" + "github.com/SigNoz/signoz/pkg/contextlinks" traceFunnelsModule "github.com/SigNoz/signoz/pkg/modules/tracefunnel" "github.com/SigNoz/signoz/pkg/query-service/agentConf" "github.com/SigNoz/signoz/pkg/query-service/app/cloudintegrations" @@ -55,7 +56,6 @@ import ( tracesV3 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v3" tracesV4 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v4" "github.com/SigNoz/signoz/pkg/query-service/constants" - "github.com/SigNoz/signoz/pkg/query-service/contextlinks" v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" "github.com/SigNoz/signoz/pkg/query-service/postprocess" "github.com/SigNoz/signoz/pkg/types" @@ -1073,6 +1073,7 @@ func (aH *APIHandler) getRuleStateHistoryTopContributors(w http.ResponseWriter, if err != nil { continue } + // TODO(srikanthccv): fix the links here after first QB milestone filterItems, groupBy, keys := aH.metaForLinks(r.Context(), rule) newFilters := contextlinks.PrepareFilters(lbls, filterItems, groupBy, keys) end := time.Unix(params.End/1000, 0) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 14bb1fd46721..e1e3f3448055 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -118,7 +118,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz, jwt *authtypes.JWT) LicensingAPI: nooplicensing.NewLicenseAPI(), FieldsAPI: fields.NewAPI(signoz.Instrumentation.ToProviderSettings(), signoz.TelemetryStore), Signoz: signoz, - QuerierAPI: querierAPI.NewAPI(signoz.Instrumentation.ToProviderSettings(), signoz.Querier), + QuerierAPI: querierAPI.NewAPI(signoz.Instrumentation.ToProviderSettings(), signoz.Querier, signoz.Analytics), }) if err != nil { return nil, err diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index bc090621d167..f85c276874f1 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -1459,28 +1459,6 @@ type MetricMetadataResponse struct { Temporality string `json:"temporality"` } -type URLShareableTimeRange struct { - Start int64 `json:"start"` - End int64 `json:"end"` - PageSize int64 `json:"pageSize"` -} - -type URLShareableBuilderQuery struct { - QueryData []BuilderQuery `json:"queryData"` - QueryFormulas []string `json:"queryFormulas"` -} - -type URLShareableCompositeQuery struct { - QueryType string `json:"queryType"` - Builder URLShareableBuilderQuery `json:"builder"` -} - -type URLShareableOptions struct { - MaxLines int `json:"maxLines"` - Format string `json:"format"` - SelectColumns []AttributeKey `json:"selectColumns"` -} - type QBOptions struct { GraphLimitQtype string IsLivetailQuery bool diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index beafe84a96f0..ec7d511c14c8 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -12,12 +12,13 @@ import ( "go.uber.org/zap" + "github.com/SigNoz/signoz/pkg/contextlinks" "github.com/SigNoz/signoz/pkg/query-service/common" - "github.com/SigNoz/signoz/pkg/query-service/contextlinks" "github.com/SigNoz/signoz/pkg/query-service/model" "github.com/SigNoz/signoz/pkg/query-service/postprocess" "github.com/SigNoz/signoz/pkg/transition" ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" "github.com/SigNoz/signoz/pkg/query-service/app/querier" @@ -200,18 +201,25 @@ func (r *ThresholdRule) prepareQueryRangeV5(ts time.Time) (*qbtypes.QueryRangeRe startTs, endTs := r.Timestamps(ts) start, end := startTs.UnixMilli(), endTs.UnixMilli() - return &qbtypes.QueryRangeRequest{ + req := &qbtypes.QueryRangeRequest{ Start: uint64(start), End: uint64(end), RequestType: qbtypes.RequestTypeTimeSeries, CompositeQuery: qbtypes.CompositeQuery{ - Queries: r.Condition().CompositeQuery.Queries, + Queries: make([]qbtypes.QueryEnvelope, 0), }, NoCache: true, - }, nil + } + copy(r.Condition().CompositeQuery.Queries, req.CompositeQuery.Queries) + return req, nil } func (r *ThresholdRule) prepareLinksToLogs(ts time.Time, lbls labels.Labels) string { + + if r.version == "v5" { + return r.prepareLinksToLogsV5(ts, lbls) + } + selectedQuery := r.GetSelectedQuery() qr, err := r.prepareQueryRange(ts) @@ -246,6 +254,11 @@ func (r *ThresholdRule) prepareLinksToLogs(ts time.Time, lbls labels.Labels) str } func (r *ThresholdRule) prepareLinksToTraces(ts time.Time, lbls labels.Labels) string { + + if r.version == "v5" { + return r.prepareLinksToTracesV5(ts, lbls) + } + selectedQuery := r.GetSelectedQuery() qr, err := r.prepareQueryRange(ts) @@ -279,6 +292,86 @@ func (r *ThresholdRule) prepareLinksToTraces(ts time.Time, lbls labels.Labels) s return contextlinks.PrepareLinksToTraces(start, end, filterItems) } +func (r *ThresholdRule) prepareLinksToLogsV5(ts time.Time, lbls labels.Labels) string { + selectedQuery := r.GetSelectedQuery() + + qr, err := r.prepareQueryRangeV5(ts) + if err != nil { + return "" + } + start := time.UnixMilli(int64(qr.Start)) + end := time.UnixMilli(int64(qr.End)) + + // TODO(srikanthccv): handle formula queries + if selectedQuery < "A" || selectedQuery > "Z" { + return "" + } + + var q qbtypes.QueryBuilderQuery[qbtypes.LogAggregation] + + for _, query := range r.ruleCondition.CompositeQuery.Queries { + if query.Type == qbtypes.QueryTypeBuilder { + switch spec := query.Spec.(type) { + case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]: + q = spec + } + } + } + + if q.Signal != telemetrytypes.SignalLogs { + return "" + } + + filterExpr := "" + if q.Filter != nil && q.Filter.Expression != "" { + filterExpr = q.Filter.Expression + } + + whereClause := contextlinks.PrepareFilterExpression(lbls.Map(), filterExpr, q.GroupBy) + + return contextlinks.PrepareLinksToLogsV5(start, end, whereClause) +} + +func (r *ThresholdRule) prepareLinksToTracesV5(ts time.Time, lbls labels.Labels) string { + selectedQuery := r.GetSelectedQuery() + + qr, err := r.prepareQueryRangeV5(ts) + if err != nil { + return "" + } + start := time.UnixMilli(int64(qr.Start)) + end := time.UnixMilli(int64(qr.End)) + + // TODO(srikanthccv): handle formula queries + if selectedQuery < "A" || selectedQuery > "Z" { + return "" + } + + var q qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation] + + for _, query := range r.ruleCondition.CompositeQuery.Queries { + if query.Type == qbtypes.QueryTypeBuilder { + switch spec := query.Spec.(type) { + case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]: + q = spec + } + } + } + + if q.Signal != telemetrytypes.SignalTraces { + return "" + } + + filterExpr := "" + if q.Filter != nil && q.Filter.Expression != "" { + filterExpr = q.Filter.Expression + } + + whereClause := contextlinks.PrepareFilterExpression(lbls.Map(), filterExpr, q.GroupBy) + + return contextlinks.PrepareLinksToTracesV5(start, end, whereClause) +} + func (r *ThresholdRule) GetSelectedQuery() string { return r.ruleCondition.GetSelectedQueryName() } @@ -399,17 +492,20 @@ func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUI } var results []*v3.Result - var queryErrors map[string]error v5Result, err := r.querierV5.QueryRange(ctx, orgID, params) + if err != nil { + zap.L().Error("failed to get alert query result", zap.String("rule", r.Name()), zap.Error(err), zap.Error(err)) + return nil, fmt.Errorf("internal error while querying") + } + data, ok := v5Result.Data.(struct { Results []any `json:"results"` Warnings []string `json:"warnings"` }) if !ok { - return nil, fmt.Errorf("upexpected result from v5 querier") } @@ -421,11 +517,6 @@ func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUI } } - if err != nil { - zap.L().Error("failed to get alert query result", zap.String("rule", r.Name()), zap.Error(err), zap.Any("errors", queryErrors)) - return nil, fmt.Errorf("internal error while querying") - } - selectedQuery := r.GetSelectedQuery() var queryResult *v3.Result @@ -476,8 +567,10 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er var err error if r.version == "v5" { + zap.L().Info("running v5 query") res, err = r.buildAndRunQueryV5(ctx, r.orgID, ts) } else { + zap.L().Info("running v4 query") res, err = r.buildAndRunQuery(ctx, r.orgID, ts) } diff --git a/pkg/telemetrytraces/statement_builder.go b/pkg/telemetrytraces/statement_builder.go index d231bdfd1800..8a61af8e3a23 100644 --- a/pkg/telemetrytraces/statement_builder.go +++ b/pkg/telemetrytraces/statement_builder.go @@ -103,6 +103,8 @@ func (b *traceQueryStatementBuilder) Build( return b.buildTimeSeriesQuery(ctx, q, query, start, end, keys, variables) case qbtypes.RequestTypeScalar: return b.buildScalarQuery(ctx, q, query, start, end, keys, variables, false, false) + case qbtypes.RequestTypeTrace: + return b.buildTraceExplorerQuery(ctx, q, query, start, end, keys, variables) } return nil, fmt.Errorf("unsupported request type: %s", requestType) @@ -338,6 +340,114 @@ func (b *traceQueryStatementBuilder) buildListQuery( }, nil } +func (b *traceQueryStatementBuilder) buildTraceExplorerQuery( + ctx context.Context, + _ *sqlbuilder.SelectBuilder, + query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], + start, end uint64, + keys map[string][]*telemetrytypes.TelemetryFieldKey, + variables map[string]qbtypes.VariableItem, +) (*qbtypes.Statement, error) { + + startBucket := start/querybuilder.NsToSeconds - querybuilder.BucketAdjustment + endBucket := end / querybuilder.NsToSeconds + + distSB := sqlbuilder.NewSelectBuilder() + distSB.Select("trace_id") + distSB.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName)) + + var ( + cteFragments []string + cteArgs [][]any + ) + + if frag, args, err := b.maybeAttachResourceFilter(ctx, distSB, query, start, end, variables); err != nil { + return nil, err + } else if frag != "" { + cteFragments = append(cteFragments, frag) + cteArgs = append(cteArgs, args) + } + + // Add filter conditions + warnings, err := b.addFilterCondition(ctx, distSB, start, end, query, keys, variables) + if err != nil { + return nil, err + } + + distSQL, distArgs := distSB.BuildWithFlavor(sqlbuilder.ClickHouse) + + cteFragments = append(cteFragments, fmt.Sprintf("__toe AS (%s)", distSQL)) + cteArgs = append(cteArgs, distArgs) + + // Build the inner subquery for root spans + innerSB := sqlbuilder.NewSelectBuilder() + innerSB.Select("trace_id", "duration_nano", sqlbuilder.Escape("resource_string_service$$name as `service.name`"), "name") + innerSB.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName)) + innerSB.Where("parent_span_id = ''") + + // Add time filter to inner query + innerSB.Where( + innerSB.GE("timestamp", fmt.Sprintf("%d", start)), + innerSB.L("timestamp", fmt.Sprintf("%d", end)), + innerSB.GE("ts_bucket_start", startBucket), + innerSB.LE("ts_bucket_start", endBucket)) + + // order by duration and limit 1 per trace + innerSB.OrderBy("duration_nano DESC") + innerSB.SQL("LIMIT 1 BY trace_id") + + innerSQL, innerArgs := innerSB.BuildWithFlavor(sqlbuilder.ClickHouse) + + cteFragments = append(cteFragments, fmt.Sprintf("__toe_duration_sorted AS (%s)", innerSQL)) + cteArgs = append(cteArgs, innerArgs) + + // main query that joins everything + mainSB := sqlbuilder.NewSelectBuilder() + mainSB.Select( + "__toe_duration_sorted.`service.name` AS `service.name`", + "__toe_duration_sorted.name AS `name`", + "count() AS span_count", + "__toe_duration_sorted.duration_nano AS `duration_nano`", + "__toe_duration_sorted.trace_id AS `trace_id`", + ) + + // Join the distributed table with the inner subquery + mainSB.SQL("FROM __toe") + mainSB.SQL("INNER JOIN __toe_duration_sorted") + mainSB.SQL("ON __toe.trace_id = __toe_duration_sorted.trace_id") + + // Group by trace-level fields + mainSB.GroupBy("trace_id", "duration_nano", "name", "`service.name`") + + // order by duration only supported for now + mainSB.OrderBy("duration_nano DESC") + + // Limit by trace_id to ensure one row per trace + mainSB.SQL("LIMIT 1 BY trace_id") + + if query.Limit > 0 { + mainSB.Limit(query.Limit) + } else { + mainSB.Limit(100) + } + + if query.Offset > 0 { + mainSB.Offset(query.Offset) + } + + mainSQL, mainArgs := mainSB.BuildWithFlavor(sqlbuilder.ClickHouse) + + // combine it all together: WITH … SELECT … + finalSQL := querybuilder.CombineCTEs(cteFragments) + mainSQL + finalArgs := querybuilder.PrependArgs(cteArgs, mainArgs) + + return &qbtypes.Statement{ + Query: finalSQL, + Args: finalArgs, + Warnings: warnings, + }, nil +} + func (b *traceQueryStatementBuilder) buildTimeSeriesQuery( ctx context.Context, sb *sqlbuilder.SelectBuilder, diff --git a/pkg/telemetrytraces/stmt_builder_test.go b/pkg/telemetrytraces/stmt_builder_test.go index f81823cc1932..d50c9d9f9dc2 100644 --- a/pkg/telemetrytraces/stmt_builder_test.go +++ b/pkg/telemetrytraces/stmt_builder_test.go @@ -458,3 +458,65 @@ func TestStatementBuilderListQuery(t *testing.T) { }) } } + +func TestStatementBuilderTraceQuery(t *testing.T) { + cases := []struct { + name string + requestType qbtypes.RequestType + query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation] + expected qbtypes.Statement + expectedErr error + }{ + { + name: "List query with mat selected fields", + requestType: qbtypes.RequestTypeTrace, + query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{ + Signal: telemetrytypes.SignalTraces, + Filter: &qbtypes.Filter{ + Expression: "service.name = 'redis-manual'", + }, + Limit: 10, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY __toe.trace_id, __toe.duration_nano, __toe.name, __toe.`service.name` ORDER BY __toe.duration_nano DESC LIMIT 1 BY __toe.trace_id LIMIT ?", + Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + } + + fm := NewFieldMapper() + cb := NewConditionBuilder(fm) + mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() + aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + + resourceFilterStmtBuilder := resourceFilterStmtBuilder() + + statementBuilder := NewTraceQueryStatementBuilder( + instrumentationtest.New().ToProviderSettings(), + mockMetadataStore, + fm, + cb, + resourceFilterStmtBuilder, + aggExprRewriter, + nil, + ) + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil) + + if c.expectedErr != nil { + require.Error(t, err) + require.Contains(t, err.Error(), c.expectedErr.Error()) + } else { + require.NoError(t, err) + require.Equal(t, c.expected.Query, q.Query) + require.Equal(t, c.expected.Args, q.Args) + require.Equal(t, c.expected.Warnings, q.Warnings) + } + }) + } +} diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/request_type.go b/pkg/types/querybuildertypes/querybuildertypesv5/request_type.go index 81ca5c322f5c..eea59ce406b0 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/request_type.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/request_type.go @@ -14,6 +14,8 @@ var ( RequestTypeTimeSeries = RequestType{valuer.NewString("time_series")} // [][]any, SQL result set, but paginated, example: list view RequestTypeRaw = RequestType{valuer.NewString("raw")} + // [][]any, Specialized SQL result set, paginated + RequestTypeTrace = RequestType{valuer.NewString("trace")} // []Bucket (struct{Lower,Upper,Count float64}), example: histogram RequestTypeDistribution = RequestType{valuer.NewString("distribution")} ) diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/resp.go b/pkg/types/querybuildertypes/querybuildertypesv5/resp.go index 6f1f85bcc997..51919b91e0d3 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/resp.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/resp.go @@ -13,10 +13,25 @@ import ( "github.com/SigNoz/signoz/pkg/valuer" ) +type QBEvent struct { + Version string `json:"version"` + LogsUsed bool `json:"logs_used,omitempty"` + MetricsUsed bool `json:"metrics_used,omitempty"` + TracesUsed bool `json:"traces_used,omitempty"` + FilterApplied bool `json:"filter_applied,omitempty"` + GroupByApplied bool `json:"group_by_applied,omitempty"` + QueryType string `json:"query_type,omitempty"` + PanelType string `json:"panel_type,omitempty"` + NumberOfQueries int `json:"number_of_queries,omitempty"` + HasData bool `json:"-"` +} + type QueryRangeResponse struct { Type RequestType `json:"type"` Data any `json:"data"` Meta ExecStats `json:"meta"` + + QBEvent *QBEvent `json:"-"` } type TimeSeriesData struct { diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/validation.go b/pkg/types/querybuildertypes/querybuildertypesv5/validation.go index e413f195cc5b..d8679a49c121 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/validation.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/validation.go @@ -108,7 +108,7 @@ func (q *QueryBuilderQuery[T]) Validate(requestType RequestType) error { } // Validate aggregations only for non-raw request types - if requestType != RequestTypeRaw { + if requestType != RequestTypeRaw && requestType != RequestTypeTrace { if err := q.validateAggregations(); err != nil { return err } @@ -129,7 +129,7 @@ func (q *QueryBuilderQuery[T]) Validate(requestType RequestType) error { return err } - if requestType != RequestTypeRaw && len(q.Aggregations) > 0 { + if requestType != RequestTypeRaw && requestType != RequestTypeTrace && len(q.Aggregations) > 0 { if err := q.validateOrderByForAggregation(); err != nil { return err } @@ -139,7 +139,7 @@ func (q *QueryBuilderQuery[T]) Validate(requestType RequestType) error { } } - if requestType != RequestTypeRaw { + if requestType != RequestTypeRaw && requestType != RequestTypeTrace { if err := q.validateHaving(); err != nil { return err } @@ -440,7 +440,7 @@ func (r *QueryRangeRequest) Validate() error { // Validate request type switch r.RequestType { - case RequestTypeRaw, RequestTypeTimeSeries, RequestTypeScalar: + case RequestTypeRaw, RequestTypeTimeSeries, RequestTypeScalar, RequestTypeTrace: // Valid request types default: return errors.NewInvalidInputf(