diff --git a/pkg/apis/fields/api.go b/pkg/apis/fields/api.go index 0d4b20fef879..0bc2f6958e50 100644 --- a/pkg/apis/fields/api.go +++ b/pkg/apis/fields/api.go @@ -31,6 +31,7 @@ func NewAPI( telemetryStore, telemetrytraces.DBName, telemetrytraces.TagAttributesV2TableName, + telemetrytraces.SpanAttributesKeysTblName, telemetrytraces.SpanIndexV3TableName, telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName, @@ -39,6 +40,8 @@ func NewAPI( telemetrylogs.DBName, telemetrylogs.LogsV2TableName, telemetrylogs.TagAttributesV2TableName, + telemetrylogs.LogAttributeKeysTblName, + telemetrylogs.LogResourceKeysTblName, telemetrymetadata.DBName, telemetrymetadata.AttributesMetadataLocalTableName, ) diff --git a/pkg/querier/signozquerier/provider.go b/pkg/querier/signozquerier/provider.go index 9d4cb1dbdae4..999de129555c 100644 --- a/pkg/querier/signozquerier/provider.go +++ b/pkg/querier/signozquerier/provider.go @@ -50,6 +50,7 @@ func newProvider( telemetryStore, telemetrytraces.DBName, telemetrytraces.TagAttributesV2TableName, + telemetrytraces.SpanAttributesKeysTblName, telemetrytraces.SpanIndexV3TableName, telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName, @@ -58,6 +59,8 @@ func newProvider( telemetrylogs.DBName, telemetrylogs.LogsV2TableName, telemetrylogs.TagAttributesV2TableName, + telemetrylogs.LogAttributeKeysTblName, + telemetrylogs.LogResourceKeysTblName, telemetrymetadata.DBName, telemetrymetadata.AttributesMetadataLocalTableName, ) @@ -69,12 +72,13 @@ func newProvider( resourceFilterFieldMapper := resourcefilter.NewFieldMapper() resourceFilterConditionBuilder := resourcefilter.NewConditionBuilder(resourceFilterFieldMapper) resourceFilterStmtBuilder := resourcefilter.NewTraceResourceFilterStatementBuilder( + settings, resourceFilterFieldMapper, resourceFilterConditionBuilder, telemetryMetadataStore, ) - traceAggExprRewriter := querybuilder.NewAggExprRewriter(nil, traceFieldMapper, traceConditionBuilder, "", nil) + traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, "", nil) traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder( settings, telemetryMetadataStore, @@ -89,6 +93,7 @@ func newProvider( logFieldMapper := telemetrylogs.NewFieldMapper() logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper) logResourceFilterStmtBuilder := resourcefilter.NewLogResourceFilterStatementBuilder( + settings, resourceFilterFieldMapper, resourceFilterConditionBuilder, telemetryMetadataStore, @@ -97,6 +102,7 @@ func newProvider( telemetrylogs.GetBodyJSONKey, ) logAggExprRewriter := querybuilder.NewAggExprRewriter( + settings, telemetrylogs.DefaultFullTextColumn, logFieldMapper, logConditionBuilder, diff --git a/pkg/querybuilder/agg_rewrite.go b/pkg/querybuilder/agg_rewrite.go index 7f49aafc2295..9f948e14dd48 100644 --- a/pkg/querybuilder/agg_rewrite.go +++ b/pkg/querybuilder/agg_rewrite.go @@ -3,10 +3,12 @@ package querybuilder import ( "context" "fmt" + "log/slog" "strings" chparser "github.com/AfterShip/clickhouse-sql-parser/parser" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/factory" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -14,6 +16,7 @@ import ( ) type aggExprRewriter struct { + logger *slog.Logger fullTextColumn *telemetrytypes.TelemetryFieldKey fieldMapper qbtypes.FieldMapper conditionBuilder qbtypes.ConditionBuilder @@ -24,13 +27,17 @@ type aggExprRewriter struct { var _ qbtypes.AggExprRewriter = (*aggExprRewriter)(nil) func NewAggExprRewriter( + settings factory.ProviderSettings, fullTextColumn *telemetrytypes.TelemetryFieldKey, fieldMapper qbtypes.FieldMapper, conditionBuilder qbtypes.ConditionBuilder, jsonBodyPrefix string, jsonKeyToKey qbtypes.JsonKeyToFieldFunc, ) *aggExprRewriter { + set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/agg_rewrite") + return &aggExprRewriter{ + logger: set.Logger(), fullTextColumn: fullTextColumn, fieldMapper: fieldMapper, conditionBuilder: conditionBuilder, @@ -70,7 +77,7 @@ func (r *aggExprRewriter) Rewrite( return "", nil, errors.NewInternalf(errors.CodeInternal, "no SELECT items for %q", expr) } - visitor := newExprVisitor(keys, + visitor := newExprVisitor(r.logger, keys, r.fullTextColumn, r.fieldMapper, r.conditionBuilder, @@ -117,6 +124,7 @@ func (r *aggExprRewriter) RewriteMulti( // exprVisitor walks FunctionExpr nodes and applies the mappers. type exprVisitor struct { chparser.DefaultASTVisitor + logger *slog.Logger fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey fullTextColumn *telemetrytypes.TelemetryFieldKey fieldMapper qbtypes.FieldMapper @@ -129,6 +137,7 @@ type exprVisitor struct { } func newExprVisitor( + logger *slog.Logger, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, fullTextColumn *telemetrytypes.TelemetryFieldKey, fieldMapper qbtypes.FieldMapper, @@ -137,6 +146,7 @@ func newExprVisitor( jsonKeyToKey qbtypes.JsonKeyToFieldFunc, ) *exprVisitor { return &exprVisitor{ + logger: logger, fieldKeys: fieldKeys, fullTextColumn: fullTextColumn, fieldMapper: fieldMapper, @@ -183,6 +193,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error { whereClause, err := PrepareWhereClause( origPred, FilterExprVisitorOpts{ + Logger: v.logger, FieldKeys: v.fieldKeys, FieldMapper: v.fieldMapper, ConditionBuilder: v.conditionBuilder, diff --git a/pkg/querybuilder/resourcefilter/statement_builder.go b/pkg/querybuilder/resourcefilter/statement_builder.go index 1cedb6944723..f0ac3d78483f 100644 --- a/pkg/querybuilder/resourcefilter/statement_builder.go +++ b/pkg/querybuilder/resourcefilter/statement_builder.go @@ -3,8 +3,10 @@ package resourcefilter import ( "context" "fmt" + "log/slog" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" @@ -34,6 +36,7 @@ var signalConfigs = map[telemetrytypes.Signal]signalConfig{ // Generic resource filter statement builder type resourceFilterStatementBuilder[T any] struct { + logger *slog.Logger fieldMapper qbtypes.FieldMapper conditionBuilder qbtypes.ConditionBuilder metadataStore telemetrytypes.MetadataStore @@ -52,11 +55,14 @@ var ( // Constructor functions func NewTraceResourceFilterStatementBuilder( + settings factory.ProviderSettings, fieldMapper qbtypes.FieldMapper, conditionBuilder qbtypes.ConditionBuilder, metadataStore telemetrytypes.MetadataStore, ) *resourceFilterStatementBuilder[qbtypes.TraceAggregation] { + set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter") return &resourceFilterStatementBuilder[qbtypes.TraceAggregation]{ + logger: set.Logger(), fieldMapper: fieldMapper, conditionBuilder: conditionBuilder, metadataStore: metadataStore, @@ -65,6 +71,7 @@ func NewTraceResourceFilterStatementBuilder( } func NewLogResourceFilterStatementBuilder( + settings factory.ProviderSettings, fieldMapper qbtypes.FieldMapper, conditionBuilder qbtypes.ConditionBuilder, metadataStore telemetrytypes.MetadataStore, @@ -72,7 +79,9 @@ func NewLogResourceFilterStatementBuilder( jsonBodyPrefix string, jsonKeyToKey qbtypes.JsonKeyToFieldFunc, ) *resourceFilterStatementBuilder[qbtypes.LogAggregation] { + set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter") return &resourceFilterStatementBuilder[qbtypes.LogAggregation]{ + logger: set.Logger(), fieldMapper: fieldMapper, conditionBuilder: conditionBuilder, metadataStore: metadataStore, @@ -148,6 +157,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions( // warnings would be encountered as part of the main condition already filterWhereClause, err := querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fieldMapper, ConditionBuilder: b.conditionBuilder, FieldKeys: keys, diff --git a/pkg/querybuilder/where_clause_visitor.go b/pkg/querybuilder/where_clause_visitor.go index 09b35c3f339f..a14f4996bd4c 100644 --- a/pkg/querybuilder/where_clause_visitor.go +++ b/pkg/querybuilder/where_clause_visitor.go @@ -3,6 +3,7 @@ package querybuilder import ( "context" "fmt" + "log/slog" "strconv" "strings" @@ -20,6 +21,7 @@ var searchTroubleshootingGuideURL = "https://signoz.io/docs/userguide/search-tro // filterExpressionVisitor implements the FilterQueryVisitor interface // to convert the parsed filter expressions into ClickHouse WHERE clause type filterExpressionVisitor struct { + logger *slog.Logger fieldMapper qbtypes.FieldMapper conditionBuilder qbtypes.ConditionBuilder warnings []string @@ -41,6 +43,7 @@ type filterExpressionVisitor struct { } type FilterExprVisitorOpts struct { + Logger *slog.Logger FieldMapper qbtypes.FieldMapper ConditionBuilder qbtypes.ConditionBuilder FieldKeys map[string][]*telemetrytypes.TelemetryFieldKey @@ -58,6 +61,7 @@ type FilterExprVisitorOpts struct { // newFilterExpressionVisitor creates a new filterExpressionVisitor func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVisitor { return &filterExpressionVisitor{ + logger: opts.Logger, fieldMapper: opts.FieldMapper, conditionBuilder: opts.ConditionBuilder, fieldKeys: opts.FieldKeys, @@ -786,15 +790,35 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any { } if len(fieldKeysForName) > 1 && !v.keysWithWarnings[keyName] { - v.mainWarnURL = "https://signoz.io/docs/userguide/field-context-data-types/" - // this is warning state, we must have a unambiguous key - v.warnings = append(v.warnings, fmt.Sprintf( - "key `%s` is ambiguous, found %d different combinations of field context / data type: %v", + warnMsg := fmt.Sprintf( + "Key `%s` is ambiguous, found %d different combinations of field context / data type: %v.", fieldKey.Name, len(fieldKeysForName), fieldKeysForName, - )) + ) + mixedFieldContext := map[string]bool{} + for _, item := range fieldKeysForName { + mixedFieldContext[item.FieldContext.StringValue()] = true + } + + if mixedFieldContext[telemetrytypes.FieldContextResource.StringValue()] && + mixedFieldContext[telemetrytypes.FieldContextAttribute.StringValue()] { + filteredKeys := []*telemetrytypes.TelemetryFieldKey{} + for _, item := range fieldKeysForName { + if item.FieldContext != telemetrytypes.FieldContextResource { + continue + } + filteredKeys = append(filteredKeys, item) + } + fieldKeysForName = filteredKeys + warnMsg += " " + "Using `resource` context by default. To query attributes explicitly, " + + fmt.Sprintf("use the fully qualified name (e.g., 'attribute.%s')", fieldKey.Name) + } + v.mainWarnURL = "https://signoz.io/docs/userguide/field-context-data-types/" + // this is warning state, we must have a unambiguous key + v.warnings = append(v.warnings, warnMsg) v.keysWithWarnings[keyName] = true + v.logger.Warn("ambiguous key", "field_key_name", fieldKey.Name) //nolint:sloglint } return fieldKeysForName diff --git a/pkg/telemetrylogs/filter_expr_logs_body_json_test.go b/pkg/telemetrylogs/filter_expr_logs_body_json_test.go index 3d3140768dc7..58a8c7ada827 100644 --- a/pkg/telemetrylogs/filter_expr_logs_body_json_test.go +++ b/pkg/telemetrylogs/filter_expr_logs_body_json_test.go @@ -4,6 +4,7 @@ import ( "fmt" "testing" + "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/huandu/go-sqlbuilder" @@ -19,6 +20,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) { keys := buildCompleteFieldKeyMap() opts := querybuilder.FilterExprVisitorOpts{ + Logger: instrumentationtest.New().Logger(), FieldMapper: fm, ConditionBuilder: cb, FieldKeys: keys, diff --git a/pkg/telemetrylogs/filter_expr_logs_test.go b/pkg/telemetrylogs/filter_expr_logs_test.go index 7096ee875c90..4e5f778b7350 100644 --- a/pkg/telemetrylogs/filter_expr_logs_test.go +++ b/pkg/telemetrylogs/filter_expr_logs_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/huandu/go-sqlbuilder" @@ -21,6 +22,7 @@ func TestFilterExprLogs(t *testing.T) { keys := buildCompleteFieldKeyMap() opts := querybuilder.FilterExprVisitorOpts{ + Logger: instrumentationtest.New().Logger(), FieldMapper: fm, ConditionBuilder: cb, FieldKeys: keys, diff --git a/pkg/telemetrylogs/statement_builder.go b/pkg/telemetrylogs/statement_builder.go index 105425007138..c9d2a1bb2cba 100644 --- a/pkg/telemetrylogs/statement_builder.go +++ b/pkg/telemetrylogs/statement_builder.go @@ -553,6 +553,7 @@ func (b *logQueryStatementBuilder) addFilterCondition( if query.Filter != nil && query.Filter.Expression != "" { // add filter expression preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fm, ConditionBuilder: b.cb, FieldKeys: keys, diff --git a/pkg/telemetrylogs/stmt_builder_test.go b/pkg/telemetrylogs/stmt_builder_test.go index 9993367edd0a..b6c3f9b432a0 100644 --- a/pkg/telemetrylogs/stmt_builder_test.go +++ b/pkg/telemetrylogs/stmt_builder_test.go @@ -27,6 +27,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation mockMetadataStore.KeysMap = keysMap return resourcefilter.NewLogResourceFilterStatementBuilder( + instrumentationtest.New().ToProviderSettings(), fm, cb, mockMetadataStore, @@ -119,7 +120,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) { mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() @@ -212,7 +213,7 @@ func TestStatementBuilderListQuery(t *testing.T) { mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() @@ -321,7 +322,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() diff --git a/pkg/telemetrylogs/tables.go b/pkg/telemetrylogs/tables.go index 1d0c35fbc662..8d674bf28a9f 100644 --- a/pkg/telemetrylogs/tables.go +++ b/pkg/telemetrylogs/tables.go @@ -6,4 +6,6 @@ const ( LogsV2LocalTableName = "logs_v2" TagAttributesV2TableName = "distributed_tag_attributes_v2" TagAttributesV2LocalTableName = "tag_attributes_v2" + LogAttributeKeysTblName = "distributed_logs_attribute_keys" + LogResourceKeysTblName = "distributed_logs_resource_keys" ) diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index 088a332561cf..e1e6c8ffc830 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "os" "slices" "strings" @@ -32,20 +31,23 @@ var ( ) type telemetryMetaStore struct { - logger *slog.Logger - telemetrystore telemetrystore.TelemetryStore - tracesDBName string - tracesFieldsTblName string - indexV3TblName string - metricsDBName string - metricsFieldsTblName string - meterDBName string - meterFieldsTblName string - logsDBName string - logsFieldsTblName string - logsV2TblName string - relatedMetadataDBName string - relatedMetadataTblName string + logger *slog.Logger + telemetrystore telemetrystore.TelemetryStore + tracesDBName string + tracesFieldsTblName string + spanAttributesKeysTblName string + indexV3TblName string + metricsDBName string + metricsFieldsTblName string + meterDBName string + meterFieldsTblName string + logsDBName string + logsFieldsTblName string + logAttributeKeysTblName string + logResourceKeysTblName string + logsV2TblName string + relatedMetadataDBName string + relatedMetadataTblName string fm qbtypes.FieldMapper conditionBuilder qbtypes.ConditionBuilder @@ -60,6 +62,7 @@ func NewTelemetryMetaStore( telemetrystore telemetrystore.TelemetryStore, tracesDBName string, tracesFieldsTblName string, + spanAttributesKeysTblName string, indexV3TblName string, metricsDBName string, metricsFieldsTblName string, @@ -68,26 +71,31 @@ func NewTelemetryMetaStore( logsDBName string, logsV2TblName string, logsFieldsTblName string, + logAttributeKeysTblName string, + logResourceKeysTblName string, relatedMetadataDBName string, relatedMetadataTblName string, ) telemetrytypes.MetadataStore { metadataSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetadata") t := &telemetryMetaStore{ - logger: metadataSettings.Logger(), - telemetrystore: telemetrystore, - tracesDBName: tracesDBName, - tracesFieldsTblName: tracesFieldsTblName, - indexV3TblName: indexV3TblName, - metricsDBName: metricsDBName, - metricsFieldsTblName: metricsFieldsTblName, - meterDBName: meterDBName, - meterFieldsTblName: meterFieldsTblName, - logsDBName: logsDBName, - logsV2TblName: logsV2TblName, - logsFieldsTblName: logsFieldsTblName, - relatedMetadataDBName: relatedMetadataDBName, - relatedMetadataTblName: relatedMetadataTblName, + logger: metadataSettings.Logger(), + telemetrystore: telemetrystore, + tracesDBName: tracesDBName, + tracesFieldsTblName: tracesFieldsTblName, + spanAttributesKeysTblName: spanAttributesKeysTblName, + indexV3TblName: indexV3TblName, + metricsDBName: metricsDBName, + metricsFieldsTblName: metricsFieldsTblName, + meterDBName: meterDBName, + meterFieldsTblName: meterFieldsTblName, + logsDBName: logsDBName, + logsV2TblName: logsV2TblName, + logsFieldsTblName: logsFieldsTblName, + logAttributeKeysTblName: logAttributeKeysTblName, + logResourceKeysTblName: logResourceKeysTblName, + relatedMetadataDBName: relatedMetadataDBName, + relatedMetadataTblName: relatedMetadataTblName, } fm := NewFieldMapper() @@ -136,14 +144,18 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key } - sb := sqlbuilder.Select("tag_key", "tag_type", "tag_data_type", ` - CASE - WHEN tag_type = 'spanfield' THEN 1 - WHEN tag_type = 'resource' THEN 2 - WHEN tag_type = 'scope' THEN 3 - WHEN tag_type = 'tag' THEN 4 - ELSE 5 - END as priority`).From(t.tracesDBName + "." + t.tracesFieldsTblName) + sb := sqlbuilder.Select( + "tagKey AS tag_key", + "tagType AS tag_type", + "dataType AS tag_data_type", + `CASE + // WHEN tagType = 'spanfield' THEN 1 + WHEN tagType = 'resource' THEN 2 + // WHEN tagType = 'scope' THEN 3 + WHEN tagType = 'tag' THEN 4 + ELSE 5 + END as priority`, + ).From(t.tracesDBName + "." + t.spanAttributesKeysTblName) var limit int searchTexts := []string{} @@ -152,19 +164,20 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector conds := []string{} for _, fieldKeySelector := range fieldKeySelectors { - if fieldKeySelector.StartUnixMilli != 0 { - conds = append(conds, sb.GE("unix_milli", fieldKeySelector.StartUnixMilli)) - } - if fieldKeySelector.EndUnixMilli != 0 { - conds = append(conds, sb.LE("unix_milli", fieldKeySelector.EndUnixMilli)) - } + // TODO(srikanthccv): support time filtering for span attribute keys + // if fieldKeySelector.StartUnixMilli != 0 { + // conds = append(conds, sb.GE("unix_milli", fieldKeySelector.StartUnixMilli)) + // } + // if fieldKeySelector.EndUnixMilli != 0 { + // conds = append(conds, sb.LE("unix_milli", fieldKeySelector.EndUnixMilli)) + // } // key part of the selector fieldKeyConds := []string{} if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact { - fieldKeyConds = append(fieldKeyConds, sb.E("tag_key", fieldKeySelector.Name)) + fieldKeyConds = append(fieldKeyConds, sb.E("tagKey", fieldKeySelector.Name)) } else { - fieldKeyConds = append(fieldKeyConds, sb.ILike("tag_key", "%"+escapeForLike(fieldKeySelector.Name)+"%")) + fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%")) } searchTexts = append(searchTexts, fieldKeySelector.Name) @@ -172,29 +185,25 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector dataTypes = append(dataTypes, fieldKeySelector.FieldDataType) } // now look at the field context - // we don't write most of intrinsic fields to tag attributes table - // for this reason we don't want to apply tag_type if the field context - // if not attribute or resource attribute + // we don't write most of intrinsic fields to keys table + // for this reason we don't want to apply tagType if the field context + // is not attribute or resource attribute if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified && (fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute || fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) { - fieldKeyConds = append(fieldKeyConds, sb.E("tag_type", fieldKeySelector.FieldContext.TagType())) + fieldKeyConds = append(fieldKeyConds, sb.E("tagType", fieldKeySelector.FieldContext.TagType())) } // now look at the field data type if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { - fieldKeyConds = append(fieldKeyConds, sb.E("tag_data_type", fieldKeySelector.FieldDataType.TagDataType())) + fieldKeyConds = append(fieldKeyConds, sb.E("dataType", fieldKeySelector.FieldDataType.TagDataType())) } conds = append(conds, sb.And(fieldKeyConds...)) limit += fieldKeySelector.Limit - if strings.TrimSpace(fieldKeySelector.Name) == "" { - sb.Limit(200) - } } sb.Where(sb.Or(conds...)) - sb.GroupBy("tag_key", "tag_type", "tag_data_type") - + sb.GroupBy("tagKey", "tagType", "dataType") if limit == 0 { limit = 1000 } @@ -347,89 +356,145 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key } - tblName := t.logsFieldsTblName - if os.Getenv("LOGS_TAG_ATTRS_KEYS_TABLE") != "" { - tblName = os.Getenv("LOGS_TAG_ATTRS_KEYS_TABLE") + // queries for both attribute and resource keys tables + var queries []string + var allArgs []any + + // tables to query based on field selectors + queryAttributeTable := false + queryResourceTable := false + + for _, selector := range fieldKeySelectors { + if selector.FieldContext == telemetrytypes.FieldContextUnspecified { + // unspecified context, query both tables + queryAttributeTable = true + queryResourceTable = true + break + } else if selector.FieldContext == telemetrytypes.FieldContextAttribute { + queryAttributeTable = true + } else if selector.FieldContext == telemetrytypes.FieldContextResource { + queryResourceTable = true + } } - sb := sqlbuilder.Select("tag_key", "tag_type", "tag_data_type", ` - CASE - WHEN tag_type = 'logfield' THEN 1 - WHEN tag_type = 'resource' THEN 2 - WHEN tag_type = 'scope' THEN 3 - WHEN tag_type = 'tag' THEN 4 - ELSE 5 - END as priority`).From(t.logsDBName + "." + tblName) - var limit int + tablesToQuery := []struct { + fieldContext telemetrytypes.FieldContext + shouldQuery bool + }{ + {telemetrytypes.FieldContextAttribute, queryAttributeTable}, + {telemetrytypes.FieldContextResource, queryResourceTable}, + } - conds := []string{} - searchTexts := []string{} - dataTypes := []telemetrytypes.FieldDataType{} - - for _, fieldKeySelector := range fieldKeySelectors { - - if fieldKeySelector.StartUnixMilli != 0 { - conds = append(conds, sb.GE("unix_milli", fieldKeySelector.StartUnixMilli)) - } - if fieldKeySelector.EndUnixMilli != 0 { - conds = append(conds, sb.LE("unix_milli", fieldKeySelector.EndUnixMilli)) + for _, table := range tablesToQuery { + if !table.shouldQuery { + continue } - // key part of the selector - fieldKeyConds := []string{} - if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact { - fieldKeyConds = append(fieldKeyConds, sb.E("tag_key", fieldKeySelector.Name)) + fieldContext := table.fieldContext + + // table name based on field context + var tblName string + if fieldContext == telemetrytypes.FieldContextAttribute { + tblName = t.logsDBName + "." + t.logAttributeKeysTblName } else { - fieldKeyConds = append(fieldKeyConds, sb.ILike("tag_key", "%"+escapeForLike(fieldKeySelector.Name)+"%")) - } - searchTexts = append(searchTexts, fieldKeySelector.Name) - if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { - dataTypes = append(dataTypes, fieldKeySelector.FieldDataType) + tblName = t.logsDBName + "." + t.logResourceKeysTblName } - // now look at the field context - // we don't write most of intrinsic fields to tag attributes table - // for this reason we don't want to apply tag_type if the field context - // if not attribute or resource attribute - if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified && - (fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute || - fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) { - fieldKeyConds = append(fieldKeyConds, sb.E("tag_type", fieldKeySelector.FieldContext.TagType())) + sb := sqlbuilder.Select( + "name AS tag_key", + fmt.Sprintf("'%s' AS tag_type", fieldContext.TagType()), + "datatype AS tag_data_type", + fmt.Sprintf(`%d AS priority`, getPriorityForContext(fieldContext)), + ).From(tblName) + + var limit int + conds := []string{} + + for _, fieldKeySelector := range fieldKeySelectors { + // Include this selector if: + // 1. It has unspecified context (matches all tables) + // 2. Its context matches the current table's context + if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified && + fieldKeySelector.FieldContext != fieldContext { + continue + } + + // key part of the selector + fieldKeyConds := []string{} + if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact { + fieldKeyConds = append(fieldKeyConds, sb.E("name", fieldKeySelector.Name)) + } else { + fieldKeyConds = append(fieldKeyConds, sb.ILike("name", "%"+escapeForLike(fieldKeySelector.Name)+"%")) + } + + // now look at the field data type + if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { + fieldKeyConds = append(fieldKeyConds, sb.E("datatype", fieldKeySelector.FieldDataType.TagDataType())) + } + + if len(fieldKeyConds) > 0 { + conds = append(conds, sb.And(fieldKeyConds...)) + } + limit += fieldKeySelector.Limit } - // now look at the field data type - if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { - fieldKeyConds = append(fieldKeyConds, sb.E("tag_data_type", fieldKeySelector.FieldDataType.TagDataType())) + if len(conds) > 0 { + sb.Where(sb.Or(conds...)) } - conds = append(conds, sb.And(fieldKeyConds...)) - limit += fieldKeySelector.Limit - if strings.TrimSpace(fieldKeySelector.Name) == "" { - sb.Limit(200) + sb.GroupBy("name", "datatype") + if limit == 0 { + limit = 1000 } + + query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + queries = append(queries, query) + allArgs = append(allArgs, args...) + } + + if len(queries) == 0 { + // No matching contexts, return empty result + return []*telemetrytypes.TelemetryFieldKey{}, true, nil + } + + // Combine queries with UNION ALL + var limit int + for _, fieldKeySelector := range fieldKeySelectors { + limit += fieldKeySelector.Limit } - sb.Where(sb.Or(conds...)) - sb.GroupBy("tag_key", "tag_type", "tag_data_type") if limit == 0 { limit = 1000 } - mainSb := sqlbuilder.Select("tag_key", "tag_type", "tag_data_type", "max(priority) as priority") - mainSb.From(mainSb.BuilderAs(sb, "sub_query")) - mainSb.GroupBy("tag_key", "tag_type", "tag_data_type") - mainSb.OrderBy("priority") - // query one extra to check if we hit the limit - mainSb.Limit(limit + 1) + mainQuery := fmt.Sprintf(` + SELECT tag_key, tag_type, tag_data_type, max(priority) as priority + FROM ( + %s + ) AS combined_results + GROUP BY tag_key, tag_type, tag_data_type + ORDER BY priority + LIMIT %d + `, strings.Join(queries, " UNION ALL "), limit+1) - query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse) - - rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...) + rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...) if err != nil { return nil, false, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error()) } defer rows.Close() + keys := []*telemetrytypes.TelemetryFieldKey{} rowCount := 0 + searchTexts := []string{} + dataTypes := []telemetrytypes.FieldDataType{} + + // Collect search texts and data types for static field matching + for _, fieldKeySelector := range fieldKeySelectors { + searchTexts = append(searchTexts, fieldKeySelector.Name) + if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { + dataTypes = append(dataTypes, fieldKeySelector.FieldDataType) + } + } + for rows.Next() { rowCount++ // reached the limit, we know there are more results @@ -510,6 +575,21 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors return keys, complete, nil } +func getPriorityForContext(ctx telemetrytypes.FieldContext) int { + switch ctx { + case telemetrytypes.FieldContextLog: + return 1 + case telemetrytypes.FieldContextResource: + return 2 + case telemetrytypes.FieldContextScope: + return 3 + case telemetrytypes.FieldContextAttribute: + return 4 + default: + return 5 + } +} + // getMetricsKeys returns the keys from the metrics that match the field selection criteria func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) { if len(fieldKeySelectors) == 0 { @@ -856,6 +936,7 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel } whereClause, err := querybuilder.PrepareWhereClause(fieldValueSelector.ExistingQuery, querybuilder.FilterExprVisitorOpts{ + Logger: t.logger, FieldMapper: t.fm, ConditionBuilder: t.conditionBuilder, FieldKeys: keys, diff --git a/pkg/telemetrymetadata/metadata_test.go b/pkg/telemetrymetadata/metadata_test.go index a2b9442e304a..beca13eb426a 100644 --- a/pkg/telemetrymetadata/metadata_test.go +++ b/pkg/telemetrymetadata/metadata_test.go @@ -40,6 +40,7 @@ func TestGetKeys(t *testing.T) { mockTelemetryStore, telemetrytraces.DBName, telemetrytraces.TagAttributesV2TableName, + telemetrytraces.SpanAttributesKeysTblName, telemetrytraces.SpanIndexV3TableName, telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName, @@ -48,6 +49,8 @@ func TestGetKeys(t *testing.T) { telemetrylogs.DBName, telemetrylogs.LogsV2TableName, telemetrylogs.TagAttributesV2TableName, + telemetrylogs.LogAttributeKeysTblName, + telemetrylogs.LogResourceKeysTblName, DBName, AttributesMetadataLocalTableName, ) diff --git a/pkg/telemetrymeter/statement_builder.go b/pkg/telemetrymeter/statement_builder.go index 2a6949fe8c2a..0b46415c1710 100644 --- a/pkg/telemetrymeter/statement_builder.go +++ b/pkg/telemetrymeter/statement_builder.go @@ -141,6 +141,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath( ) if query.Filter != nil && query.Filter.Expression != "" { filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fm, ConditionBuilder: b.cb, FieldKeys: keys, @@ -223,6 +224,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta( if query.Filter != nil && query.Filter.Expression != "" { filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fm, ConditionBuilder: b.cb, FieldKeys: keys, @@ -286,6 +288,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified( ) if query.Filter != nil && query.Filter.Expression != "" { filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fm, ConditionBuilder: b.cb, FieldKeys: keys, diff --git a/pkg/telemetrymetrics/statement_builder.go b/pkg/telemetrymetrics/statement_builder.go index 839e15757824..318ebe51edd7 100644 --- a/pkg/telemetrymetrics/statement_builder.go +++ b/pkg/telemetrymetrics/statement_builder.go @@ -68,6 +68,9 @@ func GetKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) for idx := range keySelectors { keySelectors[idx].Signal = telemetrytypes.SignalMetrics keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact + keySelectors[idx].MetricContext = &telemetrytypes.MetricContext{ + MetricName: query.Aggregations[0].MetricName, + } } return keySelectors } @@ -295,6 +298,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE( if query.Filter != nil && query.Filter.Expression != "" { preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fm, ConditionBuilder: b.cb, FieldKeys: keys, diff --git a/pkg/telemetrytraces/span_scope_simple_test.go b/pkg/telemetrytraces/span_scope_simple_test.go index 3fd43b21069c..2a7f0445fb1e 100644 --- a/pkg/telemetrytraces/span_scope_simple_test.go +++ b/pkg/telemetrytraces/span_scope_simple_test.go @@ -3,6 +3,7 @@ package telemetrytraces import ( "testing" + "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/huandu/go-sqlbuilder" @@ -64,6 +65,7 @@ func TestSpanScopeFilterExpression(t *testing.T) { }} whereClause, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{ + Logger: instrumentationtest.New().Logger(), FieldMapper: fm, ConditionBuilder: cb, FieldKeys: fieldKeys, @@ -130,6 +132,7 @@ func TestSpanScopeWithResourceFilter(t *testing.T) { }} _, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{ + Logger: instrumentationtest.New().Logger(), FieldMapper: fm, ConditionBuilder: cb, FieldKeys: fieldKeys, diff --git a/pkg/telemetrytraces/statement_builder.go b/pkg/telemetrytraces/statement_builder.go index cd521cefb023..65554e71a132 100644 --- a/pkg/telemetrytraces/statement_builder.go +++ b/pkg/telemetrytraces/statement_builder.go @@ -735,6 +735,7 @@ func (b *traceQueryStatementBuilder) addFilterCondition( if query.Filter != nil && query.Filter.Expression != "" { // add filter expression preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{ + Logger: b.logger, FieldMapper: b.fm, ConditionBuilder: b.cb, FieldKeys: keys, diff --git a/pkg/telemetrytraces/stmt_builder_test.go b/pkg/telemetrytraces/stmt_builder_test.go index e94916018e4b..9585a373581b 100644 --- a/pkg/telemetrytraces/stmt_builder_test.go +++ b/pkg/telemetrytraces/stmt_builder_test.go @@ -21,6 +21,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.TraceAggregati mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() return resourcefilter.NewTraceResourceFilterStatementBuilder( + instrumentationtest.New().ToProviderSettings(), fm, cb, mockMetadataStore, @@ -327,7 +328,7 @@ func TestStatementBuilder(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() @@ -495,7 +496,7 @@ func TestStatementBuilderListQuery(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() @@ -557,7 +558,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() diff --git a/pkg/telemetrytraces/tables.go b/pkg/telemetrytraces/tables.go index e9dde960fde8..38cd66ad8667 100644 --- a/pkg/telemetrytraces/tables.go +++ b/pkg/telemetrytraces/tables.go @@ -8,4 +8,5 @@ const ( TagAttributesV2LocalTableName = "tag_attributes_v2" TopLevelOperationsTableName = "distributed_top_level_operations" TraceSummaryTableName = "distributed_trace_summary" + SpanAttributesKeysTblName = "distributed_span_attributes_keys" ) diff --git a/pkg/telemetrytraces/trace_time_range_test.go b/pkg/telemetrytraces/trace_time_range_test.go index 7957fc81f866..0f2479207562 100644 --- a/pkg/telemetrytraces/trace_time_range_test.go +++ b/pkg/telemetrytraces/trace_time_range_test.go @@ -38,12 +38,13 @@ func TestTraceTimeRangeOptimization(t *testing.T) { resourceFilterFM := resourcefilter.NewFieldMapper() resourceFilterCB := resourcefilter.NewConditionBuilder(resourceFilterFM) resourceFilterStmtBuilder := resourcefilter.NewTraceResourceFilterStatementBuilder( + instrumentationtest.New().ToProviderSettings(), resourceFilterFM, resourceFilterCB, mockMetadataStore, ) - aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(),