chore: use *_keys tables instead of tag_attributes_v2 for suggestions (#8753)

pkg/apis/fields/api.go

@@ -31,6 +31,7 @@ func NewAPI(
 		telemetryStore,
 		telemetrytraces.DBName,
 		telemetrytraces.TagAttributesV2TableName,
+		telemetrytraces.SpanAttributesKeysTblName,
 		telemetrytraces.SpanIndexV3TableName,
 		telemetrymetrics.DBName,
 		telemetrymetrics.AttributesMetadataTableName,
@@ -39,6 +40,8 @@ func NewAPI(
 		telemetrylogs.DBName,
 		telemetrylogs.LogsV2TableName,
 		telemetrylogs.TagAttributesV2TableName,
+		telemetrylogs.LogAttributeKeysTblName,
+		telemetrylogs.LogResourceKeysTblName,
 		telemetrymetadata.DBName,
 		telemetrymetadata.AttributesMetadataLocalTableName,
 	)

pkg/querier/signozquerier/provider.go

@@ -50,6 +50,7 @@ func newProvider(
 		telemetryStore,
 		telemetrytraces.DBName,
 		telemetrytraces.TagAttributesV2TableName,
+		telemetrytraces.SpanAttributesKeysTblName,
 		telemetrytraces.SpanIndexV3TableName,
 		telemetrymetrics.DBName,
 		telemetrymetrics.AttributesMetadataTableName,
@@ -58,6 +59,8 @@ func newProvider(
 		telemetrylogs.DBName,
 		telemetrylogs.LogsV2TableName,
 		telemetrylogs.TagAttributesV2TableName,
+		telemetrylogs.LogAttributeKeysTblName,
+		telemetrylogs.LogResourceKeysTblName,
 		telemetrymetadata.DBName,
 		telemetrymetadata.AttributesMetadataLocalTableName,
 	)
@@ -69,12 +72,13 @@ func newProvider(
 	resourceFilterFieldMapper := resourcefilter.NewFieldMapper()
 	resourceFilterConditionBuilder := resourcefilter.NewConditionBuilder(resourceFilterFieldMapper)
 	resourceFilterStmtBuilder := resourcefilter.NewTraceResourceFilterStatementBuilder(
+		settings,
 		resourceFilterFieldMapper,
 		resourceFilterConditionBuilder,
 		telemetryMetadataStore,
 	)
 
-	traceAggExprRewriter := querybuilder.NewAggExprRewriter(nil, traceFieldMapper, traceConditionBuilder, "", nil)
+	traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, "", nil)
 	traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
 		settings,
 		telemetryMetadataStore,
@@ -89,6 +93,7 @@ func newProvider(
 	logFieldMapper := telemetrylogs.NewFieldMapper()
 	logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper)
 	logResourceFilterStmtBuilder := resourcefilter.NewLogResourceFilterStatementBuilder(
+		settings,
 		resourceFilterFieldMapper,
 		resourceFilterConditionBuilder,
 		telemetryMetadataStore,
@@ -97,6 +102,7 @@ func newProvider(
 		telemetrylogs.GetBodyJSONKey,
 	)
 	logAggExprRewriter := querybuilder.NewAggExprRewriter(
+		settings,
 		telemetrylogs.DefaultFullTextColumn,
 		logFieldMapper,
 		logConditionBuilder,

pkg/querybuilder/agg_rewrite.go

@@ -3,10 +3,12 @@ package querybuilder
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"strings"
 
 	chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
@@ -14,6 +16,7 @@ import (
 )
 
 type aggExprRewriter struct {
+	logger           *slog.Logger
 	fullTextColumn   *telemetrytypes.TelemetryFieldKey
 	fieldMapper      qbtypes.FieldMapper
 	conditionBuilder qbtypes.ConditionBuilder
@@ -24,13 +27,17 @@ type aggExprRewriter struct {
 var _ qbtypes.AggExprRewriter = (*aggExprRewriter)(nil)
 
 func NewAggExprRewriter(
+	settings factory.ProviderSettings,
 	fullTextColumn *telemetrytypes.TelemetryFieldKey,
 	fieldMapper qbtypes.FieldMapper,
 	conditionBuilder qbtypes.ConditionBuilder,
 	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *aggExprRewriter {
+	set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/agg_rewrite")
+
 	return &aggExprRewriter{
+		logger:           set.Logger(),
 		fullTextColumn:   fullTextColumn,
 		fieldMapper:      fieldMapper,
 		conditionBuilder: conditionBuilder,
@@ -70,7 +77,7 @@ func (r *aggExprRewriter) Rewrite(
 		return "", nil, errors.NewInternalf(errors.CodeInternal, "no SELECT items for %q", expr)
 	}
 
-	visitor := newExprVisitor(keys,
+	visitor := newExprVisitor(r.logger, keys,
 		r.fullTextColumn,
 		r.fieldMapper,
 		r.conditionBuilder,
@@ -117,6 +124,7 @@ func (r *aggExprRewriter) RewriteMulti(
 // exprVisitor walks FunctionExpr nodes and applies the mappers.
 type exprVisitor struct {
 	chparser.DefaultASTVisitor
+	logger           *slog.Logger
 	fieldKeys        map[string][]*telemetrytypes.TelemetryFieldKey
 	fullTextColumn   *telemetrytypes.TelemetryFieldKey
 	fieldMapper      qbtypes.FieldMapper
@@ -129,6 +137,7 @@ type exprVisitor struct {
 }
 
 func newExprVisitor(
+	logger *slog.Logger,
 	fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
 	fullTextColumn *telemetrytypes.TelemetryFieldKey,
 	fieldMapper qbtypes.FieldMapper,
@@ -137,6 +146,7 @@ func newExprVisitor(
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *exprVisitor {
 	return &exprVisitor{
+		logger:           logger,
 		fieldKeys:        fieldKeys,
 		fullTextColumn:   fullTextColumn,
 		fieldMapper:      fieldMapper,
@@ -183,6 +193,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 		whereClause, err := PrepareWhereClause(
 			origPred,
 			FilterExprVisitorOpts{
+				Logger:           v.logger,
 				FieldKeys:        v.fieldKeys,
 				FieldMapper:      v.fieldMapper,
 				ConditionBuilder: v.conditionBuilder,

pkg/querybuilder/resourcefilter/statement_builder.go

@@ -3,8 +3,10 @@ package resourcefilter
 import (
 	"context"
 	"fmt"
+	"log/slog"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -34,6 +36,7 @@ var signalConfigs = map[telemetrytypes.Signal]signalConfig{
 
 // Generic resource filter statement builder
 type resourceFilterStatementBuilder[T any] struct {
+	logger           *slog.Logger
 	fieldMapper      qbtypes.FieldMapper
 	conditionBuilder qbtypes.ConditionBuilder
 	metadataStore    telemetrytypes.MetadataStore
@@ -52,11 +55,14 @@ var (
 
 // Constructor functions
 func NewTraceResourceFilterStatementBuilder(
+	settings factory.ProviderSettings,
 	fieldMapper qbtypes.FieldMapper,
 	conditionBuilder qbtypes.ConditionBuilder,
 	metadataStore telemetrytypes.MetadataStore,
 ) *resourceFilterStatementBuilder[qbtypes.TraceAggregation] {
+	set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter")
 	return &resourceFilterStatementBuilder[qbtypes.TraceAggregation]{
+		logger:           set.Logger(),
 		fieldMapper:      fieldMapper,
 		conditionBuilder: conditionBuilder,
 		metadataStore:    metadataStore,
@@ -65,6 +71,7 @@ func NewTraceResourceFilterStatementBuilder(
 }
 
 func NewLogResourceFilterStatementBuilder(
+	settings factory.ProviderSettings,
 	fieldMapper qbtypes.FieldMapper,
 	conditionBuilder qbtypes.ConditionBuilder,
 	metadataStore telemetrytypes.MetadataStore,
@@ -72,7 +79,9 @@ func NewLogResourceFilterStatementBuilder(
 	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *resourceFilterStatementBuilder[qbtypes.LogAggregation] {
+	set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter")
 	return &resourceFilterStatementBuilder[qbtypes.LogAggregation]{
+		logger:           set.Logger(),
 		fieldMapper:      fieldMapper,
 		conditionBuilder: conditionBuilder,
 		metadataStore:    metadataStore,
@@ -148,6 +157,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
 
 		// warnings would be encountered as part of the main condition already
 		filterWhereClause, err := querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:             b.logger,
 			FieldMapper:        b.fieldMapper,
 			ConditionBuilder:   b.conditionBuilder,
 			FieldKeys:          keys,

pkg/querybuilder/where_clause_visitor.go

@@ -3,6 +3,7 @@ package querybuilder
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"strconv"
 	"strings"
 
@@ -20,6 +21,7 @@ var searchTroubleshootingGuideURL = "https://signoz.io/docs/userguide/search-tro
 // filterExpressionVisitor implements the FilterQueryVisitor interface
 // to convert the parsed filter expressions into ClickHouse WHERE clause
 type filterExpressionVisitor struct {
+	logger             *slog.Logger
 	fieldMapper        qbtypes.FieldMapper
 	conditionBuilder   qbtypes.ConditionBuilder
 	warnings           []string
@@ -41,6 +43,7 @@ type filterExpressionVisitor struct {
 }
 
 type FilterExprVisitorOpts struct {
+	Logger             *slog.Logger
 	FieldMapper        qbtypes.FieldMapper
 	ConditionBuilder   qbtypes.ConditionBuilder
 	FieldKeys          map[string][]*telemetrytypes.TelemetryFieldKey
@@ -58,6 +61,7 @@ type FilterExprVisitorOpts struct {
 // newFilterExpressionVisitor creates a new filterExpressionVisitor
 func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVisitor {
 	return &filterExpressionVisitor{
+		logger:             opts.Logger,
 		fieldMapper:        opts.FieldMapper,
 		conditionBuilder:   opts.ConditionBuilder,
 		fieldKeys:          opts.FieldKeys,
@@ -786,15 +790,35 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
 	}
 
 	if len(fieldKeysForName) > 1 && !v.keysWithWarnings[keyName] {
-		v.mainWarnURL = "https://signoz.io/docs/userguide/field-context-data-types/"
+		warnMsg := fmt.Sprintf(
-		// this is warning state, we must have a unambiguous key
+			"Key `%s` is ambiguous, found %d different combinations of field context / data type: %v.",
-		v.warnings = append(v.warnings, fmt.Sprintf(
-			"key `%s` is ambiguous, found %d different combinations of field context / data type: %v",
 			fieldKey.Name,
 			len(fieldKeysForName),
 			fieldKeysForName,
-		))
+		)
+		mixedFieldContext := map[string]bool{}
+		for _, item := range fieldKeysForName {
+			mixedFieldContext[item.FieldContext.StringValue()] = true
+		}
+
+		if mixedFieldContext[telemetrytypes.FieldContextResource.StringValue()] &&
+			mixedFieldContext[telemetrytypes.FieldContextAttribute.StringValue()] {
+			filteredKeys := []*telemetrytypes.TelemetryFieldKey{}
+			for _, item := range fieldKeysForName {
+				if item.FieldContext != telemetrytypes.FieldContextResource {
+					continue
+				}
+				filteredKeys = append(filteredKeys, item)
+			}
+			fieldKeysForName = filteredKeys
+			warnMsg += " " + "Using `resource` context by default. To query attributes explicitly, " +
+				fmt.Sprintf("use the fully qualified name (e.g., 'attribute.%s')", fieldKey.Name)
+		}
+		v.mainWarnURL = "https://signoz.io/docs/userguide/field-context-data-types/"
+		// this is warning state, we must have a unambiguous key
+		v.warnings = append(v.warnings, warnMsg)
 		v.keysWithWarnings[keyName] = true
+		v.logger.Warn("ambiguous key", "field_key_name", fieldKey.Name) //nolint:sloglint
 	}
 
 	return fieldKeysForName

pkg/telemetrylogs/filter_expr_logs_body_json_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/huandu/go-sqlbuilder"
@@ -19,6 +20,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
 	keys := buildCompleteFieldKeyMap()
 
 	opts := querybuilder.FilterExprVisitorOpts{
+		Logger:           instrumentationtest.New().Logger(),
 		FieldMapper:      fm,
 		ConditionBuilder: cb,
 		FieldKeys:        keys,

pkg/telemetrylogs/filter_expr_logs_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/huandu/go-sqlbuilder"
@@ -21,6 +22,7 @@ func TestFilterExprLogs(t *testing.T) {
 	keys := buildCompleteFieldKeyMap()
 
 	opts := querybuilder.FilterExprVisitorOpts{
+		Logger:           instrumentationtest.New().Logger(),
 		FieldMapper:      fm,
 		ConditionBuilder: cb,
 		FieldKeys:        keys,

pkg/telemetrylogs/statement_builder.go

@@ -553,6 +553,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
 	if query.Filter != nil && query.Filter.Expression != "" {
 		// add filter expression
 		preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:             b.logger,
 			FieldMapper:        b.fm,
 			ConditionBuilder:   b.cb,
 			FieldKeys:          keys,

pkg/telemetrylogs/stmt_builder_test.go

@@ -27,6 +27,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation
 	mockMetadataStore.KeysMap = keysMap
 
 	return resourcefilter.NewLogResourceFilterStatementBuilder(
+		instrumentationtest.New().ToProviderSettings(),
 		fm,
 		cb,
 		mockMetadataStore,
@@ -119,7 +120,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -212,7 +213,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -321,7 +322,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 

pkg/telemetrylogs/tables.go

@@ -6,4 +6,6 @@ const (
 	LogsV2LocalTableName          = "logs_v2"
 	TagAttributesV2TableName      = "distributed_tag_attributes_v2"
 	TagAttributesV2LocalTableName = "tag_attributes_v2"
+	LogAttributeKeysTblName       = "distributed_logs_attribute_keys"
+	LogResourceKeysTblName        = "distributed_logs_resource_keys"
 )

pkg/telemetrymetadata/metadata.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
-	"os"
 	"slices"
 	"strings"
 
@@ -36,6 +35,7 @@ type telemetryMetaStore struct {
 	telemetrystore            telemetrystore.TelemetryStore
 	tracesDBName              string
 	tracesFieldsTblName       string
+	spanAttributesKeysTblName string
 	indexV3TblName            string
 	metricsDBName             string
 	metricsFieldsTblName      string
@@ -43,6 +43,8 @@ type telemetryMetaStore struct {
 	meterFieldsTblName        string
 	logsDBName                string
 	logsFieldsTblName         string
+	logAttributeKeysTblName   string
+	logResourceKeysTblName    string
 	logsV2TblName             string
 	relatedMetadataDBName     string
 	relatedMetadataTblName    string
@@ -60,6 +62,7 @@ func NewTelemetryMetaStore(
 	telemetrystore telemetrystore.TelemetryStore,
 	tracesDBName string,
 	tracesFieldsTblName string,
+	spanAttributesKeysTblName string,
 	indexV3TblName string,
 	metricsDBName string,
 	metricsFieldsTblName string,
@@ -68,6 +71,8 @@ func NewTelemetryMetaStore(
 	logsDBName string,
 	logsV2TblName string,
 	logsFieldsTblName string,
+	logAttributeKeysTblName string,
+	logResourceKeysTblName string,
 	relatedMetadataDBName string,
 	relatedMetadataTblName string,
 ) telemetrytypes.MetadataStore {
@@ -78,6 +83,7 @@ func NewTelemetryMetaStore(
 		telemetrystore:            telemetrystore,
 		tracesDBName:              tracesDBName,
 		tracesFieldsTblName:       tracesFieldsTblName,
+		spanAttributesKeysTblName: spanAttributesKeysTblName,
 		indexV3TblName:            indexV3TblName,
 		metricsDBName:             metricsDBName,
 		metricsFieldsTblName:      metricsFieldsTblName,
@@ -86,6 +92,8 @@ func NewTelemetryMetaStore(
 		logsDBName:                logsDBName,
 		logsV2TblName:             logsV2TblName,
 		logsFieldsTblName:         logsFieldsTblName,
+		logAttributeKeysTblName:   logAttributeKeysTblName,
+		logResourceKeysTblName:    logResourceKeysTblName,
 		relatedMetadataDBName:     relatedMetadataDBName,
 		relatedMetadataTblName:    relatedMetadataTblName,
 	}
@@ -136,14 +144,18 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
 		mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
 	}
 
-	sb := sqlbuilder.Select("tag_key", "tag_type", "tag_data_type", `
+	sb := sqlbuilder.Select(
-			CASE
+		"tagKey AS tag_key",
-				WHEN tag_type = 'spanfield' THEN 1
+		"tagType AS tag_type",
-				WHEN tag_type = 'resource' THEN 2
+		"dataType AS tag_data_type",
-				WHEN tag_type = 'scope' THEN 3
+		`CASE
-				WHEN tag_type = 'tag' THEN 4
+			// WHEN tagType = 'spanfield' THEN 1
+			WHEN tagType = 'resource' THEN 2
+			// WHEN tagType = 'scope' THEN 3
+			WHEN tagType = 'tag' THEN 4
 			ELSE 5
-			END as priority`).From(t.tracesDBName + "." + t.tracesFieldsTblName)
+		END as priority`,
+	).From(t.tracesDBName + "." + t.spanAttributesKeysTblName)
 	var limit int
 
 	searchTexts := []string{}
@@ -152,19 +164,20 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
 	conds := []string{}
 	for _, fieldKeySelector := range fieldKeySelectors {
 
-		if fieldKeySelector.StartUnixMilli != 0 {
+		// TODO(srikanthccv): support time filtering for span attribute keys
-			conds = append(conds, sb.GE("unix_milli", fieldKeySelector.StartUnixMilli))
+		// if fieldKeySelector.StartUnixMilli != 0 {
-		}
+		// 	conds = append(conds, sb.GE("unix_milli", fieldKeySelector.StartUnixMilli))
-		if fieldKeySelector.EndUnixMilli != 0 {
+		// }
-			conds = append(conds, sb.LE("unix_milli", fieldKeySelector.EndUnixMilli))
+		// if fieldKeySelector.EndUnixMilli != 0 {
-		}
+		// 	conds = append(conds, sb.LE("unix_milli", fieldKeySelector.EndUnixMilli))
+		// }
 
 		// key part of the selector
 		fieldKeyConds := []string{}
 		if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
-			fieldKeyConds = append(fieldKeyConds, sb.E("tag_key", fieldKeySelector.Name))
+			fieldKeyConds = append(fieldKeyConds, sb.E("tagKey", fieldKeySelector.Name))
 		} else {
-			fieldKeyConds = append(fieldKeyConds, sb.ILike("tag_key", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
+			fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
 		}
 
 		searchTexts = append(searchTexts, fieldKeySelector.Name)
@@ -172,29 +185,25 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
 			dataTypes = append(dataTypes, fieldKeySelector.FieldDataType)
 		}
 		// now look at the field context
-		// we don't write most of intrinsic fields to tag attributes table
+		// we don't write most of intrinsic fields to keys table
-		// for this reason we don't want to apply tag_type if the field context
+		// for this reason we don't want to apply tagType if the field context
-		// if not attribute or resource attribute
+		// is not attribute or resource attribute
 		if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified &&
 			(fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute ||
 				fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) {
-			fieldKeyConds = append(fieldKeyConds, sb.E("tag_type", fieldKeySelector.FieldContext.TagType()))
+			fieldKeyConds = append(fieldKeyConds, sb.E("tagType", fieldKeySelector.FieldContext.TagType()))
 		}
 
 		// now look at the field data type
 		if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
-			fieldKeyConds = append(fieldKeyConds, sb.E("tag_data_type", fieldKeySelector.FieldDataType.TagDataType()))
+			fieldKeyConds = append(fieldKeyConds, sb.E("dataType", fieldKeySelector.FieldDataType.TagDataType()))
 		}
 
 		conds = append(conds, sb.And(fieldKeyConds...))
 		limit += fieldKeySelector.Limit
-		if strings.TrimSpace(fieldKeySelector.Name) == "" {
-			sb.Limit(200)
-		}
 	}
 	sb.Where(sb.Or(conds...))
-	sb.GroupBy("tag_key", "tag_type", "tag_data_type")
+	sb.GroupBy("tagKey", "tagType", "dataType")
-
 	if limit == 0 {
 		limit = 1000
 	}
@@ -347,89 +356,145 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 		mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
 	}
 
-	tblName := t.logsFieldsTblName
+	// queries for both attribute and resource keys tables
-	if os.Getenv("LOGS_TAG_ATTRS_KEYS_TABLE") != "" {
+	var queries []string
-		tblName = os.Getenv("LOGS_TAG_ATTRS_KEYS_TABLE")
+	var allArgs []any
+
+	// tables to query based on field selectors
+	queryAttributeTable := false
+	queryResourceTable := false
+
+	for _, selector := range fieldKeySelectors {
+		if selector.FieldContext == telemetrytypes.FieldContextUnspecified {
+			// unspecified context, query both tables
+			queryAttributeTable = true
+			queryResourceTable = true
+			break
+		} else if selector.FieldContext == telemetrytypes.FieldContextAttribute {
+			queryAttributeTable = true
+		} else if selector.FieldContext == telemetrytypes.FieldContextResource {
+			queryResourceTable = true
+		}
 	}
 
-	sb := sqlbuilder.Select("tag_key", "tag_type", "tag_data_type", `
+	tablesToQuery := []struct {
-			CASE
+		fieldContext telemetrytypes.FieldContext
-				WHEN tag_type = 'logfield' THEN 1
+		shouldQuery  bool
-				WHEN tag_type = 'resource' THEN 2
+	}{
-				WHEN tag_type = 'scope' THEN 3
+		{telemetrytypes.FieldContextAttribute, queryAttributeTable},
-				WHEN tag_type = 'tag' THEN 4
+		{telemetrytypes.FieldContextResource, queryResourceTable},
-				ELSE 5
+	}
-			END as priority`).From(t.logsDBName + "." + tblName)
-	var limit int
 
+	for _, table := range tablesToQuery {
+		if !table.shouldQuery {
+			continue
+		}
+
+		fieldContext := table.fieldContext
+
+		// table name based on field context
+		var tblName string
+		if fieldContext == telemetrytypes.FieldContextAttribute {
+			tblName = t.logsDBName + "." + t.logAttributeKeysTblName
+		} else {
+			tblName = t.logsDBName + "." + t.logResourceKeysTblName
+		}
+
+		sb := sqlbuilder.Select(
+			"name AS tag_key",
+			fmt.Sprintf("'%s' AS tag_type", fieldContext.TagType()),
+			"datatype AS tag_data_type",
+			fmt.Sprintf(`%d AS priority`, getPriorityForContext(fieldContext)),
+		).From(tblName)
+
+		var limit int
 		conds := []string{}
-	searchTexts := []string{}
-	dataTypes := []telemetrytypes.FieldDataType{}
 
 		for _, fieldKeySelector := range fieldKeySelectors {
-
+			// Include this selector if:
-		if fieldKeySelector.StartUnixMilli != 0 {
+			// 1. It has unspecified context (matches all tables)
-			conds = append(conds, sb.GE("unix_milli", fieldKeySelector.StartUnixMilli))
+			// 2. Its context matches the current table's context
-		}
+			if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified &&
-		if fieldKeySelector.EndUnixMilli != 0 {
+				fieldKeySelector.FieldContext != fieldContext {
-			conds = append(conds, sb.LE("unix_milli", fieldKeySelector.EndUnixMilli))
+				continue
 			}
 
 			// key part of the selector
 			fieldKeyConds := []string{}
 			if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
-			fieldKeyConds = append(fieldKeyConds, sb.E("tag_key", fieldKeySelector.Name))
+				fieldKeyConds = append(fieldKeyConds, sb.E("name", fieldKeySelector.Name))
 			} else {
-			fieldKeyConds = append(fieldKeyConds, sb.ILike("tag_key", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
+				fieldKeyConds = append(fieldKeyConds, sb.ILike("name", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
-		}
-		searchTexts = append(searchTexts, fieldKeySelector.Name)
-		if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
-			dataTypes = append(dataTypes, fieldKeySelector.FieldDataType)
-		}
-
-		// now look at the field context
-		// we don't write most of intrinsic fields to tag attributes table
-		// for this reason we don't want to apply tag_type if the field context
-		// if not attribute or resource attribute
-		if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified &&
-			(fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute ||
-				fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) {
-			fieldKeyConds = append(fieldKeyConds, sb.E("tag_type", fieldKeySelector.FieldContext.TagType()))
 			}
 
 			// now look at the field data type
 			if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
-			fieldKeyConds = append(fieldKeyConds, sb.E("tag_data_type", fieldKeySelector.FieldDataType.TagDataType()))
+				fieldKeyConds = append(fieldKeyConds, sb.E("datatype", fieldKeySelector.FieldDataType.TagDataType()))
 			}
 
+			if len(fieldKeyConds) > 0 {
 				conds = append(conds, sb.And(fieldKeyConds...))
+			}
 			limit += fieldKeySelector.Limit
-		if strings.TrimSpace(fieldKeySelector.Name) == "" {
-			sb.Limit(200)
-		}
 		}
+
+		if len(conds) > 0 {
 			sb.Where(sb.Or(conds...))
-	sb.GroupBy("tag_key", "tag_type", "tag_data_type")
+		}
+
+		sb.GroupBy("name", "datatype")
 		if limit == 0 {
 			limit = 1000
 		}
 
-	mainSb := sqlbuilder.Select("tag_key", "tag_type", "tag_data_type", "max(priority) as priority")
+		query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
-	mainSb.From(mainSb.BuilderAs(sb, "sub_query"))
+		queries = append(queries, query)
-	mainSb.GroupBy("tag_key", "tag_type", "tag_data_type")
+		allArgs = append(allArgs, args...)
-	mainSb.OrderBy("priority")
+	}
-	// query one extra to check if we hit the limit
-	mainSb.Limit(limit + 1)
 
-	query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	if len(queries) == 0 {
+		// No matching contexts, return empty result
+		return []*telemetrytypes.TelemetryFieldKey{}, true, nil
+	}
 
-	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
+	// Combine queries with UNION ALL
+	var limit int
+	for _, fieldKeySelector := range fieldKeySelectors {
+		limit += fieldKeySelector.Limit
+	}
+	if limit == 0 {
+		limit = 1000
+	}
+
+	mainQuery := fmt.Sprintf(`
+		SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
+		FROM (
+			%s
+		) AS combined_results
+		GROUP BY tag_key, tag_type, tag_data_type
+		ORDER BY priority
+		LIMIT %d
+	`, strings.Join(queries, " UNION ALL "), limit+1)
+
+	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
 	if err != nil {
 		return nil, false, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
 	}
 	defer rows.Close()
+
 	keys := []*telemetrytypes.TelemetryFieldKey{}
 	rowCount := 0
+	searchTexts := []string{}
+	dataTypes := []telemetrytypes.FieldDataType{}
+
+	// Collect search texts and data types for static field matching
+	for _, fieldKeySelector := range fieldKeySelectors {
+		searchTexts = append(searchTexts, fieldKeySelector.Name)
+		if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
+			dataTypes = append(dataTypes, fieldKeySelector.FieldDataType)
+		}
+	}
+
 	for rows.Next() {
 		rowCount++
 		// reached the limit, we know there are more results
@@ -510,6 +575,21 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 	return keys, complete, nil
 }
 
+func getPriorityForContext(ctx telemetrytypes.FieldContext) int {
+	switch ctx {
+	case telemetrytypes.FieldContextLog:
+		return 1
+	case telemetrytypes.FieldContextResource:
+		return 2
+	case telemetrytypes.FieldContextScope:
+		return 3
+	case telemetrytypes.FieldContextAttribute:
+		return 4
+	default:
+		return 5
+	}
+}
+
 // getMetricsKeys returns the keys from the metrics that match the field selection criteria
 func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
 	if len(fieldKeySelectors) == 0 {
@@ -856,6 +936,7 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
 		}
 
 		whereClause, err := querybuilder.PrepareWhereClause(fieldValueSelector.ExistingQuery, querybuilder.FilterExprVisitorOpts{
+			Logger:           t.logger,
 			FieldMapper:      t.fm,
 			ConditionBuilder: t.conditionBuilder,
 			FieldKeys:        keys,

pkg/telemetrymetadata/metadata_test.go

@@ -40,6 +40,7 @@ func TestGetKeys(t *testing.T) {
 		mockTelemetryStore,
 		telemetrytraces.DBName,
 		telemetrytraces.TagAttributesV2TableName,
+		telemetrytraces.SpanAttributesKeysTblName,
 		telemetrytraces.SpanIndexV3TableName,
 		telemetrymetrics.DBName,
 		telemetrymetrics.AttributesMetadataTableName,
@@ -48,6 +49,8 @@ func TestGetKeys(t *testing.T) {
 		telemetrylogs.DBName,
 		telemetrylogs.LogsV2TableName,
 		telemetrylogs.TagAttributesV2TableName,
+		telemetrylogs.LogAttributeKeysTblName,
+		telemetrylogs.LogResourceKeysTblName,
 		DBName,
 		AttributesMetadataLocalTableName,
 	)

pkg/telemetrymeter/statement_builder.go

@@ -141,6 +141,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
 	)
 	if query.Filter != nil && query.Filter.Expression != "" {
 		filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:           b.logger,
 			FieldMapper:      b.fm,
 			ConditionBuilder: b.cb,
 			FieldKeys:        keys,
@@ -223,6 +224,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
 
 	if query.Filter != nil && query.Filter.Expression != "" {
 		filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:           b.logger,
 			FieldMapper:      b.fm,
 			ConditionBuilder: b.cb,
 			FieldKeys:        keys,
@@ -286,6 +288,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
 	)
 	if query.Filter != nil && query.Filter.Expression != "" {
 		filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:           b.logger,
 			FieldMapper:      b.fm,
 			ConditionBuilder: b.cb,
 			FieldKeys:        keys,

pkg/telemetrymetrics/statement_builder.go

@@ -68,6 +68,9 @@ func GetKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation])
 	for idx := range keySelectors {
 		keySelectors[idx].Signal = telemetrytypes.SignalMetrics
 		keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact
+		keySelectors[idx].MetricContext = &telemetrytypes.MetricContext{
+			MetricName: query.Aggregations[0].MetricName,
+		}
 	}
 	return keySelectors
 }
@@ -295,6 +298,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
 
 	if query.Filter != nil && query.Filter.Expression != "" {
 		preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:           b.logger,
 			FieldMapper:      b.fm,
 			ConditionBuilder: b.cb,
 			FieldKeys:        keys,

pkg/telemetrytraces/span_scope_simple_test.go

@@ -3,6 +3,7 @@ package telemetrytraces
 import (
 	"testing"
 
+	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/huandu/go-sqlbuilder"
@@ -64,6 +65,7 @@ func TestSpanScopeFilterExpression(t *testing.T) {
 			}}
 
 			whereClause, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
+				Logger:           instrumentationtest.New().Logger(),
 				FieldMapper:      fm,
 				ConditionBuilder: cb,
 				FieldKeys:        fieldKeys,
@@ -130,6 +132,7 @@ func TestSpanScopeWithResourceFilter(t *testing.T) {
 			}}
 
 			_, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
+				Logger:             instrumentationtest.New().Logger(),
 				FieldMapper:        fm,
 				ConditionBuilder:   cb,
 				FieldKeys:          fieldKeys,

pkg/telemetrytraces/statement_builder.go

@@ -735,6 +735,7 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
 	if query.Filter != nil && query.Filter.Expression != "" {
 		// add filter expression
 		preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
+			Logger:             b.logger,
 			FieldMapper:        b.fm,
 			ConditionBuilder:   b.cb,
 			FieldKeys:          keys,

pkg/telemetrytraces/stmt_builder_test.go

@@ -21,6 +21,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.TraceAggregati
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
 
 	return resourcefilter.NewTraceResourceFilterStatementBuilder(
+		instrumentationtest.New().ToProviderSettings(),
 		fm,
 		cb,
 		mockMetadataStore,
@@ -327,7 +328,7 @@ func TestStatementBuilder(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -495,7 +496,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -557,7 +558,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 

pkg/telemetrytraces/tables.go

@@ -8,4 +8,5 @@ const (
 	TagAttributesV2LocalTableName = "tag_attributes_v2"
 	TopLevelOperationsTableName   = "distributed_top_level_operations"
 	TraceSummaryTableName         = "distributed_trace_summary"
+	SpanAttributesKeysTblName     = "distributed_span_attributes_keys"
 )

pkg/telemetrytraces/trace_time_range_test.go

@@ -38,12 +38,13 @@ func TestTraceTimeRangeOptimization(t *testing.T) {
 	resourceFilterFM := resourcefilter.NewFieldMapper()
 	resourceFilterCB := resourcefilter.NewConditionBuilder(resourceFilterFM)
 	resourceFilterStmtBuilder := resourcefilter.NewTraceResourceFilterStatementBuilder(
+		instrumentationtest.New().ToProviderSettings(),
 		resourceFilterFM,
 		resourceFilterCB,
 		mockMetadataStore,
 	)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
 
 	statementBuilder := NewTraceQueryStatementBuilder(
 		instrumentationtest.New().ToProviderSettings(),