From f96dda7c716b8274f8593d04736eb9a5729880d8 Mon Sep 17 00:00:00 2001 From: aniket Date: Thu, 12 Jun 2025 12:40:12 +0530 Subject: [PATCH] chore(2354): added preloaded metrics metadata at first api call --- ee/query-service/app/server.go | 11 ---- .../app/clickhouseReader/reader.go | 57 ++++++------------- pkg/query-service/app/server.go | 13 +---- pkg/query-service/constants/constants.go | 1 + .../model/updatedMetricsMetadata.go | 9 +++ 5 files changed, 27 insertions(+), 64 deletions(-) diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 1ba66fdff112..e16f582d9500 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -227,17 +227,6 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { &opAmpModel.AllAgents, agentConfMgr, ) - orgs, err := apiHandler.Signoz.Modules.OrgGetter.ListByOwnedKeyRange(context.Background()) - if err != nil { - return nil, err - } - for _, org := range orgs { - errorList := reader.PreloadMetricsMetadata(context.Background(), org.ID) - for _, er := range errorList { - zap.L().Error("failed to preload metrics metadata", zap.Error(er)) - } - } - return s, nil } diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index c4dd8dbde15c..2749172cf7b3 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -6257,7 +6257,22 @@ func (r *ClickHouseReader) PreloadMetricsMetadata(ctx context.Context, orgID val func (r *ClickHouseReader) GetUpdatedMetricsMetadata(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]*model.UpdateMetricsMetadata, *model.ApiError) { cachedMetadata := make(map[string]*model.UpdateMetricsMetadata) - var missingMetrics []string + + preCacheLoaded := new(model.CacheLoaded) + err := r.cache.Get(ctx, orgID, constants.METRICS_UPDATED_METADATA_CACHE_LOADED_KEY, preCacheLoaded, false) + if err != nil { + *preCacheLoaded = false + zap.L().Warn("Failed to get cached metrics updated metadata", zap.Error(err)) + } + + if !*preCacheLoaded { + r.PreloadMetricsMetadata(ctx, orgID) + *preCacheLoaded = true + err := r.cache.Set(ctx, orgID, constants.METRICS_UPDATED_METADATA_CACHE_LOADED_KEY, preCacheLoaded, -1) + if err != nil { + zap.L().Warn("Failed to set cached metrics updated metadata", zap.Error(err)) + } + } // First, try retrieving each metric from cache. for _, metricName := range metricNames { @@ -6266,48 +6281,8 @@ func (r *ClickHouseReader) GetUpdatedMetricsMetadata(ctx context.Context, orgID err := r.cache.Get(ctx, orgID, cacheKey, metadata, true) if err == nil { cachedMetadata[metricName] = metadata - } else { - missingMetrics = append(missingMetrics, metricName) } } - - // If there are any metrics missing in the cache, query them from the database. - if len(missingMetrics) > 0 { - // Join the missing metric names; ensure proper quoting if needed. - metricList := "'" + strings.Join(metricNames, "', '") + "'" - query := fmt.Sprintf(`SELECT metric_name, type, description, temporality, is_monotonic, unit - FROM %s.%s - WHERE metric_name IN (%s);`, signozMetricDBName, signozUpdatedMetricsMetadataTable, metricList) - - valueCtx := context.WithValue(ctx, "clickhouse_max_threads", constants.MetricsExplorerClickhouseThreads) - rows, err := r.db.Query(valueCtx, query) - if err != nil { - return cachedMetadata, &model.ApiError{Typ: "ClickhouseErr", Err: fmt.Errorf("error querying metrics metadata: %v", err)} - } - defer rows.Close() - - for rows.Next() { - metadata := new(model.UpdateMetricsMetadata) - if err := rows.Scan( - &metadata.MetricName, - &metadata.MetricType, - &metadata.Description, - &metadata.Temporality, - &metadata.IsMonotonic, - &metadata.Unit, - ); err != nil { - return cachedMetadata, &model.ApiError{Typ: "ClickhouseErr", Err: fmt.Errorf("error scanning metrics metadata: %v", err)} - } - - // Cache the result for future requests. - cacheKey := constants.UpdatedMetricsMetadataCachePrefix + metadata.MetricName - if cacheErr := r.cache.Set(ctx, orgID, cacheKey, metadata, -1); cacheErr != nil { - zap.L().Error("Failed to store metrics metadata in cache", zap.String("metric_name", metadata.MetricName), zap.Error(cacheErr)) - } - cachedMetadata[metadata.MetricName] = metadata - } - } - return cachedMetadata, nil } diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 479a4df27a3c..4a9a5431938d 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -197,18 +197,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { s.opampServer = opamp.InitializeServer( &opAmpModel.AllAgents, agentConfMgr, ) - - orgs, err := apiHandler.Signoz.Modules.OrgGetter.ListByOwnedKeyRange(context.Background()) - if err != nil { - return nil, err - } - for _, org := range orgs { - errorList := reader.PreloadMetricsMetadata(context.Background(), org.ID) - for _, er := range errorList { - zap.L().Error("failed to preload metrics metadata", zap.Error(er)) - } - } - + return s, nil } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index e69b88d3e75e..631518e33f99 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -668,3 +668,4 @@ func GetDefaultSiteURL() string { } const DotMetricsEnabled = "DOT_METRICS_ENABLED" +const METRICS_UPDATED_METADATA_CACHE_LOADED_KEY = "METRICS_UPDATED_METADATA_CACHE_LOADED" diff --git a/pkg/query-service/model/updatedMetricsMetadata.go b/pkg/query-service/model/updatedMetricsMetadata.go index 01890eaf68a5..37faad9b61f1 100644 --- a/pkg/query-service/model/updatedMetricsMetadata.go +++ b/pkg/query-service/model/updatedMetricsMetadata.go @@ -22,3 +22,12 @@ func (c *UpdateMetricsMetadata) MarshalBinary() (data []byte, err error) { func (c *UpdateMetricsMetadata) UnmarshalBinary(data []byte) error { return json.Unmarshal(data, c) } + +type CacheLoaded bool + +func (c *CacheLoaded) MarshalBinary() (data []byte, err error) { + return json.Marshal(c) +} +func (c *CacheLoaded) UnmarshalBinary(data []byte) error { + return json.Unmarshal(data, c) +}