From a023a7514e69d461f5e5b37a1f9a0053375e3468 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Sat, 14 Sep 2024 13:23:49 +0530 Subject: [PATCH] =?UTF-8?q?chore:=20move=20analytics=20related=20methods?= =?UTF-8?q?=20from=20CH=20reader=20to=20their=20own=20mod=E2=80=A6=20(#593?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/clickhouseReader/reader.go | 168 ------------------ pkg/query-service/app/dashboards/model.go | 127 +++++++++++++ pkg/query-service/app/explorer/db.go | 23 +++ pkg/query-service/dao/sqlite/connection.go | 1 + pkg/query-service/interfaces/interface.go | 3 - pkg/query-service/telemetry/telemetry.go | 27 ++- 6 files changed, 172 insertions(+), 177 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 58f0683c07df..cf2e8fadf2cf 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -42,14 +42,11 @@ import ( "go.uber.org/zap" queryprogress "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader/query_progress" - "go.signoz.io/signoz/pkg/query-service/app/dashboards" - "go.signoz.io/signoz/pkg/query-service/app/explorer" "go.signoz.io/signoz/pkg/query-service/app/logs" "go.signoz.io/signoz/pkg/query-service/app/services" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" - "go.signoz.io/signoz/pkg/query-service/dao" chErrors "go.signoz.io/signoz/pkg/query-service/errors" am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -3243,171 +3240,6 @@ func removeUnderscoreDuplicateFields(fields []model.LogField) []model.LogField { return updatedFields } -// GetDashboardsInfo returns analytics data for dashboards -func (r *ClickHouseReader) GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) { - dashboardsInfo := model.DashboardsInfo{} - // fetch dashboards from dashboard db - query := "SELECT data FROM dashboards" - var dashboardsData []dashboards.Dashboard - err := r.localDB.Select(&dashboardsData, query) - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return &dashboardsInfo, err - } - totalDashboardsWithPanelAndName := 0 - var dashboardNames []string - count := 0 - logChQueriesCount := 0 - for _, dashboard := range dashboardsData { - if isDashboardWithPanelAndName(dashboard.Data) { - totalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName + 1 - } - dashboardName := extractDashboardName(dashboard.Data) - if dashboardName != "" { - dashboardNames = append(dashboardNames, dashboardName) - } - dashboardInfo := countPanelsInDashboard(dashboard.Data) - dashboardsInfo.LogsBasedPanels += dashboardInfo.LogsBasedPanels - dashboardsInfo.TracesBasedPanels += dashboardInfo.TracesBasedPanels - dashboardsInfo.MetricBasedPanels += dashboardsInfo.MetricBasedPanels - if isDashboardWithTSV2(dashboard.Data) { - count = count + 1 - } - if isDashboardWithLogsClickhouseQuery(dashboard.Data) { - logChQueriesCount = logChQueriesCount + 1 - } - } - - dashboardsInfo.DashboardNames = dashboardNames - dashboardsInfo.TotalDashboards = len(dashboardsData) - dashboardsInfo.TotalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName - dashboardsInfo.QueriesWithTSV2 = count - dashboardsInfo.DashboardsWithLogsChQuery = logChQueriesCount - return &dashboardsInfo, nil -} - -func isDashboardWithTSV2(data map[string]interface{}) bool { - jsonData, err := json.Marshal(data) - if err != nil { - return false - } - return strings.Contains(string(jsonData), "time_series_v2") -} - -func isDashboardWithLogsClickhouseQuery(data map[string]interface{}) bool { - jsonData, err := json.Marshal(data) - if err != nil { - return false - } - result := strings.Contains(string(jsonData), "signoz_logs.distributed_logs") || - strings.Contains(string(jsonData), "signoz_logs.logs") - return result -} - -func isDashboardWithPanelAndName(data map[string]interface{}) bool { - isDashboardName := false - isDashboardWithPanelAndName := false - if data != nil && data["title"] != nil && data["widgets"] != nil { - title, ok := data["title"].(string) - if ok && title != "Sample Title" { - isDashboardName = true - } - widgets, ok := data["widgets"] - if ok && isDashboardName { - data, ok := widgets.([]interface{}) - if ok && len(data) > 0 { - isDashboardWithPanelAndName = true - } - } - } - - return isDashboardWithPanelAndName -} - -func extractDashboardName(data map[string]interface{}) string { - - if data != nil && data["title"] != nil { - title, ok := data["title"].(string) - if ok { - return title - } - } - - return "" -} - -func countPanelsInDashboard(data map[string]interface{}) model.DashboardsInfo { - var logsPanelCount, tracesPanelCount, metricsPanelCount int - // totalPanels := 0 - if data != nil && data["widgets"] != nil { - widgets, ok := data["widgets"] - if ok { - data, ok := widgets.([]interface{}) - if ok { - for _, widget := range data { - sData, ok := widget.(map[string]interface{}) - if ok && sData["query"] != nil { - // totalPanels++ - query, ok := sData["query"].(map[string]interface{}) - if ok && query["queryType"] == "builder" && query["builder"] != nil { - builderData, ok := query["builder"].(map[string]interface{}) - if ok && builderData["queryData"] != nil { - builderQueryData, ok := builderData["queryData"].([]interface{}) - if ok { - for _, queryData := range builderQueryData { - data, ok := queryData.(map[string]interface{}) - if ok { - if data["dataSource"] == "traces" { - tracesPanelCount++ - } else if data["dataSource"] == "metrics" { - metricsPanelCount++ - } else if data["dataSource"] == "logs" { - logsPanelCount++ - } - } - } - } - } - } - } - } - } - } - } - return model.DashboardsInfo{ - LogsBasedPanels: logsPanelCount, - TracesBasedPanels: tracesPanelCount, - MetricBasedPanels: metricsPanelCount, - } -} - -func (r *ClickHouseReader) GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error) { - savedViewsInfo := model.SavedViewsInfo{} - savedViews, err := explorer.GetViews() - if err != nil { - zap.S().Debug("Error in fetching saved views info: ", err) - return &savedViewsInfo, err - } - savedViewsInfo.TotalSavedViews = len(savedViews) - for _, view := range savedViews { - if view.SourcePage == "traces" { - savedViewsInfo.TracesSavedViews += 1 - } else if view.SourcePage == "logs" { - savedViewsInfo.LogsSavedViews += 1 - } - } - return &savedViewsInfo, nil -} - -func (r *ClickHouseReader) GetUsers(ctx context.Context) ([]model.UserPayload, error) { - - users, apiErr := dao.DB().GetUsers(ctx) - if apiErr != nil { - return nil, apiErr.Err - } - return users, nil -} - func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) { // response will contain top level fields from the otel log model response := model.GetFieldsResponse{ diff --git a/pkg/query-service/app/dashboards/model.go b/pkg/query-service/app/dashboards/model.go index 306934c44355..1d7e51e82013 100644 --- a/pkg/query-service/app/dashboards/model.go +++ b/pkg/query-service/app/dashboards/model.go @@ -15,6 +15,8 @@ import ( "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/interfaces" "go.signoz.io/signoz/pkg/query-service/model" + + "go.signoz.io/signoz/pkg/query-service/telemetry" "go.uber.org/zap" ) @@ -149,6 +151,8 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { return nil, fmt.Errorf("error in adding column locked to dashboards table: %s", err.Error()) } + telemetry.GetInstance().SetDashboardsInfoCallback(GetDashboardsInfo) + return db, nil } @@ -434,3 +438,126 @@ func getIdDifference(existingIds []string, newIds []string) []string { return difference } + +// GetDashboardsInfo returns analytics data for dashboards +func GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) { + dashboardsInfo := model.DashboardsInfo{} + // fetch dashboards from dashboard db + query := "SELECT data FROM dashboards" + var dashboardsData []Dashboard + err := db.Select(&dashboardsData, query) + if err != nil { + zap.L().Error("Error in processing sql query", zap.Error(err)) + return &dashboardsInfo, err + } + totalDashboardsWithPanelAndName := 0 + var dashboardNames []string + count := 0 + for _, dashboard := range dashboardsData { + if isDashboardWithPanelAndName(dashboard.Data) { + totalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName + 1 + } + dashboardName := extractDashboardName(dashboard.Data) + if dashboardName != "" { + dashboardNames = append(dashboardNames, dashboardName) + } + dashboardInfo := countPanelsInDashboard(dashboard.Data) + dashboardsInfo.LogsBasedPanels += dashboardInfo.LogsBasedPanels + dashboardsInfo.TracesBasedPanels += dashboardInfo.TracesBasedPanels + dashboardsInfo.MetricBasedPanels += dashboardsInfo.MetricBasedPanels + if isDashboardWithTSV2(dashboard.Data) { + count = count + 1 + } + } + + dashboardsInfo.DashboardNames = dashboardNames + dashboardsInfo.TotalDashboards = len(dashboardsData) + dashboardsInfo.TotalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName + dashboardsInfo.QueriesWithTSV2 = count + return &dashboardsInfo, nil +} + +func isDashboardWithTSV2(data map[string]interface{}) bool { + jsonData, err := json.Marshal(data) + if err != nil { + return false + } + return strings.Contains(string(jsonData), "time_series_v2") +} + +func isDashboardWithPanelAndName(data map[string]interface{}) bool { + isDashboardName := false + isDashboardWithPanelAndName := false + if data != nil && data["title"] != nil && data["widgets"] != nil { + title, ok := data["title"].(string) + if ok && title != "Sample Title" { + isDashboardName = true + } + widgets, ok := data["widgets"] + if ok && isDashboardName { + data, ok := widgets.([]interface{}) + if ok && len(data) > 0 { + isDashboardWithPanelAndName = true + } + } + } + + return isDashboardWithPanelAndName +} + +func extractDashboardName(data map[string]interface{}) string { + + if data != nil && data["title"] != nil { + title, ok := data["title"].(string) + if ok { + return title + } + } + + return "" +} + +func countPanelsInDashboard(data map[string]interface{}) model.DashboardsInfo { + var logsPanelCount, tracesPanelCount, metricsPanelCount int + // totalPanels := 0 + if data != nil && data["widgets"] != nil { + widgets, ok := data["widgets"] + if ok { + data, ok := widgets.([]interface{}) + if ok { + for _, widget := range data { + sData, ok := widget.(map[string]interface{}) + if ok && sData["query"] != nil { + // totalPanels++ + query, ok := sData["query"].(map[string]interface{}) + if ok && query["queryType"] == "builder" && query["builder"] != nil { + builderData, ok := query["builder"].(map[string]interface{}) + if ok && builderData["queryData"] != nil { + builderQueryData, ok := builderData["queryData"].([]interface{}) + if ok { + for _, queryData := range builderQueryData { + data, ok := queryData.(map[string]interface{}) + if ok { + if data["dataSource"] == "traces" { + tracesPanelCount++ + } else if data["dataSource"] == "metrics" { + metricsPanelCount++ + } else if data["dataSource"] == "logs" { + logsPanelCount++ + } + } + } + } + } + } + } + } + } + } + } + return model.DashboardsInfo{ + LogsBasedPanels: logsPanelCount, + TracesBasedPanels: tracesPanelCount, + MetricBasedPanels: metricsPanelCount, + } +} diff --git a/pkg/query-service/app/explorer/db.go b/pkg/query-service/app/explorer/db.go index e0fbee4e5184..140b0b48d809 100644 --- a/pkg/query-service/app/explorer/db.go +++ b/pkg/query-service/app/explorer/db.go @@ -10,7 +10,10 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" "go.signoz.io/signoz/pkg/query-service/auth" + "go.signoz.io/signoz/pkg/query-service/model" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/telemetry" + "go.uber.org/zap" ) var db *sqlx.DB @@ -57,6 +60,8 @@ func InitWithDSN(dataSourceName string) (*sqlx.DB, error) { return nil, fmt.Errorf("error in creating saved views table: %s", err.Error()) } + telemetry.GetInstance().SetSavedViewsInfoCallback(GetSavedViewsInfo) + return db, nil } @@ -228,3 +233,21 @@ func DeleteView(uuid_ string) error { } return nil } + +func GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error) { + savedViewsInfo := model.SavedViewsInfo{} + savedViews, err := GetViews() + if err != nil { + zap.S().Debug("Error in fetching saved views info: ", err) + return &savedViewsInfo, err + } + savedViewsInfo.TotalSavedViews = len(savedViews) + for _, view := range savedViews { + if view.SourcePage == "traces" { + savedViewsInfo.TracesSavedViews += 1 + } else if view.SourcePage == "logs" { + savedViewsInfo.LogsSavedViews += 1 + } + } + return &savedViewsInfo, nil +} diff --git a/pkg/query-service/dao/sqlite/connection.go b/pkg/query-service/dao/sqlite/connection.go index a4373d5ecde5..a7e77334ef08 100644 --- a/pkg/query-service/dao/sqlite/connection.go +++ b/pkg/query-service/dao/sqlite/connection.go @@ -105,6 +105,7 @@ func InitDB(dataSourceName string) (*ModelDaoSqlite, error) { telemetry.GetInstance().SetUserCountCallback(mds.GetUserCount) telemetry.GetInstance().SetUserRoleCallback(mds.GetUserRole) + telemetry.GetInstance().SetGetUsersCallback(mds.GetUsers) return mds, nil } diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 9cf8857b92de..dd5b26151ce8 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -71,8 +71,6 @@ type Reader interface { LiveTailLogsV3(ctx context.Context, query string, timestampStart uint64, idStart string, client *model.LogsLiveTailClient) LiveTailLogsV4(ctx context.Context, query string, timestampStart uint64, idStart string, client *model.LogsLiveTailClientV2) - GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) - GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error) GetTotalSpans(ctx context.Context) (uint64, error) GetTotalLogs(ctx context.Context) (uint64, error) GetTotalSamples(ctx context.Context) (uint64, error) @@ -91,7 +89,6 @@ type Reader interface { GetLogAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) GetLogAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) GetLogAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) - GetUsers(ctx context.Context) ([]model.UserPayload, error) GetQBFilterSuggestionsForLogs( ctx context.Context, req *v3.QBFilterSuggestionsRequest, diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 048d23498fb8..aad1745907ea 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -178,9 +178,12 @@ type Telemetry struct { patTokenUser bool mutex sync.RWMutex - alertsInfoCallback func(ctx context.Context) (*model.AlertsInfo, error) - userCountCallback func(ctx context.Context) (int, error) - userRoleCallback func(ctx context.Context, groupId string) (string, error) + alertsInfoCallback func(ctx context.Context) (*model.AlertsInfo, error) + userCountCallback func(ctx context.Context) (int, error) + userRoleCallback func(ctx context.Context, groupId string) (string, error) + getUsersCallback func(ctx context.Context) ([]model.UserPayload, *model.ApiError) + dashboardsInfoCallback func(ctx context.Context) (*model.DashboardsInfo, error) + savedViewsInfoCallback func(ctx context.Context) (*model.SavedViewsInfo, error) } func (a *Telemetry) SetAlertsInfoCallback(callback func(ctx context.Context) (*model.AlertsInfo, error)) { @@ -195,6 +198,18 @@ func (a *Telemetry) SetUserRoleCallback(callback func(ctx context.Context, group a.userRoleCallback = callback } +func (a *Telemetry) SetGetUsersCallback(callback func(ctx context.Context) ([]model.UserPayload, *model.ApiError)) { + a.getUsersCallback = callback +} + +func (a *Telemetry) SetSavedViewsInfoCallback(callback func(ctx context.Context) (*model.SavedViewsInfo, error)) { + a.savedViewsInfoCallback = callback +} + +func (a *Telemetry) SetDashboardsInfoCallback(callback func(ctx context.Context) (*model.DashboardsInfo, error)) { + a.dashboardsInfoCallback = callback +} + func createTelemetry() { // Do not do anything in CI (not even resolving the outbound IP address) if testing.Testing() { @@ -296,7 +311,7 @@ func createTelemetry() { data[key] = value } - users, apiErr := telemetry.reader.GetUsers(ctx) + users, apiErr := telemetry.getUsersCallback(ctx) if apiErr == nil { for _, user := range users { if user.Email == DEFAULT_CLOUD_EMAIL { @@ -308,7 +323,7 @@ func createTelemetry() { alertsInfo, err := telemetry.alertsInfoCallback(ctx) if err == nil { - dashboardsInfo, err := telemetry.reader.GetDashboardsInfo(ctx) + dashboardsInfo, err := telemetry.dashboardsInfoCallback(ctx) if err == nil { channels, err := telemetry.reader.GetChannels() if err == nil { @@ -328,7 +343,7 @@ func createTelemetry() { alertsInfo.MSTeamsChannels++ } } - savedViewsInfo, err := telemetry.reader.GetSavedViewsInfo(ctx) + savedViewsInfo, err := telemetry.savedViewsInfoCallback(ctx) if err == nil { dashboardsAlertsData := map[string]interface{}{ "totalDashboards": dashboardsInfo.TotalDashboards,