From a1fa2769e494e3fdb5c7bcf836b96533b056d8dc Mon Sep 17 00:00:00 2001 From: Vibhu Pandey Date: Mon, 9 Jun 2025 16:43:29 +0530 Subject: [PATCH] feat(statsreporter): build a statsreporter service (#8177) - build a new statsreporter service --- .github/workflows/build-community.yaml | 3 +- .github/workflows/build-enterprise.yaml | 3 +- .github/workflows/build-staging.yaml | 3 +- conf/example.yaml | 21 +- ee/licensing/httplicensing/provider.go | 13 ++ ee/query-service/.goreleaser.yaml | 1 + pkg/alertmanager/alertmanager.go | 4 + .../legacyalertmanager/provider.go | 9 + .../signozalertmanager/provider.go | 9 + pkg/analytics/analyticstest/provider.go | 30 +++ pkg/analytics/config.go | 26 ++- pkg/analytics/noopanalytics/provider.go | 14 +- pkg/analytics/segmentanalytics/logger.go | 28 +++ pkg/analytics/segmentanalytics/provider.go | 25 +- pkg/http/middleware/analytics.go | 17 +- pkg/licensing/licensing.go | 3 + pkg/licensing/nooplicensing/provider.go | 4 + pkg/modules/dashboard/dashboard.go | 5 +- .../dashboard/impldashboard/handler.go | 2 +- pkg/modules/dashboard/impldashboard/module.go | 39 +++- .../organization/implorganization/getter.go | 4 - pkg/modules/organization/organization.go | 3 - pkg/modules/savedview/implsavedview/module.go | 17 ++ pkg/modules/savedview/savedview.go | 3 + pkg/modules/user/impluser/module.go | 82 ++++++- pkg/modules/user/impluser/store.go | 17 ++ pkg/modules/user/user.go | 3 + pkg/query-service/.goreleaser.yaml | 1 + .../app/cloudintegrations/controller_test.go | 13 +- .../app/integrations/manager_test.go | 4 +- pkg/query-service/constants/constants.go | 5 + pkg/query-service/rules/manager.go | 2 +- pkg/query-service/telemetry/ignored.go | 8 - pkg/query-service/telemetry/telemetry.go | 1 - .../integration/filter_suggestions_test.go | 4 +- .../integration/logparsingpipeline_test.go | 4 +- .../signoz_cloud_integrations_test.go | 4 +- .../integration/signoz_integrations_test.go | 4 +- pkg/ruler/config.go | 20 ++ pkg/ruler/ruler.go | 7 + pkg/ruler/rulestore/sqlrulestore/rule.go | 6 +- pkg/ruler/signozruler/provider.go | 35 +++ pkg/signoz/config.go | 25 ++ pkg/signoz/handler_test.go | 2 +- pkg/signoz/module.go | 6 +- pkg/signoz/module_test.go | 2 +- pkg/signoz/provider.go | 29 +++ pkg/signoz/provider_test.go | 13 ++ pkg/signoz/signoz.go | 58 ++++- .../analyticsstatsreporter/provider.go | 216 ++++++++++++++++++ pkg/statsreporter/config.go | 38 +++ .../noopstatsreporter/provider.go | 36 +++ pkg/statsreporter/statsreporter.go | 18 ++ pkg/types/alertmanagertypes/channel.go | 16 ++ pkg/types/analyticstypes/message.go | 24 ++ pkg/types/dashboardtypes/dashboard.go | 62 +++++ pkg/types/licensetypes/license.go | 7 + pkg/types/ruletypes/rule.go | 29 +++ pkg/types/savedview.go | 17 ++ pkg/types/user.go | 2 + pkg/version/deployment.go | 155 +++++++++++++ 61 files changed, 1163 insertions(+), 98 deletions(-) create mode 100644 pkg/analytics/analyticstest/provider.go create mode 100644 pkg/analytics/segmentanalytics/logger.go create mode 100644 pkg/ruler/config.go create mode 100644 pkg/ruler/ruler.go create mode 100644 pkg/ruler/signozruler/provider.go create mode 100644 pkg/statsreporter/analyticsstatsreporter/provider.go create mode 100644 pkg/statsreporter/config.go create mode 100644 pkg/statsreporter/noopstatsreporter/provider.go create mode 100644 pkg/statsreporter/statsreporter.go create mode 100644 pkg/version/deployment.go diff --git a/.github/workflows/build-community.yaml b/.github/workflows/build-community.yaml index a4f0ba93f15d..b0d0ec70b326 100644 --- a/.github/workflows/build-community.yaml +++ b/.github/workflows/build-community.yaml @@ -74,7 +74,8 @@ jobs: -X github.com/SigNoz/signoz/pkg/version.variant=community -X github.com/SigNoz/signoz/pkg/version.hash=${{ needs.prepare.outputs.hash }} -X github.com/SigNoz/signoz/pkg/version.time=${{ needs.prepare.outputs.time }} - -X github.com/SigNoz/signoz/pkg/version.branch=${{ needs.prepare.outputs.branch }}' + -X github.com/SigNoz/signoz/pkg/version.branch=${{ needs.prepare.outputs.branch }} + -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr' GO_CGO_ENABLED: 1 DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}' DOCKER_DOCKERFILE_PATH: ./pkg/query-service/Dockerfile.multi-arch diff --git a/.github/workflows/build-enterprise.yaml b/.github/workflows/build-enterprise.yaml index 99b80bafa700..36aba8533f98 100644 --- a/.github/workflows/build-enterprise.yaml +++ b/.github/workflows/build-enterprise.yaml @@ -108,7 +108,8 @@ jobs: -X github.com/SigNoz/signoz/ee/zeus.url=https://api.signoz.cloud -X github.com/SigNoz/signoz/ee/zeus.deprecatedURL=https://license.signoz.io -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.signoz.cloud - -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.signoz.io/api/v1' + -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.signoz.io/api/v1 + -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr' GO_CGO_ENABLED: 1 DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}' DOCKER_DOCKERFILE_PATH: ./ee/query-service/Dockerfile.multi-arch diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index 9006f0233955..dec3ba121772 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -107,7 +107,8 @@ jobs: -X github.com/SigNoz/signoz/ee/zeus.url=https://api.staging.signoz.cloud -X github.com/SigNoz/signoz/ee/zeus.deprecatedURL=https://license.staging.signoz.cloud -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.staging.signoz.cloud - -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.staging.signoz.cloud/api/v1' + -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.staging.signoz.cloud/api/v1 + -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr' GO_CGO_ENABLED: 1 DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}' DOCKER_DOCKERFILE_PATH: ./ee/query-service/Dockerfile.multi-arch diff --git a/conf/example.yaml b/conf/example.yaml index 7c9f285da52d..46c3a7b02644 100644 --- a/conf/example.yaml +++ b/conf/example.yaml @@ -165,12 +165,6 @@ alertmanager: # Retention of the notification logs. retention: 120h - -##################### Analytics ##################### -analytics: - # Whether to enable analytics. - enabled: false - ##################### Emailing ##################### emailing: # Whether to enable emailing. @@ -215,3 +209,18 @@ sharder: single: # The org id to which this instance belongs to. org_id: org_id + +##################### Analytics ##################### +analytics: + # Whether to enable analytics. + enabled: false + segment: + # The key to use for segment. + key: "" + +##################### StatsReporter ##################### +statsreporter: + # Whether to enable stats reporter. This is used to provide valuable insights to the SigNoz team. It does not collect any sensitive/PII data. + enabled: true + # The interval at which the stats are collected. + interval: 6h diff --git a/ee/licensing/httplicensing/provider.go b/ee/licensing/httplicensing/provider.go index 34cf90c46ee7..69dd05c7601d 100644 --- a/ee/licensing/httplicensing/provider.go +++ b/ee/licensing/httplicensing/provider.go @@ -211,3 +211,16 @@ func (provider *provider) GetFeatureFlags(ctx context.Context, organizationID va return license.Features, nil } + +func (provider *provider) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + activeLicense, err := provider.GetActive(ctx, orgID) + if err != nil { + if errors.Ast(err, errors.TypeNotFound) { + return map[string]any{}, nil + } + + return nil, err + } + + return licensetypes.NewStatsFromLicense(activeLicense), nil +} diff --git a/ee/query-service/.goreleaser.yaml b/ee/query-service/.goreleaser.yaml index c4bf6cb011e7..e3982597d11c 100644 --- a/ee/query-service/.goreleaser.yaml +++ b/ee/query-service/.goreleaser.yaml @@ -39,6 +39,7 @@ builds: - -X github.com/SigNoz/signoz/ee/zeus.deprecatedURL=https://license.signoz.io - -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.signoz.cloud - -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.signoz.io/api/v1 + - -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr - >- {{- if eq .Os "linux" }}-linkmode external -extldflags '-static'{{- end }} mod_timestamp: "{{ .CommitTimestamp }}" diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index 6d89b0b463ac..18b790f4de76 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -5,6 +5,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/types/alertmanagertypes" "github.com/SigNoz/signoz/pkg/valuer" ) @@ -53,4 +54,7 @@ type Alertmanager interface { // SetDefaultConfig sets the default config for the organization. SetDefaultConfig(context.Context, string) error + + // Collects stats for the organization. + statsreporter.StatsCollector } diff --git a/pkg/alertmanager/legacyalertmanager/provider.go b/pkg/alertmanager/legacyalertmanager/provider.go index 33dbd54aff6a..13137e196c08 100644 --- a/pkg/alertmanager/legacyalertmanager/provider.go +++ b/pkg/alertmanager/legacyalertmanager/provider.go @@ -471,3 +471,12 @@ func (provider *provider) SetDefaultConfig(ctx context.Context, orgID string) er return provider.configStore.Set(ctx, config) } + +func (provider *provider) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + channels, err := provider.configStore.ListChannels(ctx, orgID.String()) + if err != nil { + return nil, err + } + + return alertmanagertypes.NewStatsFromChannels(channels), nil +} diff --git a/pkg/alertmanager/signozalertmanager/provider.go b/pkg/alertmanager/signozalertmanager/provider.go index 8c3c0872ae66..3f2b274585c7 100644 --- a/pkg/alertmanager/signozalertmanager/provider.go +++ b/pkg/alertmanager/signozalertmanager/provider.go @@ -182,3 +182,12 @@ func (provider *provider) SetDefaultConfig(ctx context.Context, orgID string) er return provider.configStore.Set(ctx, config) } + +func (provider *provider) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + channels, err := provider.configStore.ListChannels(ctx, orgID.String()) + if err != nil { + return nil, err + } + + return alertmanagertypes.NewStatsFromChannels(channels), nil +} diff --git a/pkg/analytics/analyticstest/provider.go b/pkg/analytics/analyticstest/provider.go new file mode 100644 index 000000000000..663d8bf96b17 --- /dev/null +++ b/pkg/analytics/analyticstest/provider.go @@ -0,0 +1,30 @@ +package analyticstest + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/analytics" + "github.com/SigNoz/signoz/pkg/types/analyticstypes" +) + +var _ analytics.Analytics = (*Provider)(nil) + +type Provider struct { + stopC chan struct{} +} + +func New() *Provider { + return &Provider{stopC: make(chan struct{})} +} + +func (provider *Provider) Start(_ context.Context) error { + <-provider.stopC + return nil +} + +func (provider *Provider) Send(ctx context.Context, messages ...analyticstypes.Message) {} + +func (provider *Provider) Stop(_ context.Context) error { + close(provider.stopC) + return nil +} diff --git a/pkg/analytics/config.go b/pkg/analytics/config.go index 1bcbdd31c2ac..77880e001d07 100644 --- a/pkg/analytics/config.go +++ b/pkg/analytics/config.go @@ -1,8 +1,6 @@ package analytics import ( - "fmt" - "github.com/SigNoz/signoz/pkg/factory" ) @@ -12,8 +10,12 @@ var ( ) type Config struct { - Enabled bool `mapstructure:"enabled"` - Key string `mapstructure:"key"` + Enabled bool `mapstructure:"enabled"` + Segment Segment `mapstructure:"segment"` +} + +type Segment struct { + Key string `mapstructure:"key"` } func NewConfigFactory() factory.ConfigFactory { @@ -23,14 +25,20 @@ func NewConfigFactory() factory.ConfigFactory { func newConfig() factory.Config { return Config{ Enabled: false, - Key: key, + Segment: Segment{ + Key: key, + }, } } func (c Config) Validate() error { - if c.Key != key { - return fmt.Errorf("cannot override key set at build time with key: %s", c.Key) - } - return nil } + +func (c Config) Provider() string { + if c.Enabled { + return "segment" + } + + return "noop" +} diff --git a/pkg/analytics/noopanalytics/provider.go b/pkg/analytics/noopanalytics/provider.go index 0e243343e2c7..3e6b8dfbcb55 100644 --- a/pkg/analytics/noopanalytics/provider.go +++ b/pkg/analytics/noopanalytics/provider.go @@ -9,31 +9,27 @@ import ( ) type provider struct { - settings factory.ScopedProviderSettings - startC chan struct{} + stopC chan struct{} } -func NewProviderFactory() factory.ProviderFactory[analytics.Analytics, analytics.Config] { +func NewFactory() factory.ProviderFactory[analytics.Analytics, analytics.Config] { return factory.NewProviderFactory(factory.MustNewName("noop"), New) } func New(ctx context.Context, providerSettings factory.ProviderSettings, config analytics.Config) (analytics.Analytics, error) { - settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/analytics/noopanalytics") - return &provider{ - settings: settings, - startC: make(chan struct{}), + stopC: make(chan struct{}), }, nil } func (provider *provider) Start(_ context.Context) error { - <-provider.startC + <-provider.stopC return nil } func (provider *provider) Send(ctx context.Context, messages ...analyticstypes.Message) {} func (provider *provider) Stop(_ context.Context) error { - close(provider.startC) + close(provider.stopC) return nil } diff --git a/pkg/analytics/segmentanalytics/logger.go b/pkg/analytics/segmentanalytics/logger.go new file mode 100644 index 000000000000..0be6282ca220 --- /dev/null +++ b/pkg/analytics/segmentanalytics/logger.go @@ -0,0 +1,28 @@ +package segmentanalytics + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + segment "github.com/segmentio/analytics-go/v3" +) + +type logger struct { + settings factory.ScopedProviderSettings +} + +func newSegmentLogger(settings factory.ScopedProviderSettings) segment.Logger { + return &logger{ + settings: settings, + } +} + +func (logger *logger) Logf(format string, args ...interface{}) { + // the no lint directive is needed because the segmentlogger is not a slog.Logger + logger.settings.Logger().InfoContext(context.TODO(), format, args...) //nolint:sloglint +} + +func (logger *logger) Errorf(format string, args ...interface{}) { + // the no lint directive is needed because the segment logger is not a slog.Logger + logger.settings.Logger().ErrorContext(context.TODO(), format, args...) //nolint:sloglint +} diff --git a/pkg/analytics/segmentanalytics/provider.go b/pkg/analytics/segmentanalytics/provider.go index 35fbf8820614..e11295106834 100644 --- a/pkg/analytics/segmentanalytics/provider.go +++ b/pkg/analytics/segmentanalytics/provider.go @@ -12,25 +12,32 @@ import ( type provider struct { settings factory.ScopedProviderSettings client segment.Client - startC chan struct{} + stopC chan struct{} } -func NewProviderFactory() factory.ProviderFactory[analytics.Analytics, analytics.Config] { +func NewFactory() factory.ProviderFactory[analytics.Analytics, analytics.Config] { return factory.NewProviderFactory(factory.MustNewName("segment"), New) } func New(ctx context.Context, providerSettings factory.ProviderSettings, config analytics.Config) (analytics.Analytics, error) { settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/analytics/segmentanalytics") + client, err := segment.NewWithConfig(config.Segment.Key, segment.Config{ + Logger: newSegmentLogger(settings), + }) + if err != nil { + return nil, err + } + return &provider{ settings: settings, - client: segment.New(config.Key), - startC: make(chan struct{}), + client: client, + stopC: make(chan struct{}), }, nil } func (provider *provider) Start(_ context.Context) error { - <-provider.startC + <-provider.stopC return nil } @@ -43,7 +50,11 @@ func (provider *provider) Send(ctx context.Context, messages ...analyticstypes.M } } -func (provider *provider) Stop(_ context.Context) error { - close(provider.startC) +func (provider *provider) Stop(ctx context.Context) error { + if err := provider.client.Close(); err != nil { + provider.settings.Logger().WarnContext(ctx, "unable to close segment client", "err", err) + } + + close(provider.stopC) return nil } diff --git a/pkg/http/middleware/analytics.go b/pkg/http/middleware/analytics.go index db8d871b12a7..48ca5691046b 100644 --- a/pkg/http/middleware/analytics.go +++ b/pkg/http/middleware/analytics.go @@ -24,27 +24,12 @@ func (a *Analytics) Wrap(next http.Handler) http.Handler { route := mux.CurrentRoute(r) path, _ := route.GetPathTemplate() - queryRangeData, metadataExists := a.extractQueryRangeData(path, r) + _, _ = a.extractQueryRangeData(path, r) a.getActiveLogs(path, r) badResponseBuffer := new(bytes.Buffer) writer := newBadResponseLoggingWriter(w, badResponseBuffer) next.ServeHTTP(writer, r) - - data := map[string]interface{}{"path": path, "statusCode": writer.StatusCode()} - if metadataExists { - for key, value := range queryRangeData { - data[key] = value - } - } - - if _, ok := telemetry.EnabledPaths()[path]; ok { - claims, err := authtypes.ClaimsFromContext(r.Context()) - if err == nil { - telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data, claims.Email, true, false) - } - } - }) } diff --git a/pkg/licensing/licensing.go b/pkg/licensing/licensing.go index 25385ee9af92..3d77d7e156da 100644 --- a/pkg/licensing/licensing.go +++ b/pkg/licensing/licensing.go @@ -6,6 +6,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/types/licensetypes" "github.com/SigNoz/signoz/pkg/valuer" ) @@ -32,6 +33,8 @@ type Licensing interface { Portal(ctx context.Context, organizationID valuer.UUID, postableSubscription *licensetypes.PostableSubscription) (*licensetypes.GettableSubscription, error) // GetFeatureFlags fetches all the defined feature flags GetFeatureFlags(ctx context.Context, organizationID valuer.UUID) ([]*licensetypes.Feature, error) + + statsreporter.StatsCollector } type API interface { diff --git a/pkg/licensing/nooplicensing/provider.go b/pkg/licensing/nooplicensing/provider.go index 396879fbe7c4..be40b968858c 100644 --- a/pkg/licensing/nooplicensing/provider.go +++ b/pkg/licensing/nooplicensing/provider.go @@ -62,3 +62,7 @@ func (provider *noopLicensing) GetActive(ctx context.Context, organizationID val func (provider *noopLicensing) GetFeatureFlags(_ context.Context, _ valuer.UUID) ([]*licensetypes.Feature, error) { return licensetypes.DefaultFeatureSet, nil } + +func (provider *noopLicensing) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + return map[string]any{}, nil +} diff --git a/pkg/modules/dashboard/dashboard.go b/pkg/modules/dashboard/dashboard.go index d8ee841f8045..dcbe2a6ac017 100644 --- a/pkg/modules/dashboard/dashboard.go +++ b/pkg/modules/dashboard/dashboard.go @@ -4,12 +4,13 @@ import ( "context" "net/http" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/types/dashboardtypes" "github.com/SigNoz/signoz/pkg/valuer" ) type Module interface { - Create(ctx context.Context, orgID valuer.UUID, createdBy string, data dashboardtypes.PostableDashboard) (*dashboardtypes.Dashboard, error) + Create(ctx context.Context, orgID valuer.UUID, createdBy string, creator valuer.UUID, data dashboardtypes.PostableDashboard) (*dashboardtypes.Dashboard, error) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*dashboardtypes.Dashboard, error) @@ -22,6 +23,8 @@ type Module interface { Delete(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]map[string]string, error) + + statsreporter.StatsCollector } type Handler interface { diff --git a/pkg/modules/dashboard/impldashboard/handler.go b/pkg/modules/dashboard/impldashboard/handler.go index 449cc228ea83..e07a691f062a 100644 --- a/pkg/modules/dashboard/impldashboard/handler.go +++ b/pkg/modules/dashboard/impldashboard/handler.go @@ -46,7 +46,7 @@ func (handler *handler) Create(rw http.ResponseWriter, r *http.Request) { return } - dashboard, err := handler.module.Create(ctx, orgID, claims.Email, req) + dashboard, err := handler.module.Create(ctx, orgID, claims.Email, valuer.MustNewUUID(claims.ID), req) if err != nil { render.Error(rw, err) return diff --git a/pkg/modules/dashboard/impldashboard/module.go b/pkg/modules/dashboard/impldashboard/module.go index 83ed50a6b338..d5d53c3479df 100644 --- a/pkg/modules/dashboard/impldashboard/module.go +++ b/pkg/modules/dashboard/impldashboard/module.go @@ -4,28 +4,32 @@ import ( "context" "strings" + "github.com/SigNoz/signoz/pkg/analytics" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/modules/dashboard" "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/SigNoz/signoz/pkg/types/analyticstypes" "github.com/SigNoz/signoz/pkg/types/dashboardtypes" "github.com/SigNoz/signoz/pkg/valuer" ) type module struct { - store dashboardtypes.Store - settings factory.ScopedProviderSettings + store dashboardtypes.Store + settings factory.ScopedProviderSettings + analytics analytics.Analytics } -func NewModule(sqlstore sqlstore.SQLStore, settings factory.ProviderSettings) dashboard.Module { +func NewModule(sqlstore sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics) dashboard.Module { scopedProviderSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/modules/impldashboard") return &module{ - store: NewStore(sqlstore), - settings: scopedProviderSettings, + store: NewStore(sqlstore), + settings: scopedProviderSettings, + analytics: analytics, } } -func (module *module) Create(ctx context.Context, orgID valuer.UUID, createdBy string, postableDashboard dashboardtypes.PostableDashboard) (*dashboardtypes.Dashboard, error) { +func (module *module) Create(ctx context.Context, orgID valuer.UUID, createdBy string, creator valuer.UUID, postableDashboard dashboardtypes.PostableDashboard) (*dashboardtypes.Dashboard, error) { dashboard, err := dashboardtypes.NewDashboard(orgID, createdBy, postableDashboard) if err != nil { return nil, err @@ -35,11 +39,25 @@ func (module *module) Create(ctx context.Context, orgID valuer.UUID, createdBy s if err != nil { return nil, err } + err = module.store.Create(ctx, storableDashboard) if err != nil { return nil, err } + module.analytics.Send(ctx, + analyticstypes.Track{ + UserId: creator.String(), + Event: "Dashboard Created", + Properties: analyticstypes.NewPropertiesFromMap(dashboardtypes.NewStatsFromStorableDashboards([]*dashboardtypes.StorableDashboard{storableDashboard})), + Context: &analyticstypes.Context{ + Extra: map[string]interface{}{ + analyticstypes.KeyGroupID: orgID, + }, + }, + }, + ) + return dashboard, nil } @@ -207,3 +225,12 @@ func (module *module) GetByMetricNames(ctx context.Context, orgID valuer.UUID, m return result, nil } + +func (module *module) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + dashboards, err := module.store.List(ctx, orgID) + if err != nil { + return nil, err + } + + return dashboardtypes.NewStatsFromStorableDashboards(dashboards), nil +} diff --git a/pkg/modules/organization/implorganization/getter.go b/pkg/modules/organization/implorganization/getter.go index 78c684ece3e0..599000b7772a 100644 --- a/pkg/modules/organization/implorganization/getter.go +++ b/pkg/modules/organization/implorganization/getter.go @@ -22,10 +22,6 @@ func (module *getter) Get(ctx context.Context, id valuer.UUID) (*types.Organizat return module.store.Get(ctx, id) } -func (module *getter) List(ctx context.Context) ([]*types.Organization, error) { - return module.store.GetAll(ctx) -} - func (module *getter) ListByOwnedKeyRange(ctx context.Context) ([]*types.Organization, error) { start, end, err := module.sharder.GetMyOwnedKeyRange(ctx) if err != nil { diff --git a/pkg/modules/organization/organization.go b/pkg/modules/organization/organization.go index a48ef682d6b8..5dca6b0a5b0e 100644 --- a/pkg/modules/organization/organization.go +++ b/pkg/modules/organization/organization.go @@ -12,9 +12,6 @@ type Getter interface { // Get gets the organization based on the given id Get(context.Context, valuer.UUID) (*types.Organization, error) - // Lists all the organizations - List(context.Context) ([]*types.Organization, error) - // ListByOwnedKeyRange gets all the organizations owned by the instance ListByOwnedKeyRange(context.Context) ([]*types.Organization, error) } diff --git a/pkg/modules/savedview/implsavedview/module.go b/pkg/modules/savedview/implsavedview/module.go index 743960acb68e..f6fb34dbd545 100644 --- a/pkg/modules/savedview/implsavedview/module.go +++ b/pkg/modules/savedview/implsavedview/module.go @@ -169,3 +169,20 @@ func (module *module) DeleteView(ctx context.Context, orgID string, uuid valuer. } return nil } + +func (module *module) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + savedViews := []*types.SavedView{} + + err := module. + sqlstore. + BunDB(). + NewSelect(). + Model(&savedViews). + Where("org_id = ?", orgID). + Scan(ctx) + if err != nil { + return nil, err + } + + return types.NewStatsFromSavedViews(savedViews), nil +} diff --git a/pkg/modules/savedview/savedview.go b/pkg/modules/savedview/savedview.go index 8a8ff5a8b175..85f1d8cf7ad9 100644 --- a/pkg/modules/savedview/savedview.go +++ b/pkg/modules/savedview/savedview.go @@ -5,6 +5,7 @@ import ( "net/http" v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/valuer" ) @@ -18,6 +19,8 @@ type Module interface { UpdateView(ctx context.Context, orgID string, uuid valuer.UUID, view v3.SavedView) error DeleteView(ctx context.Context, orgID string, uuid valuer.UUID) error + + statsreporter.StatsCollector } type Handler interface { diff --git a/pkg/modules/user/impluser/module.go b/pkg/modules/user/impluser/module.go index 45eaa4ba668b..d1dbc30744aa 100644 --- a/pkg/modules/user/impluser/module.go +++ b/pkg/modules/user/impluser/module.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/SigNoz/signoz/pkg/analytics" "github.com/SigNoz/signoz/pkg/emailing" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" @@ -17,6 +18,7 @@ import ( "github.com/SigNoz/signoz/pkg/query-service/model" "github.com/SigNoz/signoz/pkg/query-service/telemetry" "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/types/analyticstypes" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/types/emailtypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -29,10 +31,11 @@ type Module struct { emailing emailing.Emailing settings factory.ScopedProviderSettings orgSetter organization.Setter + analytics analytics.Analytics } // This module is a WIP, don't take inspiration from this. -func NewModule(store types.UserStore, jwt *authtypes.JWT, emailing emailing.Emailing, providerSettings factory.ProviderSettings, orgSetter organization.Setter) user.Module { +func NewModule(store types.UserStore, jwt *authtypes.JWT, emailing emailing.Emailing, providerSettings factory.ProviderSettings, orgSetter organization.Setter, analytics analytics.Analytics) user.Module { settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/user/impluser") return &Module{ store: store, @@ -40,6 +43,7 @@ func NewModule(store types.UserStore, jwt *authtypes.JWT, emailing emailing.Emai emailing: emailing, settings: settings, orgSetter: orgSetter, + analytics: analytics, } } @@ -126,17 +130,80 @@ func (m *Module) GetInviteByEmailInOrg(ctx context.Context, orgID string, email } func (m *Module) CreateUserWithPassword(ctx context.Context, user *types.User, password *types.FactorPassword) (*types.User, error) { - user, err := m.store.CreateUserWithPassword(ctx, user, password) if err != nil { return nil, err } + m.analytics.Send(ctx, + analyticstypes.Identify{ + UserId: user.ID.String(), + Traits: analyticstypes. + NewTraits(). + SetName(user.DisplayName). + SetEmail(user.Email). + Set("role", user.Role). + SetCreatedAt(user.CreatedAt), + }, + analyticstypes.Group{ + UserId: user.ID.String(), + GroupId: user.OrgID, + }, + analyticstypes.Track{ + UserId: user.ID.String(), + Event: "User Created", + Properties: analyticstypes.NewPropertiesFromMap(map[string]any{ + "role": user.Role, + "email": user.Email, + "name": user.DisplayName, + }), + Context: &analyticstypes.Context{ + Extra: map[string]interface{}{ + analyticstypes.KeyGroupID: user.OrgID, + }, + }, + }, + ) + return user, nil } func (m *Module) CreateUser(ctx context.Context, user *types.User) error { - return m.store.CreateUser(ctx, user) + if err := m.store.CreateUser(ctx, user); err != nil { + return err + } + + m.analytics.Send(ctx, + analyticstypes.Identify{ + UserId: user.ID.String(), + Traits: analyticstypes. + NewTraits(). + SetName(user.DisplayName). + SetEmail(user.Email). + Set("role", user.Role). + SetCreatedAt(user.CreatedAt), + }, + analyticstypes.Group{ + UserId: user.ID.String(), + GroupId: user.OrgID, + }, + analyticstypes.Track{ + UserId: user.ID.String(), + Event: "User Created", + Properties: analyticstypes.NewPropertiesFromMap(map[string]any{ + "role": user.Role, + "email": user.Email, + "name": user.DisplayName, + }), + Context: &analyticstypes.Context{ + Extra: map[string]interface{}{ + analyticstypes.KeyGroupID: user.OrgID, + }, + }, + }, + ) + + return nil } func (m *Module) GetUserByID(ctx context.Context, orgID string, id string) (*types.GettableUser, error) { @@ -575,3 +642,12 @@ func (m *Module) Register(ctx context.Context, req *types.PostableRegisterOrgAnd return user, nil } + +func (m *Module) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + count, err := m.store.CountByOrgID(ctx, orgID) + if err != nil { + return nil, err + } + + return map[string]any{"user.count": count}, nil +} diff --git a/pkg/modules/user/impluser/store.go b/pkg/modules/user/impluser/store.go index f7208df78eb7..648f67088b36 100644 --- a/pkg/modules/user/impluser/store.go +++ b/pkg/modules/user/impluser/store.go @@ -809,3 +809,20 @@ func (store *store) DeleteDomain(ctx context.Context, id uuid.UUID) error { return nil } + +func (store *store) CountByOrgID(ctx context.Context, orgID valuer.UUID) (int64, error) { + user := new(types.User) + + count, err := store. + sqlstore. + BunDB(). + NewSelect(). + Model(user). + Where("org_id = ?", orgID). + Count(ctx) + if err != nil { + return 0, err + } + + return int64(count), nil +} diff --git a/pkg/modules/user/user.go b/pkg/modules/user/user.go index cb991ab95b9e..94eca22bb4c7 100644 --- a/pkg/modules/user/user.go +++ b/pkg/modules/user/user.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -65,6 +66,8 @@ type Module interface { // Register Register(ctx context.Context, req *types.PostableRegisterOrgAndAdmin) (*types.User, error) + + statsreporter.StatsCollector } type Handler interface { diff --git a/pkg/query-service/.goreleaser.yaml b/pkg/query-service/.goreleaser.yaml index 8bd46312059f..3ffd74d07f76 100644 --- a/pkg/query-service/.goreleaser.yaml +++ b/pkg/query-service/.goreleaser.yaml @@ -35,6 +35,7 @@ builds: - -X github.com/SigNoz/signoz/pkg/version.hash={{ .ShortCommit }} - -X github.com/SigNoz/signoz/pkg/version.time={{ .CommitTimestamp }} - -X github.com/SigNoz/signoz/pkg/version.branch={{ .Branch }} + - -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr - >- {{- if eq .Os "linux" }}-linkmode external -extldflags '-static'{{- end }} mod_timestamp: "{{ .CommitTimestamp }}" diff --git a/pkg/query-service/app/cloudintegrations/controller_test.go b/pkg/query-service/app/cloudintegrations/controller_test.go index dec3849b1cc8..a84c17a45684 100644 --- a/pkg/query-service/app/cloudintegrations/controller_test.go +++ b/pkg/query-service/app/cloudintegrations/controller_test.go @@ -8,6 +8,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization" @@ -38,7 +39,8 @@ func TestRegenerateConnectionUrlWithUpdatedConfig(t *testing.T) { require.NoError(err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) @@ -94,7 +96,8 @@ func TestAgentCheckIns(t *testing.T) { require.NoError(err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) @@ -189,7 +192,8 @@ func TestCantDisconnectNonExistentAccount(t *testing.T) { require.NoError(err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) @@ -216,7 +220,8 @@ func TestConfigureService(t *testing.T) { require.NoError(err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) diff --git a/pkg/query-service/app/integrations/manager_test.go b/pkg/query-service/app/integrations/manager_test.go index 39c9d1ff88de..43d18ba9b233 100644 --- a/pkg/query-service/app/integrations/manager_test.go +++ b/pkg/query-service/app/integrations/manager_test.go @@ -8,6 +8,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" @@ -31,7 +32,8 @@ func TestIntegrationLifecycle(t *testing.T) { alertmanager, _ := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Provider: "signoz", Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, store, orgGetter) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(store, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(store, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) user, apiErr := createTestUser(modules.OrgSetter, modules.User) if apiErr != nil { t.Fatalf("could not create test user: %v", apiErr) diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index dffdcf3a12ca..e69b88d3e75e 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -18,8 +18,10 @@ const ( OpAmpWsEndpoint = "0.0.0.0:4320" // address for opamp websocket ) +// Deprecated: Use the new analytics service instead var DEFAULT_TELEMETRY_ANONYMOUS = false +// Deprecated: Use the new analytics service instead func IsOSSTelemetryEnabled() bool { ossSegmentKey := GetOrDefaultEnv("OSS_TELEMETRY_ENABLED", "true") return ossSegmentKey == "true" @@ -27,6 +29,7 @@ func IsOSSTelemetryEnabled() bool { const MaxAllowedPointsInTimeSeries = 300 +// Deprecated: Use the new analytics service instead func IsTelemetryEnabled() bool { if testing.Testing() { return false @@ -48,8 +51,10 @@ const SpanSearchScopeRoot = "isroot" const SpanSearchScopeEntryPoint = "isentrypoint" const OrderBySpanCount = "span_count" +// Deprecated: Use the new statsreporter service instead var TELEMETRY_HEART_BEAT_DURATION_MINUTES = GetOrDefaultEnvInt("TELEMETRY_HEART_BEAT_DURATION_MINUTES", 720) +// Deprecated: Use the new statsreporter service instead var TELEMETRY_ACTIVE_USER_DURATION_MINUTES = GetOrDefaultEnvInt("TELEMETRY_ACTIVE_USER_DURATION_MINUTES", 360) // Deprecated: Use the new emailing service instead diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index a827e49d0771..1d4725957006 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -195,7 +195,7 @@ func defaultPrepareTaskFunc(opts PrepareTaskOptions) (Task, error) { // by calling the Run method. func NewManager(o *ManagerOptions) (*Manager, error) { o = defaultOptions(o) - ruleStore := sqlrulestore.NewRuleStore(o.DBConn, o.SQLStore) + ruleStore := sqlrulestore.NewRuleStore(o.SQLStore) maintenanceStore := sqlrulestore.NewMaintenanceStore(o.SQLStore) m := &Manager{ diff --git a/pkg/query-service/telemetry/ignored.go b/pkg/query-service/telemetry/ignored.go index f91ec7966c93..c359e5cf13a1 100644 --- a/pkg/query-service/telemetry/ignored.go +++ b/pkg/query-service/telemetry/ignored.go @@ -1,13 +1,5 @@ package telemetry -func EnabledPaths() map[string]struct{} { - enabledPaths := map[string]struct{}{ - "/api/v1/channels": {}, - } - - return enabledPaths -} - func ignoreEvents(event string, attributes map[string]interface{}) bool { if event == TELEMETRY_EVENT_ACTIVE_USER { diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 78754a3b90fc..acc86485a8d5 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -613,7 +613,6 @@ func (a *Telemetry) SendIdentifyEvent(data map[string]interface{}, userEmail str } func (a *Telemetry) SendGroupEvent(data map[string]interface{}, userEmail string) { - if !a.isTelemetryEnabled() || a.isTelemetryAnonymous() { return } diff --git a/pkg/query-service/tests/integration/filter_suggestions_test.go b/pkg/query-service/tests/integration/filter_suggestions_test.go index 22dda58f0722..dca7c9cbfadc 100644 --- a/pkg/query-service/tests/integration/filter_suggestions_test.go +++ b/pkg/query-service/tests/integration/filter_suggestions_test.go @@ -14,6 +14,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sharder/noopsharder" @@ -315,7 +316,8 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed { require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ diff --git a/pkg/query-service/tests/integration/logparsingpipeline_test.go b/pkg/query-service/tests/integration/logparsingpipeline_test.go index 8413d5c35828..17eff8cae5e5 100644 --- a/pkg/query-service/tests/integration/logparsingpipeline_test.go +++ b/pkg/query-service/tests/integration/logparsingpipeline_test.go @@ -14,6 +14,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" @@ -491,7 +492,8 @@ func NewTestbedWithoutOpamp(t *testing.T, sqlStore sqlstore.SQLStore) *LogPipeli require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ diff --git a/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go b/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go index 31197263cdcf..583c61ede64a 100644 --- a/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go @@ -12,6 +12,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sharder/noopsharder" @@ -376,7 +377,8 @@ func NewCloudIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *CloudI require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ diff --git a/pkg/query-service/tests/integration/signoz_integrations_test.go b/pkg/query-service/tests/integration/signoz_integrations_test.go index b5e83b7d021c..1eefde36a598 100644 --- a/pkg/query-service/tests/integration/signoz_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_integrations_test.go @@ -12,6 +12,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/http/middleware" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" @@ -582,7 +583,8 @@ func NewIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *Integration require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager) + analytics := analyticstest.New() + modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ diff --git a/pkg/ruler/config.go b/pkg/ruler/config.go new file mode 100644 index 000000000000..98b46b2c09a6 --- /dev/null +++ b/pkg/ruler/config.go @@ -0,0 +1,20 @@ +package ruler + +import ( + "github.com/SigNoz/signoz/pkg/factory" +) + +type Config struct { +} + +func NewConfigFactory() factory.ConfigFactory { + return factory.NewConfigFactory(factory.MustNewName("ruler"), newConfig) +} + +func newConfig() factory.Config { + return Config{} +} + +func (c Config) Validate() error { + return nil +} diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go new file mode 100644 index 000000000000..61af5319c352 --- /dev/null +++ b/pkg/ruler/ruler.go @@ -0,0 +1,7 @@ +package ruler + +import "github.com/SigNoz/signoz/pkg/statsreporter" + +type Ruler interface { + statsreporter.StatsCollector +} diff --git a/pkg/ruler/rulestore/sqlrulestore/rule.go b/pkg/ruler/rulestore/sqlrulestore/rule.go index 8875a05b3e0f..e05620bf0c0b 100644 --- a/pkg/ruler/rulestore/sqlrulestore/rule.go +++ b/pkg/ruler/rulestore/sqlrulestore/rule.go @@ -6,16 +6,14 @@ import ( "github.com/SigNoz/signoz/pkg/sqlstore" ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes" "github.com/SigNoz/signoz/pkg/valuer" - "github.com/jmoiron/sqlx" ) type rule struct { - *sqlx.DB sqlstore sqlstore.SQLStore } -func NewRuleStore(db *sqlx.DB, store sqlstore.SQLStore) ruletypes.RuleStore { - return &rule{sqlstore: store, DB: db} +func NewRuleStore(store sqlstore.SQLStore) ruletypes.RuleStore { + return &rule{sqlstore: store} } func (r *rule) CreateRule(ctx context.Context, storedRule *ruletypes.Rule, cb func(context.Context, valuer.UUID) error) (valuer.UUID, error) { diff --git a/pkg/ruler/signozruler/provider.go b/pkg/ruler/signozruler/provider.go new file mode 100644 index 000000000000..517c44052a3f --- /dev/null +++ b/pkg/ruler/signozruler/provider.go @@ -0,0 +1,35 @@ +package signozruler + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/ruler" + "github.com/SigNoz/signoz/pkg/ruler/rulestore/sqlrulestore" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/SigNoz/signoz/pkg/types/ruletypes" + "github.com/SigNoz/signoz/pkg/valuer" +) + +type provider struct { + ruleStore ruletypes.RuleStore +} + +func NewFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[ruler.Ruler, ruler.Config] { + return factory.NewProviderFactory(factory.MustNewName("signoz"), func(ctx context.Context, settings factory.ProviderSettings, config ruler.Config) (ruler.Ruler, error) { + return New(ctx, settings, config, sqlstore) + }) +} + +func New(ctx context.Context, settings factory.ProviderSettings, config ruler.Config, sqlstore sqlstore.SQLStore) (ruler.Ruler, error) { + return &provider{ruleStore: sqlrulestore.NewRuleStore(sqlstore)}, nil +} + +func (provider *provider) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) { + rules, err := provider.ruleStore.GetStoredRules(ctx, orgID.String()) + if err != nil { + return nil, err + } + + return ruletypes.NewStatsFromRules(rules), nil +} diff --git a/pkg/signoz/config.go b/pkg/signoz/config.go index 2554fb327977..2b53bf9cd6e4 100644 --- a/pkg/signoz/config.go +++ b/pkg/signoz/config.go @@ -10,6 +10,7 @@ import ( "time" "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/analytics" "github.com/SigNoz/signoz/pkg/apiserver" "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/config" @@ -17,10 +18,12 @@ import ( "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/instrumentation" "github.com/SigNoz/signoz/pkg/prometheus" + "github.com/SigNoz/signoz/pkg/ruler" "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sqlmigration" "github.com/SigNoz/signoz/pkg/sqlmigrator" "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/version" "github.com/SigNoz/signoz/pkg/web" @@ -34,6 +37,9 @@ type Config struct { // Instrumentation config Instrumentation instrumentation.Config `mapstructure:"instrumentation"` + // Analytics config + Analytics analytics.Config `mapstructure:"analytics"` + // Web config Web web.Config `mapstructure:"web"` @@ -61,11 +67,17 @@ type Config struct { // Alertmanager config Alertmanager alertmanager.Config `mapstructure:"alertmanager" yaml:"alertmanager"` + // Ruler config + Ruler ruler.Config `mapstructure:"ruler"` + // Emailing config Emailing emailing.Config `mapstructure:"emailing" yaml:"emailing"` // Sharder config Sharder sharder.Config `mapstructure:"sharder" yaml:"sharder"` + + // StatsReporter config + StatsReporter statsreporter.Config `mapstructure:"statsreporter"` } // DeprecatedFlags are the flags that are deprecated and scheduled for removal. @@ -81,6 +93,7 @@ func NewConfig(ctx context.Context, resolverConfig config.ResolverConfig, deprec configFactories := []factory.ConfigFactory{ version.NewConfigFactory(), instrumentation.NewConfigFactory(), + analytics.NewConfigFactory(), web.NewConfigFactory(), cache.NewConfigFactory(), sqlstore.NewConfigFactory(), @@ -89,8 +102,10 @@ func NewConfig(ctx context.Context, resolverConfig config.ResolverConfig, deprec telemetrystore.NewConfigFactory(), prometheus.NewConfigFactory(), alertmanager.NewConfigFactory(), + ruler.NewConfigFactory(), emailing.NewConfigFactory(), sharder.NewConfigFactory(), + statsreporter.NewConfigFactory(), } conf, err := config.New(ctx, resolverConfig, configFactories) @@ -235,4 +250,14 @@ func mergeAndEnsureBackwardCompatibility(config *Config, deprecatedFlags Depreca fmt.Println("[Deprecated] env SMTP_FROM is deprecated and scheduled for removal. Please use SIGNOZ_EMAILING_FROM instead.") config.Emailing.SMTP.From = os.Getenv("SMTP_FROM") } + + if os.Getenv("SIGNOZ_SAAS_SEGMENT_KEY") != "" { + fmt.Println("[Deprecated] env SIGNOZ_SAAS_SEGMENT_KEY is deprecated and scheduled for removal. Please use SIGNOZ_ANALYTICS_SEGMENT_KEY instead.") + config.Analytics.Segment.Key = os.Getenv("SIGNOZ_SAAS_SEGMENT_KEY") + } + + if os.Getenv("TELEMETRY_ENABLED") != "" { + fmt.Println("[Deprecated] env TELEMETRY_ENABLED is deprecated and scheduled for removal. Please use SIGNOZ_ANALYTICS_ENABLED instead.") + config.Analytics.Enabled = os.Getenv("TELEMETRY_ENABLED") == "true" + } } diff --git a/pkg/signoz/handler_test.go b/pkg/signoz/handler_test.go index 0844ebf2434d..82eb26ae2250 100644 --- a/pkg/signoz/handler_test.go +++ b/pkg/signoz/handler_test.go @@ -33,7 +33,7 @@ func TestNewHandlers(t *testing.T) { require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager) + modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager, nil) handlers := NewHandlers(modules) diff --git a/pkg/signoz/module.go b/pkg/signoz/module.go index 25b10f531297..0a1fa6487099 100644 --- a/pkg/signoz/module.go +++ b/pkg/signoz/module.go @@ -2,6 +2,7 @@ package signoz import ( "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/analytics" "github.com/SigNoz/signoz/pkg/emailing" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/modules/apdex" @@ -44,17 +45,18 @@ func NewModules( providerSettings factory.ProviderSettings, orgGetter organization.Getter, alertmanager alertmanager.Alertmanager, + analytics analytics.Analytics, ) Modules { quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore)) orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter) - user := impluser.NewModule(impluser.NewStore(sqlstore, providerSettings), jwt, emailing, providerSettings, orgSetter) + user := impluser.NewModule(impluser.NewStore(sqlstore, providerSettings), jwt, emailing, providerSettings, orgSetter, analytics) return Modules{ OrgGetter: orgGetter, OrgSetter: orgSetter, Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()), SavedView: implsavedview.NewModule(sqlstore), Apdex: implapdex.NewModule(sqlstore), - Dashboard: impldashboard.NewModule(sqlstore, providerSettings), + Dashboard: impldashboard.NewModule(sqlstore, providerSettings, analytics), User: user, QuickFilter: quickfilter, TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)), diff --git a/pkg/signoz/module_test.go b/pkg/signoz/module_test.go index f18596b05963..2445d1d67f5e 100644 --- a/pkg/signoz/module_test.go +++ b/pkg/signoz/module_test.go @@ -33,7 +33,7 @@ func TestNewModules(t *testing.T) { require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager) + modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager, nil) reflectVal := reflect.ValueOf(modules) for i := 0; i < reflectVal.NumField(); i++ { diff --git a/pkg/signoz/provider.go b/pkg/signoz/provider.go index aadc8166d2d3..91425dbff830 100644 --- a/pkg/signoz/provider.go +++ b/pkg/signoz/provider.go @@ -4,6 +4,9 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/legacyalertmanager" "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/analytics" + "github.com/SigNoz/signoz/pkg/analytics/noopanalytics" + "github.com/SigNoz/signoz/pkg/analytics/segmentanalytics" "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/cache/memorycache" "github.com/SigNoz/signoz/pkg/cache/rediscache" @@ -14,6 +17,8 @@ import ( "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus" + "github.com/SigNoz/signoz/pkg/ruler" + "github.com/SigNoz/signoz/pkg/ruler/signozruler" "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/sharder/singlesharder" @@ -21,14 +26,25 @@ import ( "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlstorehook" + "github.com/SigNoz/signoz/pkg/statsreporter" + "github.com/SigNoz/signoz/pkg/statsreporter/analyticsstatsreporter" + "github.com/SigNoz/signoz/pkg/statsreporter/noopstatsreporter" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/telemetrystore/clickhousetelemetrystore" "github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystorehook" + "github.com/SigNoz/signoz/pkg/version" "github.com/SigNoz/signoz/pkg/web" "github.com/SigNoz/signoz/pkg/web/noopweb" "github.com/SigNoz/signoz/pkg/web/routerweb" ) +func NewAnalyticsProviderFactories() factory.NamedMap[factory.ProviderFactory[analytics.Analytics, analytics.Config]] { + return factory.MustNewNamedMap( + noopanalytics.NewFactory(), + segmentanalytics.NewFactory(), + ) +} + func NewCacheProviderFactories() factory.NamedMap[factory.ProviderFactory[cache.Cache, cache.Config]] { return factory.MustNewNamedMap( memorycache.NewFactory(), @@ -114,6 +130,12 @@ func NewAlertmanagerProviderFactories(sqlstore sqlstore.SQLStore, orgGetter orga ) } +func NewRulerProviderFactories(sqlstore sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[ruler.Ruler, ruler.Config]] { + return factory.MustNewNamedMap( + signozruler.NewFactory(sqlstore), + ) +} + func NewEmailingProviderFactories() factory.NamedMap[factory.ProviderFactory[emailing.Emailing, emailing.Config]] { return factory.MustNewNamedMap( noopemailing.NewFactory(), @@ -127,3 +149,10 @@ func NewSharderProviderFactories() factory.NamedMap[factory.ProviderFactory[shar noopsharder.NewFactory(), ) } + +func NewStatsReporterProviderFactories(telemetryStore telemetrystore.TelemetryStore, collectors []statsreporter.StatsCollector, orgGetter organization.Getter, build version.Build, analyticsConfig analytics.Config) factory.NamedMap[factory.ProviderFactory[statsreporter.StatsReporter, statsreporter.Config]] { + return factory.MustNewNamedMap( + analyticsstatsreporter.NewFactory(telemetryStore, collectors, orgGetter, build, analyticsConfig), + noopstatsreporter.NewFactory(), + ) +} diff --git a/pkg/signoz/provider_test.go b/pkg/signoz/provider_test.go index 752ff19db8e4..d4c02c1ffcb7 100644 --- a/pkg/signoz/provider_test.go +++ b/pkg/signoz/provider_test.go @@ -4,11 +4,14 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/SigNoz/signoz/pkg/analytics" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest" + "github.com/SigNoz/signoz/pkg/version" "github.com/stretchr/testify/assert" ) @@ -45,6 +48,10 @@ func TestNewProviderFactories(t *testing.T) { NewAlertmanagerProviderFactories(sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual), orgGetter) }) + assert.NotPanics(t, func() { + NewRulerProviderFactories(sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual)) + }) + assert.NotPanics(t, func() { NewEmailingProviderFactories() }) @@ -52,4 +59,10 @@ func TestNewProviderFactories(t *testing.T) { assert.NotPanics(t, func() { NewSharderProviderFactories() }) + + assert.NotPanics(t, func() { + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual)), nil) + telemetryStore := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherEqual) + NewStatsReporterProviderFactories(telemetryStore, []statsreporter.StatsCollector{}, orgGetter, version.Build{}, analytics.Config{Enabled: true}) + }) } diff --git a/pkg/signoz/signoz.go b/pkg/signoz/signoz.go index 919520b5b32f..4c675f2a8cb4 100644 --- a/pkg/signoz/signoz.go +++ b/pkg/signoz/signoz.go @@ -12,10 +12,12 @@ import ( "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" "github.com/SigNoz/signoz/pkg/prometheus" + "github.com/SigNoz/signoz/pkg/ruler" "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sqlmigration" "github.com/SigNoz/signoz/pkg/sqlmigrator" "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/SigNoz/signoz/pkg/statsreporter" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/version" @@ -33,10 +35,12 @@ type SigNoz struct { TelemetryStore telemetrystore.TelemetryStore Prometheus prometheus.Prometheus Alertmanager alertmanager.Alertmanager + Rules ruler.Ruler Zeus zeus.Zeus Licensing licensing.Licensing Emailing emailing.Emailing Sharder sharder.Sharder + StatsReporter statsreporter.StatsReporter Modules Modules Handlers Handlers } @@ -48,7 +52,7 @@ func New( zeusConfig zeus.Config, zeusProviderFactory factory.ProviderFactory[zeus.Zeus, zeus.Config], licenseConfig licensing.Config, - licenseProviderFactoryCb func(sqlstore.SQLStore, zeus.Zeus, organization.Getter) factory.ProviderFactory[licensing.Licensing, licensing.Config], + licenseProviderFactory func(sqlstore.SQLStore, zeus.Zeus, organization.Getter) factory.ProviderFactory[licensing.Licensing, licensing.Config], emailingProviderFactories factory.NamedMap[factory.ProviderFactory[emailing.Emailing, emailing.Config]], cacheProviderFactories factory.NamedMap[factory.ProviderFactory[cache.Cache, cache.Config]], webProviderFactories factory.NamedMap[factory.ProviderFactory[web.Web, web.Config]], @@ -67,6 +71,18 @@ func New( // Get the provider settings from instrumentation providerSettings := instrumentation.ToProviderSettings() + // Initialize analytics just after instrumentation, as providers might require it + analytics, err := factory.NewProviderFromNamedMap( + ctx, + providerSettings, + config.Analytics, + NewAnalyticsProviderFactories(), + config.Analytics.Provider(), + ) + if err != nil { + return nil, err + } + // Initialize zeus from the available zeus provider factory. This is not config controlled // and depends on the variant of the build. zeus, err := zeusProviderFactory.New( @@ -193,7 +209,19 @@ func New( return nil, err } - licensingProviderFactory := licenseProviderFactoryCb(sqlstore, zeus, orgGetter) + // Initialize ruler from the available ruler provider factories + ruler, err := factory.NewProviderFromNamedMap( + ctx, + providerSettings, + config.Ruler, + NewRulerProviderFactories(sqlstore), + "signoz", + ) + if err != nil { + return nil, err + } + + licensingProviderFactory := licenseProviderFactory(sqlstore, zeus, orgGetter) licensing, err := licensingProviderFactory.New( ctx, providerSettings, @@ -204,16 +232,40 @@ func New( } // Initialize all modules - modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager) + modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics) // Initialize all handlers for the modules handlers := NewHandlers(modules) + // Create a list of all stats collectors + statsCollectors := []statsreporter.StatsCollector{ + alertmanager, + ruler, + modules.Dashboard, + modules.SavedView, + modules.User, + licensing, + } + + // Initialize stats reporter from the available stats reporter provider factories + statsReporter, err := factory.NewProviderFromNamedMap( + ctx, + providerSettings, + config.StatsReporter, + NewStatsReporterProviderFactories(telemetrystore, statsCollectors, orgGetter, version.Info, config.Analytics), + config.StatsReporter.Provider(), + ) + if err != nil { + return nil, err + } + registry, err := factory.NewRegistry( instrumentation.Logger(), factory.NewNamedService(factory.MustNewName("instrumentation"), instrumentation), + factory.NewNamedService(factory.MustNewName("analytics"), analytics), factory.NewNamedService(factory.MustNewName("alertmanager"), alertmanager), factory.NewNamedService(factory.MustNewName("licensing"), licensing), + factory.NewNamedService(factory.MustNewName("statsreporter"), statsReporter), ) if err != nil { return nil, err diff --git a/pkg/statsreporter/analyticsstatsreporter/provider.go b/pkg/statsreporter/analyticsstatsreporter/provider.go new file mode 100644 index 000000000000..f94c83d34421 --- /dev/null +++ b/pkg/statsreporter/analyticsstatsreporter/provider.go @@ -0,0 +1,216 @@ +package analyticsstatsreporter + +import ( + "context" + "sync" + "time" + + "github.com/SigNoz/signoz/pkg/analytics" + "github.com/SigNoz/signoz/pkg/analytics/segmentanalytics" + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/organization" + "github.com/SigNoz/signoz/pkg/statsreporter" + "github.com/SigNoz/signoz/pkg/telemetrystore" + "github.com/SigNoz/signoz/pkg/types/analyticstypes" + "github.com/SigNoz/signoz/pkg/valuer" + "github.com/SigNoz/signoz/pkg/version" +) + +type provider struct { + // settings + settings factory.ScopedProviderSettings + + // config + config statsreporter.Config + + // used to get telemetry details. srikanthcvv to move this to the querier layer + telemetryStore telemetrystore.TelemetryStore + + // a list of collectors, used to collect stats from across the codebase + collectors []statsreporter.StatsCollector + + // used to get organizations + orgGetter organization.Getter + + // used to send stats to an analytics backend + analytics analytics.Analytics + + // used to get build information + build version.Build + + // used to get deployment information + deployment version.Deployment + + // used to stop the provider + stopC chan struct{} +} + +func NewFactory(telemetryStore telemetrystore.TelemetryStore, collectors []statsreporter.StatsCollector, orgGetter organization.Getter, build version.Build, analyticsConfig analytics.Config) factory.ProviderFactory[statsreporter.StatsReporter, statsreporter.Config] { + return factory.NewProviderFactory(factory.MustNewName("analytics"), func(ctx context.Context, settings factory.ProviderSettings, config statsreporter.Config) (statsreporter.StatsReporter, error) { + return New(ctx, settings, config, telemetryStore, collectors, orgGetter, build, analyticsConfig) + }) +} + +func New( + ctx context.Context, + providerSettings factory.ProviderSettings, + config statsreporter.Config, + telemetryStore telemetrystore.TelemetryStore, + collectors []statsreporter.StatsCollector, + orgGetter organization.Getter, + build version.Build, + analyticsConfig analytics.Config, +) (statsreporter.StatsReporter, error) { + settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/statsreporter/analyticsstatsreporter") + deployment := version.NewDeployment() + analytics, err := segmentanalytics.New(ctx, providerSettings, analyticsConfig) + if err != nil { + return nil, err + } + + return &provider{ + settings: settings, + config: config, + telemetryStore: telemetryStore, + collectors: collectors, + orgGetter: orgGetter, + analytics: analytics, + build: build, + deployment: deployment, + stopC: make(chan struct{}), + }, nil +} + +func (provider *provider) Start(ctx context.Context) error { + go func() { + if err := provider.analytics.Start(ctx); err != nil { + provider.settings.Logger().ErrorContext(ctx, "failed to start analytics", "error", err) + } + }() + + ticker := time.NewTicker(provider.config.Interval) + defer ticker.Stop() + + for { + select { + case <-provider.stopC: + return nil + case <-ticker.C: + err := provider.Report(ctx) + if err != nil { + provider.settings.Logger().ErrorContext(ctx, "failed to report stats", "error", err) + } + } + } +} + +func (provider *provider) Report(ctx context.Context) error { + orgs, err := provider.orgGetter.ListByOwnedKeyRange(ctx) + if err != nil { + return err + } + + for _, org := range orgs { + stats := provider.collectOrg(ctx, org.ID) + if len(stats) == 0 { + provider.settings.Logger().WarnContext(ctx, "no stats collected", "org_id", org.ID) + continue + } + + stats["build.version"] = provider.build.Version() + stats["build.branch"] = provider.build.Branch() + stats["build.hash"] = provider.build.Hash() + stats["build.variant"] = provider.build.Variant() + stats["deployment.mode"] = provider.deployment.Mode() + stats["deployment.platform"] = provider.deployment.Platform() + stats["deployment.os"] = provider.deployment.OS() + stats["deployment.arch"] = provider.deployment.Arch() + + provider.settings.Logger().DebugContext(ctx, "reporting stats", "stats", stats) + provider.analytics.Send( + ctx, + analyticstypes.Track{ + UserId: org.ID.String(), + Event: "Stats Reported", + Properties: analyticstypes.NewPropertiesFromMap(stats), + Context: &analyticstypes.Context{ + Extra: map[string]interface{}{ + analyticstypes.KeyGroupID: org.ID.String(), + }, + }, + }, + analyticstypes.Group{ + UserId: org.ID.String(), + GroupId: org.ID.String(), + Traits: analyticstypes. + NewTraitsFromMap(stats). + SetName(org.DisplayName). + SetUsername(org.Name). + SetCreatedAt(org.CreatedAt), + }, + analyticstypes.Identify{ + UserId: org.ID.String(), + Traits: analyticstypes. + NewTraits(). + SetName(org.DisplayName). + SetUsername(org.Name). + SetCreatedAt(org.CreatedAt), + }, + ) + } + + return nil +} + +func (provider *provider) Stop(ctx context.Context) error { + close(provider.stopC) + if err := provider.analytics.Stop(ctx); err != nil { + provider.settings.Logger().ErrorContext(ctx, "failed to stop analytics", "error", err) + } + + return nil +} + +func (provider *provider) collectOrg(ctx context.Context, orgID valuer.UUID) map[string]any { + var wg sync.WaitGroup + wg.Add(len(provider.collectors)) + + stats := make(map[string]any, 0) + mtx := sync.Mutex{} + + for _, collector := range provider.collectors { + go func(collector statsreporter.StatsCollector) { + defer wg.Done() + + collectorStats, err := collector.Collect(ctx, orgID) + if err != nil { + provider.settings.Logger().ErrorContext(ctx, "failed to collect stats", "error", err) + return + } + + mtx.Lock() + for k, v := range collectorStats { + stats[k] = v + } + mtx.Unlock() + }(collector) + } + wg.Wait() + + var traces uint64 + if err := provider.telemetryStore.ClickhouseDB().QueryRow(ctx, "SELECT COUNT(*) FROM signoz_traces.distributed_signoz_index_v3").Scan(&traces); err == nil { + stats["telemetry.traces.count"] = traces + } + + var logs uint64 + if err := provider.telemetryStore.ClickhouseDB().QueryRow(ctx, "SELECT COUNT(*) FROM signoz_logs.distributed_logs_v2").Scan(&logs); err == nil { + stats["telemetry.logs.count"] = logs + } + + var metrics uint64 + if err := provider.telemetryStore.ClickhouseDB().QueryRow(ctx, "SELECT COUNT(*) FROM signoz_metrics.distributed_samples_v4").Scan(&metrics); err == nil { + stats["telemetry.metrics.count"] = metrics + } + + return stats +} diff --git a/pkg/statsreporter/config.go b/pkg/statsreporter/config.go new file mode 100644 index 000000000000..674a501b7ae5 --- /dev/null +++ b/pkg/statsreporter/config.go @@ -0,0 +1,38 @@ +package statsreporter + +import ( + "time" + + "github.com/SigNoz/signoz/pkg/factory" +) + +type Config struct { + // Enabled is a flag to enable or disable the stats reporter. + Enabled bool `mapstructure:"enabled"` + + // Interval is the interval at which the stats are collected. + Interval time.Duration `mapstructure:"interval"` +} + +func NewConfigFactory() factory.ConfigFactory { + return factory.NewConfigFactory(factory.MustNewName("statsreporter"), newConfig) +} + +func newConfig() factory.Config { + return Config{ + Enabled: true, + Interval: 6 * time.Hour, + } +} + +func (c Config) Validate() error { + return nil +} + +func (c Config) Provider() string { + if c.Enabled { + return "analytics" + } + + return "noop" +} diff --git a/pkg/statsreporter/noopstatsreporter/provider.go b/pkg/statsreporter/noopstatsreporter/provider.go new file mode 100644 index 000000000000..9a17c89a9540 --- /dev/null +++ b/pkg/statsreporter/noopstatsreporter/provider.go @@ -0,0 +1,36 @@ +package noopstatsreporter + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/statsreporter" +) + +type provider struct { + stopC chan struct{} +} + +func NewFactory() factory.ProviderFactory[statsreporter.StatsReporter, statsreporter.Config] { + return factory.NewProviderFactory(factory.MustNewName("noop"), New) +} + +func New(ctx context.Context, providerSettings factory.ProviderSettings, config statsreporter.Config) (statsreporter.StatsReporter, error) { + return &provider{ + stopC: make(chan struct{}), + }, nil +} + +func (provider *provider) Start(ctx context.Context) error { + <-provider.stopC + return nil +} + +func (provider *provider) Report(ctx context.Context) error { + return nil +} + +func (provider *provider) Stop(ctx context.Context) error { + close(provider.stopC) + return nil +} diff --git a/pkg/statsreporter/statsreporter.go b/pkg/statsreporter/statsreporter.go new file mode 100644 index 000000000000..d32762366dd3 --- /dev/null +++ b/pkg/statsreporter/statsreporter.go @@ -0,0 +1,18 @@ +package statsreporter + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/valuer" +) + +type StatsReporter interface { + factory.Service + + Report(context.Context) error +} + +type StatsCollector interface { + Collect(context.Context, valuer.UUID) (map[string]any, error) +} diff --git a/pkg/types/alertmanagertypes/channel.go b/pkg/types/alertmanagertypes/channel.go index 23ad962d38f2..1cb873ecb15d 100644 --- a/pkg/types/alertmanagertypes/channel.go +++ b/pkg/types/alertmanagertypes/channel.go @@ -146,6 +146,22 @@ func GetChannelByName(channels Channels, name string) (int, *Channel, error) { return 0, nil, errors.Newf(errors.TypeNotFound, ErrCodeAlertmanagerChannelNotFound, "cannot find channel with name %s", name) } +func NewStatsFromChannels(channels Channels) map[string]any { + stats := make(map[string]any) + for _, channel := range channels { + key := "alertmanager.channel.type." + channel.Type + + if _, ok := stats[key]; !ok { + stats[key] = int64(1) + } else { + stats[key] = stats[key].(int64) + 1 + } + } + + stats["alertmanager.channel.count"] = int64(len(channels)) + return stats +} + func (c *Channel) Update(receiver Receiver) error { channel := NewChannelFromReceiver(receiver, c.OrgID) if channel == nil { diff --git a/pkg/types/analyticstypes/message.go b/pkg/types/analyticstypes/message.go index 5f2c4436e333..7fc3edaa9a90 100644 --- a/pkg/types/analyticstypes/message.go +++ b/pkg/types/analyticstypes/message.go @@ -1,9 +1,15 @@ package analyticstypes import ( + "strings" + segment "github.com/segmentio/analytics-go/v3" ) +const ( + KeyGroupID string = "groupId" +) + type Message = segment.Message type Group = segment.Group type Identify = segment.Identify @@ -19,3 +25,21 @@ func NewTraits() Traits { func NewProperties() Properties { return segment.NewProperties() } + +func NewPropertiesFromMap(m map[string]any) Properties { + properties := NewProperties() + for k, v := range m { + properties.Set(strings.ReplaceAll(k, ".", "_"), v) + } + + return properties +} + +func NewTraitsFromMap(m map[string]any) Traits { + traits := NewTraits() + for k, v := range m { + traits.Set(strings.ReplaceAll(k, ".", "_"), v) + } + + return traits +} diff --git a/pkg/types/dashboardtypes/dashboard.go b/pkg/types/dashboardtypes/dashboard.go index 7cdd47bc4220..ff030cd0f791 100644 --- a/pkg/types/dashboardtypes/dashboard.go +++ b/pkg/types/dashboardtypes/dashboard.go @@ -145,6 +145,68 @@ func NewGettableDashboardFromDashboard(dashboard *Dashboard) (*GettableDashboard }, nil } +func NewStatsFromStorableDashboards(dashboards []*StorableDashboard) map[string]any { + stats := make(map[string]any) + stats["dashboard.panels.count"] = int64(0) + stats["dashboard.panels.traces.count"] = int64(0) + stats["dashboard.panels.metrics.count"] = int64(0) + stats["dashboard.panels.logs.count"] = int64(0) + for _, dashboard := range dashboards { + addStatsFromStorableDashboard(dashboard, stats) + } + + stats["dashboard.count"] = int64(len(dashboards)) + return stats +} + +func addStatsFromStorableDashboard(dashboard *StorableDashboard, stats map[string]any) { + if dashboard.Data == nil { + return + } + + if dashboard.Data["widgets"] == nil { + return + } + + widgets, ok := dashboard.Data["widgets"] + if !ok { + return + } + + data, ok := widgets.([]interface{}) + if !ok { + return + } + + for _, widget := range data { + sData, ok := widget.(map[string]interface{}) + if ok && sData["query"] != nil { + stats["dashboard.panels.count"] = stats["dashboard.panels.count"].(int64) + 1 + query, ok := sData["query"].(map[string]interface{}) + if ok && query["queryType"] == "builder" && query["builder"] != nil { + builderData, ok := query["builder"].(map[string]interface{}) + if ok && builderData["queryData"] != nil { + builderQueryData, ok := builderData["queryData"].([]interface{}) + if ok { + for _, queryData := range builderQueryData { + data, ok := queryData.(map[string]interface{}) + if ok { + if data["dataSource"] == "traces" { + stats["dashboard.panels.traces.count"] = stats["dashboard.panels.traces.count"].(int64) + 1 + } else if data["dataSource"] == "metrics" { + stats["dashboard.panels.metrics.count"] = stats["dashboard.panels.metrics.count"].(int64) + 1 + } else if data["dataSource"] == "logs" { + stats["dashboard.panels.logs.count"] = stats["dashboard.panels.logs.count"].(int64) + 1 + } + } + } + } + } + } + } + } +} + func (storableDashboardData *StorableDashboardData) GetWidgetIds() []string { data := *storableDashboardData widgetIds := []string{} diff --git a/pkg/types/licensetypes/license.go b/pkg/types/licensetypes/license.go index 55db17a88cb6..74028447a5db 100644 --- a/pkg/types/licensetypes/license.go +++ b/pkg/types/licensetypes/license.go @@ -323,6 +323,13 @@ func NewLicenseFromStorableLicense(storableLicense *StorableLicense) (*License, } +func NewStatsFromLicense(license *License) map[string]any { + return map[string]any{ + "license.plan": license.PlanName.StringValue(), + "license.id": license.ID.StringValue(), + } +} + func (license *License) UpdateFeatures(features []*Feature) { license.Features = features } diff --git a/pkg/types/ruletypes/rule.go b/pkg/types/ruletypes/rule.go index beb6fff3fe43..255825cc1b72 100644 --- a/pkg/types/ruletypes/rule.go +++ b/pkg/types/ruletypes/rule.go @@ -2,6 +2,8 @@ package ruletypes import ( "context" + "encoding/json" + "strings" "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/valuer" @@ -18,6 +20,33 @@ type Rule struct { OrgID string `bun:"org_id,type:text"` } +func NewStatsFromRules(rules []*Rule) map[string]any { + stats := make(map[string]any) + for _, rule := range rules { + gettableRule := &GettableRule{} + if err := json.Unmarshal([]byte(rule.Data), gettableRule); err != nil { + continue + } + + key := "rule.type." + strings.TrimSuffix(strings.ToLower(string(gettableRule.RuleType)), "_rule") + ".count" + if _, ok := stats[key]; !ok { + stats[key] = int64(1) + } else { + stats[key] = stats[key].(int64) + 1 + } + + key = "alert.type." + strings.TrimSuffix(strings.ToLower(string(gettableRule.AlertType)), "_based_alert") + ".count" + if _, ok := stats[key]; !ok { + stats[key] = int64(1) + } else { + stats[key] = stats[key].(int64) + 1 + } + } + + stats["rule.count"] = int64(len(rules)) + return stats +} + type RuleStore interface { CreateRule(context.Context, *Rule, func(context.Context, valuer.UUID) error) (valuer.UUID, error) EditRule(context.Context, *Rule, func(context.Context) error) error diff --git a/pkg/types/savedview.go b/pkg/types/savedview.go index 821fe36fb0c2..37e5808eb0f5 100644 --- a/pkg/types/savedview.go +++ b/pkg/types/savedview.go @@ -1,6 +1,8 @@ package types import ( + "strings" + "github.com/uptrace/bun" ) @@ -18,3 +20,18 @@ type SavedView struct { Data string `json:"data" bun:"data,type:text,notnull"` ExtraData string `json:"extraData" bun:"extra_data,type:text"` } + +func NewStatsFromSavedViews(savedViews []*SavedView) map[string]any { + stats := make(map[string]any) + for _, savedView := range savedViews { + key := "savedview.source." + strings.ToLower(string(savedView.SourcePage)) + ".count" + if _, ok := stats[key]; !ok { + stats[key] = int64(1) + } else { + stats[key] = stats[key].(int64) + 1 + } + } + + stats["savedview.count"] = int64(len(savedViews)) + return stats +} diff --git a/pkg/types/user.go b/pkg/types/user.go index 25428edde2fb..cbc3301e3df6 100644 --- a/pkg/types/user.go +++ b/pkg/types/user.go @@ -72,6 +72,8 @@ type UserStore interface { ListAPIKeys(ctx context.Context, orgID valuer.UUID) ([]*StorableAPIKeyUser, error) RevokeAPIKey(ctx context.Context, id valuer.UUID, revokedByUserID valuer.UUID) error GetAPIKey(ctx context.Context, orgID, id valuer.UUID) (*StorableAPIKeyUser, error) + + CountByOrgID(ctx context.Context, orgID valuer.UUID) (int64, error) } type GettableUser struct { diff --git a/pkg/version/deployment.go b/pkg/version/deployment.go new file mode 100644 index 000000000000..4b19126c09f1 --- /dev/null +++ b/pkg/version/deployment.go @@ -0,0 +1,155 @@ +package version + +import ( + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + gotime "time" +) + +var ( + current = Deployment{mode: "unknown", platform: "unknown", os: "unknown", arch: "unknown"} + once = sync.Once{} +) + +type Deployment struct { + // mode of deployment, e.g. kubernetes, binary, etc. + mode string + + // platform of deployment, e.g. heroku, render, aws, gcp, azure, digitalocean, etc. + platform string + + // os of deployment, e.g. linux, darwin, etc. + os string + + // arch of deployment, e.g. amd64, arm64, etc. + arch string +} + +func NewDeployment() Deployment { + once.Do(func() { + current.mode = detectMode() + current.platform = detectPlatform() + current.os = runtime.GOOS + current.arch = runtime.GOARCH + }) + + return current +} + +func (d Deployment) Mode() string { + return d.mode +} + +func (d Deployment) Platform() string { + return d.platform +} + +func (d Deployment) OS() string { + return d.os +} + +func (d Deployment) Arch() string { + return d.arch +} + +func detectMode() string { + // Check if running in Kubernetes + if os.Getenv("KUBERNETES_SERVICE_HOST") != "" { + return "kubernetes" + } + + // Check if running in a container and identify the runtime + if data, err := os.ReadFile("/proc/self/cgroup"); err == nil { + cgroupData := string(data) + switch { + case strings.Contains(cgroupData, "docker"): + return "docker" + case strings.Contains(cgroupData, "containerd"): + return "containerd" + case strings.Contains(cgroupData, "libpod") || strings.Contains(cgroupData, "podman"): + return "podman" + case strings.Contains(cgroupData, "crio"): + return "cri-o" + } + } + + // Check if running as a binary + if exe, err := os.Executable(); err == nil { + // Check if the executable is in a standard binary location + exePath := filepath.Clean(exe) + if strings.HasPrefix(exePath, "/usr/local/bin/") || + strings.HasPrefix(exePath, "/usr/bin/") || + strings.HasPrefix(exePath, "/bin/") || + strings.HasPrefix(exePath, "/opt/") { + return "binary" + } + + // Check if the executable is in the current directory + if filepath.Dir(exePath) == "." || filepath.Dir(exePath) == filepath.Clean(os.Getenv("PWD")) { + return "binary" + } + } + + return "unknown" +} + +func detectPlatform() string { + // Check for PaaS platforms first as they use environment variables + switch { + case os.Getenv("DYNO") != "" || os.Getenv("HEROKU_APP_ID") != "": + return "heroku" + case os.Getenv("RENDER") != "" || os.Getenv("RENDER_SERVICE_ID") != "": + return "render" + } + + // Try to detect cloud provider through metadata endpoints + client := &http.Client{Timeout: 1 * gotime.Second} + + // AWS metadata + if req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/latest/meta-data/", nil); err == nil { + if resp, err := client.Do(req); err == nil { + resp.Body.Close() + if resp.StatusCode == 200 { + return "aws" + } + } + } + + // GCP metadata + if req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/computeMetadata/v1/", nil); err == nil { + req.Header.Add("Metadata-Flavor", "Google") + if resp, err := client.Do(req); err == nil { + resp.Body.Close() + if resp.StatusCode == 200 { + return "gcp" + } + } + } + + // Azure metadata + if req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/metadata/instance", nil); err == nil { + req.Header.Add("Metadata", "true") + if resp, err := client.Do(req); err == nil { + resp.Body.Close() + if resp.StatusCode == 200 { + return "azure" + } + } + } + + // Digitalocean metadata + if req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/metadata/v1/", nil); err == nil { + if resp, err := client.Do(req); err == nil { + resp.Body.Close() + if resp.StatusCode == 200 { + return "digitalocean" + } + } + } + + return "unknown" +}