271 lines
6.0 KiB
Go
Raw Normal View History

feat: aws integration: UI facing QS api for cloud account management (#6771) * feat: init app/cloud_integrations * feat: get API test started for cloudintegrations account lifecycle * feat: cloudintegrations: get controller started * feat: cloud integrations: add cloudintegrations.Controller to APIHandler and servers * feat: cloud integrations: get routes started * feat: cloud integrations: get accounts table schema started * feat: cloud integrations: get cloudProviderAccountsSQLRepository started * feat: cloud integrations: cloudProviderAccountsSQLRepository.listAccounts * feat: cloud integrations: http handler and controller plumbing for /generate-connection-url * feat: cloud integrations: cloudProviderAccountsSQLRepository.upsert * feat: cloud integrations: finish up with /generate-connection-url * feat: cloud integrations: add cloudProviderAccountsRepository.get * feat: cloud integrations: add API test expectation for being able to get account status * feat: cloud integrations: add http handler and controller method for getting account status * feat: cloud integrations: ensure unconnected accounts aren't included in list of connected accounts * feat: cloud integrations: add test expectation for agent check in request * feat: cloud integrations: agent check in API * feat: cloud integrations: ensure polling for status after agent check in works * feat: cloud integrations: ensure account included in connected account list after agent check in * feat: cloud integrations: add API expectation for updating account config * feat: cloud integrations: API for updating cloud account config * feat: cloud integrations: expectation for agent receiving latest config after account config update * feat: cloud integrations: expectation for disconnecting cloud accounts from UI * feat: cloud integrations: API for disconnecting cloud accounts * feat: cloud integrations: some cleanup * feat: cloud integrations: some more cleanup * feat: cloud integrations: repo: scope rows by cloud provider * feat: testutils: refactor out helper for creating a test sqlite DB * feat: cloud integrations: controller: add test validating regeneration of connection url * feat: cloud integrations: controller: validations for agent check ins * feat: cloud integrations: connected account response structure * feat: cloud integrations: API response account structure * feat: cloud integrations: some more cleanup * feat: cloud integrations: remove cloudProviderAccountsRepository.GetById * feat: cloud integrations: shouldn't be able to disconnect non-existent account * feat: cloud integrations: validate agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: ensure agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: remove stray import of ee/model in cloudintegrations controller
2025-01-10 18:43:35 +05:30
package cloudintegrations
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"go.signoz.io/signoz/pkg/query-service/model"
)
type cloudProviderAccountsRepository interface {
listConnected(ctx context.Context, cloudProvider string) ([]AccountRecord, *model.ApiError)
get(ctx context.Context, cloudProvider string, id string) (*AccountRecord, *model.ApiError)
getConnectedCloudAccount(
ctx context.Context, cloudProvider string, cloudAccountId string,
) (*AccountRecord, *model.ApiError)
// Insert an account or update it by (cloudProvider, id)
// for specified non-empty fields
upsert(
ctx context.Context,
cloudProvider string,
id *string,
config *AccountConfig,
cloudAccountId *string,
agentReport *AgentReport,
removedAt *time.Time,
) (*AccountRecord, *model.ApiError)
}
func newCloudProviderAccountsRepository(db *sqlx.DB) (
*cloudProviderAccountsSQLRepository, error,
) {
feat: aws integration UI facing api: services (#6803) * feat: cloud service integrations: get model and repo interface started * feat: cloud service integrations: flesh out more of cloud services model * feat: cloud integrations: reorganize things a little * feat: cloud integrations: get svc controller started * feat: cloud integrations: add stubs for EC2 and RDS postgres services * feat: cloud integrations: add validation for listing and getting available svcs and some cleanup * feat: cloud integrations: refactor helpers in existing integrations code for reuse * feat: cloud integrations: parsing of cloud service definitions * feat: cloud integrations: impl for getCloudProviderService * feat: cloud integrations: some reorganization * feat: cloud integrations: some more cleanup * feat: cloud integrations: add validation for listing available cloud provider services * feat: cloud integrations: API endpoint for listing available cloud provider services * feat: cloud integrations: add validation for getting details of a particular service * feat: cloud integrations: API endpoint for getting details of a service * feat: cloud integrations: add controller validation for configuring cloud services * feat: cloud integrations: get serviceConfigRepo started * feat: cloud integrations: service config in service list summaries when queried for cloud account id * feat: cloud integrations: only a supported service for a connected cloud account can be configured * feat: cloud integrations: add validation for configuring services via the API * feat: cloud integrations: API for configuring services * feat: cloud integrations: some cleanup * feat: cloud integrations: fix broken test --------- Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
2025-01-16 17:36:09 +05:30
if err := initAccountsSqliteDBIfNeeded(db); err != nil {
return nil, fmt.Errorf("could not init sqlite DB for cloudintegrations accounts: %w", err)
feat: aws integration: UI facing QS api for cloud account management (#6771) * feat: init app/cloud_integrations * feat: get API test started for cloudintegrations account lifecycle * feat: cloudintegrations: get controller started * feat: cloud integrations: add cloudintegrations.Controller to APIHandler and servers * feat: cloud integrations: get routes started * feat: cloud integrations: get accounts table schema started * feat: cloud integrations: get cloudProviderAccountsSQLRepository started * feat: cloud integrations: cloudProviderAccountsSQLRepository.listAccounts * feat: cloud integrations: http handler and controller plumbing for /generate-connection-url * feat: cloud integrations: cloudProviderAccountsSQLRepository.upsert * feat: cloud integrations: finish up with /generate-connection-url * feat: cloud integrations: add cloudProviderAccountsRepository.get * feat: cloud integrations: add API test expectation for being able to get account status * feat: cloud integrations: add http handler and controller method for getting account status * feat: cloud integrations: ensure unconnected accounts aren't included in list of connected accounts * feat: cloud integrations: add test expectation for agent check in request * feat: cloud integrations: agent check in API * feat: cloud integrations: ensure polling for status after agent check in works * feat: cloud integrations: ensure account included in connected account list after agent check in * feat: cloud integrations: add API expectation for updating account config * feat: cloud integrations: API for updating cloud account config * feat: cloud integrations: expectation for agent receiving latest config after account config update * feat: cloud integrations: expectation for disconnecting cloud accounts from UI * feat: cloud integrations: API for disconnecting cloud accounts * feat: cloud integrations: some cleanup * feat: cloud integrations: some more cleanup * feat: cloud integrations: repo: scope rows by cloud provider * feat: testutils: refactor out helper for creating a test sqlite DB * feat: cloud integrations: controller: add test validating regeneration of connection url * feat: cloud integrations: controller: validations for agent check ins * feat: cloud integrations: connected account response structure * feat: cloud integrations: API response account structure * feat: cloud integrations: some more cleanup * feat: cloud integrations: remove cloudProviderAccountsRepository.GetById * feat: cloud integrations: shouldn't be able to disconnect non-existent account * feat: cloud integrations: validate agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: ensure agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: remove stray import of ee/model in cloudintegrations controller
2025-01-10 18:43:35 +05:30
}
return &cloudProviderAccountsSQLRepository{
db: db,
}, nil
}
feat: aws integration UI facing api: services (#6803) * feat: cloud service integrations: get model and repo interface started * feat: cloud service integrations: flesh out more of cloud services model * feat: cloud integrations: reorganize things a little * feat: cloud integrations: get svc controller started * feat: cloud integrations: add stubs for EC2 and RDS postgres services * feat: cloud integrations: add validation for listing and getting available svcs and some cleanup * feat: cloud integrations: refactor helpers in existing integrations code for reuse * feat: cloud integrations: parsing of cloud service definitions * feat: cloud integrations: impl for getCloudProviderService * feat: cloud integrations: some reorganization * feat: cloud integrations: some more cleanup * feat: cloud integrations: add validation for listing available cloud provider services * feat: cloud integrations: API endpoint for listing available cloud provider services * feat: cloud integrations: add validation for getting details of a particular service * feat: cloud integrations: API endpoint for getting details of a service * feat: cloud integrations: add controller validation for configuring cloud services * feat: cloud integrations: get serviceConfigRepo started * feat: cloud integrations: service config in service list summaries when queried for cloud account id * feat: cloud integrations: only a supported service for a connected cloud account can be configured * feat: cloud integrations: add validation for configuring services via the API * feat: cloud integrations: API for configuring services * feat: cloud integrations: some cleanup * feat: cloud integrations: fix broken test --------- Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
2025-01-16 17:36:09 +05:30
func initAccountsSqliteDBIfNeeded(db *sqlx.DB) error {
feat: aws integration: UI facing QS api for cloud account management (#6771) * feat: init app/cloud_integrations * feat: get API test started for cloudintegrations account lifecycle * feat: cloudintegrations: get controller started * feat: cloud integrations: add cloudintegrations.Controller to APIHandler and servers * feat: cloud integrations: get routes started * feat: cloud integrations: get accounts table schema started * feat: cloud integrations: get cloudProviderAccountsSQLRepository started * feat: cloud integrations: cloudProviderAccountsSQLRepository.listAccounts * feat: cloud integrations: http handler and controller plumbing for /generate-connection-url * feat: cloud integrations: cloudProviderAccountsSQLRepository.upsert * feat: cloud integrations: finish up with /generate-connection-url * feat: cloud integrations: add cloudProviderAccountsRepository.get * feat: cloud integrations: add API test expectation for being able to get account status * feat: cloud integrations: add http handler and controller method for getting account status * feat: cloud integrations: ensure unconnected accounts aren't included in list of connected accounts * feat: cloud integrations: add test expectation for agent check in request * feat: cloud integrations: agent check in API * feat: cloud integrations: ensure polling for status after agent check in works * feat: cloud integrations: ensure account included in connected account list after agent check in * feat: cloud integrations: add API expectation for updating account config * feat: cloud integrations: API for updating cloud account config * feat: cloud integrations: expectation for agent receiving latest config after account config update * feat: cloud integrations: expectation for disconnecting cloud accounts from UI * feat: cloud integrations: API for disconnecting cloud accounts * feat: cloud integrations: some cleanup * feat: cloud integrations: some more cleanup * feat: cloud integrations: repo: scope rows by cloud provider * feat: testutils: refactor out helper for creating a test sqlite DB * feat: cloud integrations: controller: add test validating regeneration of connection url * feat: cloud integrations: controller: validations for agent check ins * feat: cloud integrations: connected account response structure * feat: cloud integrations: API response account structure * feat: cloud integrations: some more cleanup * feat: cloud integrations: remove cloudProviderAccountsRepository.GetById * feat: cloud integrations: shouldn't be able to disconnect non-existent account * feat: cloud integrations: validate agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: ensure agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: remove stray import of ee/model in cloudintegrations controller
2025-01-10 18:43:35 +05:30
if db == nil {
return fmt.Errorf("db is required")
}
createTablesStatements := `
CREATE TABLE IF NOT EXISTS cloud_integrations_accounts(
cloud_provider TEXT NOT NULL,
id TEXT NOT NULL,
config_json TEXT,
cloud_account_id TEXT,
last_agent_report_json TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
removed_at TIMESTAMP,
UNIQUE(cloud_provider, id)
)
`
_, err := db.Exec(createTablesStatements)
if err != nil {
return fmt.Errorf(
feat: aws integration UI facing api: services (#6803) * feat: cloud service integrations: get model and repo interface started * feat: cloud service integrations: flesh out more of cloud services model * feat: cloud integrations: reorganize things a little * feat: cloud integrations: get svc controller started * feat: cloud integrations: add stubs for EC2 and RDS postgres services * feat: cloud integrations: add validation for listing and getting available svcs and some cleanup * feat: cloud integrations: refactor helpers in existing integrations code for reuse * feat: cloud integrations: parsing of cloud service definitions * feat: cloud integrations: impl for getCloudProviderService * feat: cloud integrations: some reorganization * feat: cloud integrations: some more cleanup * feat: cloud integrations: add validation for listing available cloud provider services * feat: cloud integrations: API endpoint for listing available cloud provider services * feat: cloud integrations: add validation for getting details of a particular service * feat: cloud integrations: API endpoint for getting details of a service * feat: cloud integrations: add controller validation for configuring cloud services * feat: cloud integrations: get serviceConfigRepo started * feat: cloud integrations: service config in service list summaries when queried for cloud account id * feat: cloud integrations: only a supported service for a connected cloud account can be configured * feat: cloud integrations: add validation for configuring services via the API * feat: cloud integrations: API for configuring services * feat: cloud integrations: some cleanup * feat: cloud integrations: fix broken test --------- Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
2025-01-16 17:36:09 +05:30
"could not ensure cloud provider accounts schema in sqlite DB: %w", err,
feat: aws integration: UI facing QS api for cloud account management (#6771) * feat: init app/cloud_integrations * feat: get API test started for cloudintegrations account lifecycle * feat: cloudintegrations: get controller started * feat: cloud integrations: add cloudintegrations.Controller to APIHandler and servers * feat: cloud integrations: get routes started * feat: cloud integrations: get accounts table schema started * feat: cloud integrations: get cloudProviderAccountsSQLRepository started * feat: cloud integrations: cloudProviderAccountsSQLRepository.listAccounts * feat: cloud integrations: http handler and controller plumbing for /generate-connection-url * feat: cloud integrations: cloudProviderAccountsSQLRepository.upsert * feat: cloud integrations: finish up with /generate-connection-url * feat: cloud integrations: add cloudProviderAccountsRepository.get * feat: cloud integrations: add API test expectation for being able to get account status * feat: cloud integrations: add http handler and controller method for getting account status * feat: cloud integrations: ensure unconnected accounts aren't included in list of connected accounts * feat: cloud integrations: add test expectation for agent check in request * feat: cloud integrations: agent check in API * feat: cloud integrations: ensure polling for status after agent check in works * feat: cloud integrations: ensure account included in connected account list after agent check in * feat: cloud integrations: add API expectation for updating account config * feat: cloud integrations: API for updating cloud account config * feat: cloud integrations: expectation for agent receiving latest config after account config update * feat: cloud integrations: expectation for disconnecting cloud accounts from UI * feat: cloud integrations: API for disconnecting cloud accounts * feat: cloud integrations: some cleanup * feat: cloud integrations: some more cleanup * feat: cloud integrations: repo: scope rows by cloud provider * feat: testutils: refactor out helper for creating a test sqlite DB * feat: cloud integrations: controller: add test validating regeneration of connection url * feat: cloud integrations: controller: validations for agent check ins * feat: cloud integrations: connected account response structure * feat: cloud integrations: API response account structure * feat: cloud integrations: some more cleanup * feat: cloud integrations: remove cloudProviderAccountsRepository.GetById * feat: cloud integrations: shouldn't be able to disconnect non-existent account * feat: cloud integrations: validate agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: ensure agents can't check in to cloud account with 2 signoz ids * feat: cloud integrations: remove stray import of ee/model in cloudintegrations controller
2025-01-10 18:43:35 +05:30
)
}
return nil
}
type cloudProviderAccountsSQLRepository struct {
db *sqlx.DB
}
func (r *cloudProviderAccountsSQLRepository) listConnected(
ctx context.Context, cloudProvider string,
) ([]AccountRecord, *model.ApiError) {
accounts := []AccountRecord{}
err := r.db.SelectContext(
ctx, &accounts, `
select
cloud_provider,
id,
config_json,
cloud_account_id,
last_agent_report_json,
created_at,
removed_at
from cloud_integrations_accounts
where
cloud_provider=$1
and removed_at is NULL
and cloud_account_id is not NULL
and last_agent_report_json is not NULL
order by created_at
`, cloudProvider,
)
if err != nil {
return nil, model.InternalError(fmt.Errorf(
"could not query connected cloud accounts: %w", err,
))
}
return accounts, nil
}
func (r *cloudProviderAccountsSQLRepository) get(
ctx context.Context, cloudProvider string, id string,
) (*AccountRecord, *model.ApiError) {
var result AccountRecord
err := r.db.GetContext(
ctx, &result, `
select
cloud_provider,
id,
config_json,
cloud_account_id,
last_agent_report_json,
created_at,
removed_at
from cloud_integrations_accounts
where
cloud_provider=$1
and id=$2
`,
cloudProvider, id,
)
if err == sql.ErrNoRows {
return nil, model.NotFoundError(fmt.Errorf(
"couldn't find account with Id %s", id,
))
} else if err != nil {
return nil, model.InternalError(fmt.Errorf(
"couldn't query cloud provider accounts: %w", err,
))
}
return &result, nil
}
func (r *cloudProviderAccountsSQLRepository) getConnectedCloudAccount(
ctx context.Context, cloudProvider string, cloudAccountId string,
) (*AccountRecord, *model.ApiError) {
var result AccountRecord
err := r.db.GetContext(
ctx, &result, `
select
cloud_provider,
id,
config_json,
cloud_account_id,
last_agent_report_json,
created_at,
removed_at
from cloud_integrations_accounts
where
cloud_provider=$1
and cloud_account_id=$2
and last_agent_report_json is not NULL
and removed_at is NULL
`,
cloudProvider, cloudAccountId,
)
if err == sql.ErrNoRows {
return nil, model.NotFoundError(fmt.Errorf(
"couldn't find connected cloud account %s", cloudAccountId,
))
} else if err != nil {
return nil, model.InternalError(fmt.Errorf(
"couldn't query cloud provider accounts: %w", err,
))
}
return &result, nil
}
func (r *cloudProviderAccountsSQLRepository) upsert(
ctx context.Context,
cloudProvider string,
id *string,
config *AccountConfig,
cloudAccountId *string,
agentReport *AgentReport,
removedAt *time.Time,
) (*AccountRecord, *model.ApiError) {
// Insert
if id == nil {
newId := uuid.NewString()
id = &newId
}
// Prepare clause for setting values in `on conflict do update`
onConflictSetStmts := []string{}
setColStatement := func(col string) string {
return fmt.Sprintf("%s=excluded.%s", col, col)
}
if config != nil {
onConflictSetStmts = append(
onConflictSetStmts, setColStatement("config_json"),
)
}
if cloudAccountId != nil {
onConflictSetStmts = append(
onConflictSetStmts, setColStatement("cloud_account_id"),
)
}
if agentReport != nil {
onConflictSetStmts = append(
onConflictSetStmts, setColStatement("last_agent_report_json"),
)
}
if removedAt != nil {
onConflictSetStmts = append(
onConflictSetStmts, setColStatement("removed_at"),
)
}
onConflictClause := ""
if len(onConflictSetStmts) > 0 {
onConflictClause = fmt.Sprintf(
"on conflict(cloud_provider, id) do update SET\n%s",
strings.Join(onConflictSetStmts, ",\n"),
)
}
insertQuery := fmt.Sprintf(`
INSERT INTO cloud_integrations_accounts (
cloud_provider,
id,
config_json,
cloud_account_id,
last_agent_report_json,
removed_at
) values ($1, $2, $3, $4, $5, $6)
%s`, onConflictClause,
)
_, dbErr := r.db.ExecContext(
ctx, insertQuery,
cloudProvider, id, config, cloudAccountId, agentReport, removedAt,
)
if dbErr != nil {
return nil, model.InternalError(fmt.Errorf(
"could not upsert cloud account record: %w", dbErr,
))
}
upsertedAccount, apiErr := r.get(ctx, cloudProvider, *id)
if apiErr != nil {
return nil, model.InternalError(fmt.Errorf(
"couldn't fetch upserted account by id: %w", apiErr.ToError(),
))
}
return upsertedAccount, nil
}