mirror of
https://github.com/SigNoz/signoz.git
synced 2025-12-17 15:36:48 +00:00
chore: add multiple thresholds support (#8816)
This commit is contained in:
parent
61acd946cc
commit
d801fcee76
@ -35,7 +35,6 @@ import (
|
|||||||
anomalyV2 "github.com/SigNoz/signoz/ee/anomaly"
|
anomalyV2 "github.com/SigNoz/signoz/ee/anomaly"
|
||||||
|
|
||||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||||
yaml "gopkg.in/yaml.v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -253,11 +252,18 @@ func (r *AnomalyRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, t
|
|||||||
r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON))
|
r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON))
|
||||||
|
|
||||||
for _, series := range queryResult.AnomalyScores {
|
for _, series := range queryResult.AnomalyScores {
|
||||||
smpl, shouldAlert := r.ShouldAlert(*series)
|
if r.Condition() != nil && r.Condition().RequireMinPoints {
|
||||||
if shouldAlert {
|
if len(series.Points) < r.Condition().RequiredNumPoints {
|
||||||
resultVector = append(resultVector, smpl)
|
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
results, err := r.Threshold.ShouldAlert(*series)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resultVector = append(resultVector, results...)
|
||||||
|
}
|
||||||
return resultVector, nil
|
return resultVector, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -296,11 +302,18 @@ func (r *AnomalyRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUID,
|
|||||||
r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON))
|
r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON))
|
||||||
|
|
||||||
for _, series := range queryResult.AnomalyScores {
|
for _, series := range queryResult.AnomalyScores {
|
||||||
smpl, shouldAlert := r.ShouldAlert(*series)
|
if r.Condition().RequireMinPoints {
|
||||||
if shouldAlert {
|
if len(series.Points) < r.Condition().RequiredNumPoints {
|
||||||
resultVector = append(resultVector, smpl)
|
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
results, err := r.Threshold.ShouldAlert(*series)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resultVector = append(resultVector, results...)
|
||||||
|
}
|
||||||
return resultVector, nil
|
return resultVector, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -499,7 +512,7 @@ func (r *AnomalyRule) String() string {
|
|||||||
PreferredChannels: r.PreferredChannels(),
|
PreferredChannels: r.PreferredChannels(),
|
||||||
}
|
}
|
||||||
|
|
||||||
byt, err := yaml.Marshal(ar)
|
byt, err := json.Marshal(ar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Sprintf("error marshaling alerting rule: %s", err.Error())
|
return fmt.Sprintf("error marshaling alerting rule: %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@ -46,6 +46,11 @@ func PrepareFilterExpression(labels map[string]string, whereClause string, group
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//delete predefined alert labels
|
||||||
|
for _, label := range PredefinedAlertLabels {
|
||||||
|
delete(labels, label)
|
||||||
|
}
|
||||||
|
|
||||||
groupBySet := make(map[string]struct{})
|
groupBySet := make(map[string]struct{})
|
||||||
for _, item := range groupByItems {
|
for _, item := range groupByItems {
|
||||||
groupBySet[item.Name] = struct{}{}
|
groupBySet[item.Name] = struct{}{}
|
||||||
|
|||||||
@ -150,6 +150,11 @@ func PrepareLinksToLogs(start, end time.Time, filterItems []v3.FilterItem) strin
|
|||||||
func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, groupByItems []v3.AttributeKey, keys map[string]v3.AttributeKey) []v3.FilterItem {
|
func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, groupByItems []v3.AttributeKey, keys map[string]v3.AttributeKey) []v3.FilterItem {
|
||||||
filterItems := make([]v3.FilterItem, 0)
|
filterItems := make([]v3.FilterItem, 0)
|
||||||
|
|
||||||
|
//delete predefined alert labels
|
||||||
|
for _, label := range PredefinedAlertLabels {
|
||||||
|
delete(labels, label)
|
||||||
|
}
|
||||||
|
|
||||||
added := make(map[string]struct{})
|
added := make(map[string]struct{})
|
||||||
|
|
||||||
for _, item := range whereClauseItems {
|
for _, item := range whereClauseItems {
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
package contextlinks
|
package contextlinks
|
||||||
|
|
||||||
import v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
import (
|
||||||
|
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||||
|
"github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||||
|
)
|
||||||
|
|
||||||
// TODO(srikanthccv): Fix the URL management
|
// TODO(srikanthccv): Fix the URL management
|
||||||
type URLShareableTimeRange struct {
|
type URLShareableTimeRange struct {
|
||||||
@ -38,3 +41,5 @@ type URLShareableOptions struct {
|
|||||||
Format string `json:"format"`
|
Format string `json:"format"`
|
||||||
SelectColumns []v3.AttributeKey `json:"selectColumns"`
|
SelectColumns []v3.AttributeKey `json:"selectColumns"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var PredefinedAlertLabels = []string{ruletypes.LabelThresholdName}
|
||||||
|
|||||||
@ -32,6 +32,8 @@ type BaseRule struct {
|
|||||||
typ ruletypes.AlertType
|
typ ruletypes.AlertType
|
||||||
|
|
||||||
ruleCondition *ruletypes.RuleCondition
|
ruleCondition *ruletypes.RuleCondition
|
||||||
|
|
||||||
|
Threshold ruletypes.RuleThreshold
|
||||||
// evalWindow is the time window used for evaluating the rule
|
// evalWindow is the time window used for evaluating the rule
|
||||||
// i.e each time we lookback from the current time, we look at data for the last
|
// i.e each time we lookback from the current time, we look at data for the last
|
||||||
// evalWindow duration
|
// evalWindow duration
|
||||||
@ -123,6 +125,10 @@ func NewBaseRule(id string, orgID valuer.UUID, p *ruletypes.PostableRule, reader
|
|||||||
if p.RuleCondition == nil || !p.RuleCondition.IsValid() {
|
if p.RuleCondition == nil || !p.RuleCondition.IsValid() {
|
||||||
return nil, fmt.Errorf("invalid rule condition")
|
return nil, fmt.Errorf("invalid rule condition")
|
||||||
}
|
}
|
||||||
|
threshold, err := p.RuleCondition.Thresholds.GetRuleThreshold()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
baseRule := &BaseRule{
|
baseRule := &BaseRule{
|
||||||
id: id,
|
id: id,
|
||||||
@ -139,6 +145,7 @@ func NewBaseRule(id string, orgID valuer.UUID, p *ruletypes.PostableRule, reader
|
|||||||
Active: map[uint64]*ruletypes.Alert{},
|
Active: map[uint64]*ruletypes.Alert{},
|
||||||
reader: reader,
|
reader: reader,
|
||||||
TemporalityMap: make(map[string]map[v3.Temporality]bool),
|
TemporalityMap: make(map[string]map[v3.Temporality]bool),
|
||||||
|
Threshold: threshold,
|
||||||
}
|
}
|
||||||
|
|
||||||
if baseRule.evalWindow == 0 {
|
if baseRule.evalWindow == 0 {
|
||||||
|
|||||||
@ -266,30 +266,17 @@ func (m *Manager) initiate(ctx context.Context) error {
|
|||||||
|
|
||||||
for _, rec := range storedRules {
|
for _, rec := range storedRules {
|
||||||
taskName := fmt.Sprintf("%s-groupname", rec.ID.StringValue())
|
taskName := fmt.Sprintf("%s-groupname", rec.ID.StringValue())
|
||||||
parsedRule, err := ruletypes.ParsePostableRule([]byte(rec.Data))
|
parsedRule := ruletypes.PostableRule{}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(rec.Data), &parsedRule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ruletypes.ErrFailedToParseJSON) {
|
zap.L().Info("failed to load rule in json format", zap.String("name", taskName))
|
||||||
zap.L().Info("failed to load rule in json format, trying yaml now:", zap.String("name", taskName))
|
|
||||||
|
|
||||||
// see if rule is stored in yaml format
|
|
||||||
parsedRule, err = ruletypes.ParsePostableRuleWithKind([]byte(rec.Data), ruletypes.RuleDataKindYaml)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Error("failed to parse and initialize yaml rule", zap.String("name", taskName), zap.Error(err))
|
|
||||||
// just one rule is being parsed so expect just one error
|
|
||||||
loadErrors = append(loadErrors, err)
|
loadErrors = append(loadErrors, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
zap.L().Error("failed to parse and initialize rule", zap.String("name", taskName), zap.Error(err))
|
|
||||||
// just one rule is being parsed so expect just one error
|
|
||||||
loadErrors = append(loadErrors, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !parsedRule.Disabled {
|
if !parsedRule.Disabled {
|
||||||
err := m.addTask(ctx, org.ID, parsedRule, taskName)
|
err := m.addTask(ctx, org.ID, &parsedRule, taskName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zap.L().Error("failed to load the rule definition", zap.String("name", taskName), zap.Error(err))
|
zap.L().Error("failed to load the rule definition", zap.String("name", taskName), zap.Error(err))
|
||||||
}
|
}
|
||||||
@ -335,8 +322,8 @@ func (m *Manager) EditRule(ctx context.Context, ruleStr string, id valuer.UUID)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
parsedRule := ruletypes.PostableRule{}
|
||||||
parsedRule, err := ruletypes.ParsePostableRule([]byte(ruleStr))
|
err = json.Unmarshal([]byte(ruleStr), &parsedRule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -380,7 +367,7 @@ func (m *Manager) EditRule(ctx context.Context, ruleStr string, id valuer.UUID)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = m.syncRuleStateWithTask(ctx, orgID, prepareTaskName(existingRule.ID.StringValue()), parsedRule)
|
err = m.syncRuleStateWithTask(ctx, orgID, prepareTaskName(existingRule.ID.StringValue()), &parsedRule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -513,8 +500,8 @@ func (m *Manager) CreateRule(ctx context.Context, ruleStr string) (*ruletypes.Ge
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
parsedRule := ruletypes.PostableRule{}
|
||||||
parsedRule, err := ruletypes.ParsePostableRule([]byte(ruleStr))
|
err = json.Unmarshal([]byte(ruleStr), &parsedRule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -567,7 +554,7 @@ func (m *Manager) CreateRule(ctx context.Context, ruleStr string) (*ruletypes.Ge
|
|||||||
}
|
}
|
||||||
|
|
||||||
taskName := prepareTaskName(id.StringValue())
|
taskName := prepareTaskName(id.StringValue())
|
||||||
if err := m.addTask(ctx, orgID, parsedRule, taskName); err != nil {
|
if err := m.addTask(ctx, orgID, &parsedRule, taskName); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -579,7 +566,7 @@ func (m *Manager) CreateRule(ctx context.Context, ruleStr string) (*ruletypes.Ge
|
|||||||
|
|
||||||
return &ruletypes.GettableRule{
|
return &ruletypes.GettableRule{
|
||||||
Id: id.StringValue(),
|
Id: id.StringValue(),
|
||||||
PostableRule: *parsedRule,
|
PostableRule: parsedRule,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -797,8 +784,9 @@ func (m *Manager) ListRuleStates(ctx context.Context) (*ruletypes.GettableRules,
|
|||||||
|
|
||||||
for _, s := range storedRules {
|
for _, s := range storedRules {
|
||||||
|
|
||||||
ruleResponse := &ruletypes.GettableRule{}
|
ruleResponse := ruletypes.GettableRule{}
|
||||||
if err := json.Unmarshal([]byte(s.Data), ruleResponse); err != nil { // Parse []byte to go struct pointer
|
err = json.Unmarshal([]byte(s.Data), &ruleResponse)
|
||||||
|
if err != nil {
|
||||||
zap.L().Error("failed to unmarshal rule from db", zap.String("id", s.ID.StringValue()), zap.Error(err))
|
zap.L().Error("failed to unmarshal rule from db", zap.String("id", s.ID.StringValue()), zap.Error(err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -816,7 +804,7 @@ func (m *Manager) ListRuleStates(ctx context.Context) (*ruletypes.GettableRules,
|
|||||||
ruleResponse.CreatedBy = &s.CreatedBy
|
ruleResponse.CreatedBy = &s.CreatedBy
|
||||||
ruleResponse.UpdatedAt = &s.UpdatedAt
|
ruleResponse.UpdatedAt = &s.UpdatedAt
|
||||||
ruleResponse.UpdatedBy = &s.UpdatedBy
|
ruleResponse.UpdatedBy = &s.UpdatedBy
|
||||||
resp = append(resp, ruleResponse)
|
resp = append(resp, &ruleResponse)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &ruletypes.GettableRules{Rules: resp}, nil
|
return &ruletypes.GettableRules{Rules: resp}, nil
|
||||||
@ -827,8 +815,10 @@ func (m *Manager) GetRule(ctx context.Context, id valuer.UUID) (*ruletypes.Getta
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
r := &ruletypes.GettableRule{}
|
r := ruletypes.GettableRule{}
|
||||||
if err := json.Unmarshal([]byte(s.Data), r); err != nil {
|
err = json.Unmarshal([]byte(s.Data), &r)
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Error("failed to unmarshal rule from db", zap.String("id", s.ID.StringValue()), zap.Error(err))
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
r.Id = id.StringValue()
|
r.Id = id.StringValue()
|
||||||
@ -844,7 +834,7 @@ func (m *Manager) GetRule(ctx context.Context, id valuer.UUID) (*ruletypes.Getta
|
|||||||
r.UpdatedAt = &s.UpdatedAt
|
r.UpdatedAt = &s.UpdatedAt
|
||||||
r.UpdatedBy = &s.UpdatedBy
|
r.UpdatedBy = &s.UpdatedBy
|
||||||
|
|
||||||
return r, nil
|
return &r, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// syncRuleStateWithTask ensures that the state of a stored rule matches
|
// syncRuleStateWithTask ensures that the state of a stored rule matches
|
||||||
@ -902,20 +892,14 @@ func (m *Manager) PatchRule(ctx context.Context, ruleStr string, id valuer.UUID)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// storedRule holds the current stored rule from DB
|
// storedRule holds the current stored rule from DB
|
||||||
storedRule := ruletypes.PostableRule{}
|
patchedRule := ruletypes.PostableRule{}
|
||||||
if err := json.Unmarshal([]byte(storedJSON.Data), &storedRule); err != nil {
|
if err := json.Unmarshal([]byte(ruleStr), &patchedRule); err != nil {
|
||||||
zap.L().Error("failed to unmarshal stored rule with given id", zap.String("id", id.StringValue()), zap.Error(err))
|
zap.L().Error("failed to unmarshal stored rule with given id", zap.String("id", id.StringValue()), zap.Error(err))
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// patchedRule is combo of stored rule and patch received in the request
|
|
||||||
patchedRule, err := ruletypes.ParseIntoRule(storedRule, []byte(ruleStr), "json")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// deploy or un-deploy task according to patched (new) rule state
|
// deploy or un-deploy task according to patched (new) rule state
|
||||||
if err := m.syncRuleStateWithTask(ctx, orgID, taskName, patchedRule); err != nil {
|
if err := m.syncRuleStateWithTask(ctx, orgID, taskName, &patchedRule); err != nil {
|
||||||
zap.L().Error("failed to sync stored rule state with the task", zap.String("taskName", taskName), zap.Error(err))
|
zap.L().Error("failed to sync stored rule state with the task", zap.String("taskName", taskName), zap.Error(err))
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -933,7 +917,7 @@ func (m *Manager) PatchRule(ctx context.Context, ruleStr string, id valuer.UUID)
|
|||||||
|
|
||||||
err = m.ruleStore.EditRule(ctx, storedJSON, func(ctx context.Context) error { return nil })
|
err = m.ruleStore.EditRule(ctx, storedJSON, func(ctx context.Context) error { return nil })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err := m.syncRuleStateWithTask(ctx, orgID, taskName, &storedRule); err != nil {
|
if err := m.syncRuleStateWithTask(ctx, orgID, taskName, &patchedRule); err != nil {
|
||||||
zap.L().Error("failed to restore rule after patch failure", zap.String("taskName", taskName), zap.Error(err))
|
zap.L().Error("failed to restore rule after patch failure", zap.String("taskName", taskName), zap.Error(err))
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -942,7 +926,7 @@ func (m *Manager) PatchRule(ctx context.Context, ruleStr string, id valuer.UUID)
|
|||||||
// prepare http response
|
// prepare http response
|
||||||
response := ruletypes.GettableRule{
|
response := ruletypes.GettableRule{
|
||||||
Id: id.StringValue(),
|
Id: id.StringValue(),
|
||||||
PostableRule: *patchedRule,
|
PostableRule: patchedRule,
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetch state of rule from memory
|
// fetch state of rule from memory
|
||||||
@ -959,15 +943,14 @@ func (m *Manager) PatchRule(ctx context.Context, ruleStr string, id valuer.UUID)
|
|||||||
// TestNotification prepares a dummy rule for given rule parameters and
|
// TestNotification prepares a dummy rule for given rule parameters and
|
||||||
// sends a test notification. returns alert count and error (if any)
|
// sends a test notification. returns alert count and error (if any)
|
||||||
func (m *Manager) TestNotification(ctx context.Context, orgID valuer.UUID, ruleStr string) (int, *model.ApiError) {
|
func (m *Manager) TestNotification(ctx context.Context, orgID valuer.UUID, ruleStr string) (int, *model.ApiError) {
|
||||||
|
parsedRule := ruletypes.PostableRule{}
|
||||||
parsedRule, err := ruletypes.ParsePostableRule([]byte(ruleStr))
|
err := json.Unmarshal([]byte(ruleStr), &parsedRule)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, model.BadRequest(err)
|
return 0, model.BadRequest(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
alertCount, apiErr := m.prepareTestRuleFunc(PrepareTestRuleOptions{
|
alertCount, apiErr := m.prepareTestRuleFunc(PrepareTestRuleOptions{
|
||||||
Rule: parsedRule,
|
Rule: &parsedRule,
|
||||||
RuleStore: m.ruleStore,
|
RuleStore: m.ruleStore,
|
||||||
MaintenanceStore: m.maintenanceStore,
|
MaintenanceStore: m.maintenanceStore,
|
||||||
Logger: m.logger,
|
Logger: m.logger,
|
||||||
@ -1001,15 +984,15 @@ func (m *Manager) GetAlertDetailsForMetricNames(ctx context.Context, metricNames
|
|||||||
|
|
||||||
for _, storedRule := range rules {
|
for _, storedRule := range rules {
|
||||||
var rule ruletypes.GettableRule
|
var rule ruletypes.GettableRule
|
||||||
if err := json.Unmarshal([]byte(storedRule.Data), &rule); err != nil {
|
err = json.Unmarshal([]byte(storedRule.Data), &rule)
|
||||||
zap.L().Error("Invalid rule data", zap.Error(err))
|
if err != nil {
|
||||||
|
zap.L().Error("failed to unmarshal rule from db", zap.String("id", storedRule.ID.StringValue()), zap.Error(err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if rule.AlertType != ruletypes.AlertTypeMetric || rule.RuleCondition == nil || rule.RuleCondition.CompositeQuery == nil {
|
if rule.AlertType != ruletypes.AlertTypeMetric || rule.RuleCondition == nil || rule.RuleCondition.CompositeQuery == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
rule.Id = storedRule.ID.StringValue()
|
rule.Id = storedRule.ID.StringValue()
|
||||||
rule.CreatedAt = &storedRule.CreatedAt
|
rule.CreatedAt = &storedRule.CreatedAt
|
||||||
rule.CreatedBy = &storedRule.CreatedBy
|
rule.CreatedBy = &storedRule.CreatedBy
|
||||||
|
|||||||
@ -16,12 +16,10 @@ import (
|
|||||||
qslabels "github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
qslabels "github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||||
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
||||||
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
||||||
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||||
ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes"
|
ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||||
"github.com/SigNoz/signoz/pkg/valuer"
|
"github.com/SigNoz/signoz/pkg/valuer"
|
||||||
"github.com/prometheus/prometheus/promql"
|
"github.com/prometheus/prometheus/promql"
|
||||||
yaml "gopkg.in/yaml.v2"
|
|
||||||
|
|
||||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type PromRule struct {
|
type PromRule struct {
|
||||||
@ -151,24 +149,26 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error)
|
|||||||
var alerts = make(map[uint64]*ruletypes.Alert, len(res))
|
var alerts = make(map[uint64]*ruletypes.Alert, len(res))
|
||||||
|
|
||||||
for _, series := range res {
|
for _, series := range res {
|
||||||
l := make(map[string]string, len(series.Metric))
|
|
||||||
for _, lbl := range series.Metric {
|
|
||||||
l[lbl.Name] = lbl.Value
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(series.Floats) == 0 {
|
if len(series.Floats) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
alertSmpl, shouldAlert := r.ShouldAlert(toCommonSeries(series))
|
results, err := r.Threshold.ShouldAlert(toCommonSeries(series))
|
||||||
if !shouldAlert {
|
if err != nil {
|
||||||
continue
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, result := range results {
|
||||||
|
l := make(map[string]string, len(series.Metric))
|
||||||
|
for _, lbl := range series.Metric {
|
||||||
|
l[lbl.Name] = lbl.Value
|
||||||
}
|
}
|
||||||
r.logger.DebugContext(ctx, "alerting for series", "rule_name", r.Name(), "series", series)
|
r.logger.DebugContext(ctx, "alerting for series", "rule_name", r.Name(), "series", series)
|
||||||
|
|
||||||
threshold := valueFormatter.Format(r.targetVal(), r.Unit())
|
threshold := valueFormatter.Format(r.targetVal(), r.Unit())
|
||||||
|
|
||||||
tmplData := ruletypes.AlertTemplateData(l, valueFormatter.Format(alertSmpl.V, r.Unit()), threshold)
|
tmplData := ruletypes.AlertTemplateData(l, valueFormatter.Format(result.V, r.Unit()), threshold)
|
||||||
// Inject some convenience variables that are easier to remember for users
|
// Inject some convenience variables that are easier to remember for users
|
||||||
// who are not used to Go's templating system.
|
// who are not used to Go's templating system.
|
||||||
defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}"
|
defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}"
|
||||||
@ -191,8 +191,8 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error)
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
lb := qslabels.NewBuilder(alertSmpl.Metric).Del(qslabels.MetricNameLabel)
|
lb := qslabels.NewBuilder(result.Metric).Del(qslabels.MetricNameLabel)
|
||||||
resultLabels := qslabels.NewBuilder(alertSmpl.Metric).Del(qslabels.MetricNameLabel).Labels()
|
resultLabels := qslabels.NewBuilder(result.Metric).Del(qslabels.MetricNameLabel).Labels()
|
||||||
|
|
||||||
for name, value := range r.labels.Map() {
|
for name, value := range r.labels.Map() {
|
||||||
lb.Set(name, expand(value))
|
lb.Set(name, expand(value))
|
||||||
@ -226,11 +226,12 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error)
|
|||||||
Annotations: annotations,
|
Annotations: annotations,
|
||||||
ActiveAt: ts,
|
ActiveAt: ts,
|
||||||
State: model.StatePending,
|
State: model.StatePending,
|
||||||
Value: alertSmpl.V,
|
Value: result.V,
|
||||||
GeneratorURL: r.GeneratorURL(),
|
GeneratorURL: r.GeneratorURL(),
|
||||||
Receivers: r.preferredChannels,
|
Receivers: r.preferredChannels,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
r.logger.InfoContext(ctx, "number of alerts found", "rule_name", r.Name(), "alerts_count", len(alerts))
|
r.logger.InfoContext(ctx, "number of alerts found", "rule_name", r.Name(), "alerts_count", len(alerts))
|
||||||
|
|
||||||
@ -327,7 +328,7 @@ func (r *PromRule) String() string {
|
|||||||
PreferredChannels: r.preferredChannels,
|
PreferredChannels: r.preferredChannels,
|
||||||
}
|
}
|
||||||
|
|
||||||
byt, err := yaml.Marshal(ar)
|
byt, err := json.Marshal(ar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Sprintf("error marshaling alerting rule: %s", err.Error())
|
return fmt.Sprintf("error marshaling alerting rule: %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@ -12,6 +12,17 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func getVectorValues(vectors []ruletypes.Sample) []float64 {
|
||||||
|
if len(vectors) == 0 {
|
||||||
|
return []float64{} // Return empty slice instead of nil
|
||||||
|
}
|
||||||
|
var values []float64
|
||||||
|
for _, v := range vectors {
|
||||||
|
values = append(values, v.V)
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromRuleShouldAlert(t *testing.T) {
|
func TestPromRuleShouldAlert(t *testing.T) {
|
||||||
postableRule := ruletypes.PostableRule{
|
postableRule := ruletypes.PostableRule{
|
||||||
AlertName: "Test Rule",
|
AlertName: "Test Rule",
|
||||||
@ -38,6 +49,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
matchType string
|
matchType string
|
||||||
target float64
|
target float64
|
||||||
expectedAlertSample v3.Point
|
expectedAlertSample v3.Point
|
||||||
|
expectedVectorValues []float64 // Expected values in result vector
|
||||||
}{
|
}{
|
||||||
// Test cases for Equals Always
|
// Test cases for Equals Always
|
||||||
{
|
{
|
||||||
@ -55,6 +67,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
matchType: "2", // Always
|
matchType: "2", // Always
|
||||||
target: 0.0,
|
target: 0.0,
|
||||||
expectedAlertSample: v3.Point{Value: 0.0},
|
expectedAlertSample: v3.Point{Value: 0.0},
|
||||||
|
expectedVectorValues: []float64{0.0},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
values: pql.Series{
|
values: pql.Series{
|
||||||
@ -70,6 +83,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
compareOp: "3", // Equals
|
compareOp: "3", // Equals
|
||||||
matchType: "2", // Always
|
matchType: "2", // Always
|
||||||
target: 0.0,
|
target: 0.0,
|
||||||
|
expectedVectorValues: []float64{},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
values: pql.Series{
|
values: pql.Series{
|
||||||
@ -85,6 +99,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
compareOp: "3", // Equals
|
compareOp: "3", // Equals
|
||||||
matchType: "2", // Always
|
matchType: "2", // Always
|
||||||
target: 0.0,
|
target: 0.0,
|
||||||
|
expectedVectorValues: []float64{},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
values: pql.Series{
|
values: pql.Series{
|
||||||
@ -117,6 +132,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
matchType: "1", // Once
|
matchType: "1", // Once
|
||||||
target: 0.0,
|
target: 0.0,
|
||||||
expectedAlertSample: v3.Point{Value: 0.0},
|
expectedAlertSample: v3.Point{Value: 0.0},
|
||||||
|
expectedVectorValues: []float64{0.0},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
values: pql.Series{
|
values: pql.Series{
|
||||||
@ -164,6 +180,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
compareOp: "3", // Equals
|
compareOp: "3", // Equals
|
||||||
matchType: "1", // Once
|
matchType: "1", // Once
|
||||||
target: 0.0,
|
target: 0.0,
|
||||||
|
expectedVectorValues: []float64{},
|
||||||
},
|
},
|
||||||
// Test cases for Greater Than Always
|
// Test cases for Greater Than Always
|
||||||
{
|
{
|
||||||
@ -181,6 +198,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
matchType: "2", // Always
|
matchType: "2", // Always
|
||||||
target: 1.5,
|
target: 1.5,
|
||||||
expectedAlertSample: v3.Point{Value: 2.0},
|
expectedAlertSample: v3.Point{Value: 2.0},
|
||||||
|
expectedVectorValues: []float64{2.0},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
values: pql.Series{
|
values: pql.Series{
|
||||||
@ -245,6 +263,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
matchType: "1", // Once
|
matchType: "1", // Once
|
||||||
target: 4.5,
|
target: 4.5,
|
||||||
expectedAlertSample: v3.Point{Value: 10.0},
|
expectedAlertSample: v3.Point{Value: 10.0},
|
||||||
|
expectedVectorValues: []float64{10.0},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
values: pql.Series{
|
values: pql.Series{
|
||||||
@ -659,13 +678,49 @@ func TestPromRuleShouldAlert(t *testing.T) {
|
|||||||
postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
|
postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
|
||||||
postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
|
postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
|
||||||
postableRule.RuleCondition.Target = &c.target
|
postableRule.RuleCondition.Target = &c.target
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: &c.target,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
rule, err := NewPromRule("69", valuer.GenerateUUID(), &postableRule, logger, nil, nil)
|
rule, err := NewPromRule("69", valuer.GenerateUUID(), &postableRule, logger, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, shoulAlert := rule.ShouldAlert(toCommonSeries(c.values))
|
resultVectors, err := rule.Threshold.ShouldAlert(toCommonSeries(c.values))
|
||||||
assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// Compare full result vector with expected vector
|
||||||
|
actualValues := getVectorValues(resultVectors)
|
||||||
|
if c.expectedVectorValues != nil {
|
||||||
|
// If expected vector values are specified, compare them exactly
|
||||||
|
assert.Equal(t, c.expectedVectorValues, actualValues, "Result vector values don't match expected for case %d", idx)
|
||||||
|
} else {
|
||||||
|
// Fallback to the old logic for cases without expectedVectorValues
|
||||||
|
if c.expectAlert {
|
||||||
|
assert.NotEmpty(t, resultVectors, "Expected alert but got no result vectors for case %d", idx)
|
||||||
|
// Verify at least one of the result vectors matches the expected alert sample
|
||||||
|
if len(resultVectors) > 0 {
|
||||||
|
found := false
|
||||||
|
for _, sample := range resultVectors {
|
||||||
|
if sample.V == c.expectedAlertSample.Value {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.True(t, found, "Expected alert sample value %.2f not found in result vectors for case %d. Got values: %v", c.expectedAlertSample.Value, idx, actualValues)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
assert.Empty(t, resultVectors, "Expected no alert but got result vectors for case %d", idx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -38,8 +38,6 @@ import (
|
|||||||
querierV5 "github.com/SigNoz/signoz/pkg/querier"
|
querierV5 "github.com/SigNoz/signoz/pkg/querier"
|
||||||
|
|
||||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||||
|
|
||||||
yaml "gopkg.in/yaml.v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ThresholdRule struct {
|
type ThresholdRule struct {
|
||||||
@ -484,11 +482,18 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID,
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, series := range queryResult.Series {
|
for _, series := range queryResult.Series {
|
||||||
smpl, shouldAlert := r.ShouldAlert(*series)
|
if r.Condition() != nil && r.Condition().RequireMinPoints {
|
||||||
if shouldAlert {
|
if len(series.Points) < r.ruleCondition.RequiredNumPoints {
|
||||||
resultVector = append(resultVector, smpl)
|
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
resultSeries, err := r.Threshold.ShouldAlert(*series)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resultVector = append(resultVector, resultSeries...)
|
||||||
|
}
|
||||||
|
|
||||||
return resultVector, nil
|
return resultVector, nil
|
||||||
}
|
}
|
||||||
@ -554,11 +559,18 @@ func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUI
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, series := range queryResult.Series {
|
for _, series := range queryResult.Series {
|
||||||
smpl, shouldAlert := r.ShouldAlert(*series)
|
if r.Condition() != nil && r.Condition().RequireMinPoints {
|
||||||
if shouldAlert {
|
if len(series.Points) < r.Condition().RequiredNumPoints {
|
||||||
resultVector = append(resultVector, smpl)
|
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
resultSeries, err := r.Threshold.ShouldAlert(*series)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resultVector = append(resultVector, resultSeries...)
|
||||||
|
}
|
||||||
|
|
||||||
return resultVector, nil
|
return resultVector, nil
|
||||||
}
|
}
|
||||||
@ -597,6 +609,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er
|
|||||||
}
|
}
|
||||||
|
|
||||||
value := valueFormatter.Format(smpl.V, r.Unit())
|
value := valueFormatter.Format(smpl.V, r.Unit())
|
||||||
|
//todo(aniket): handle different threshold
|
||||||
threshold := valueFormatter.Format(r.targetVal(), r.Unit())
|
threshold := valueFormatter.Format(r.targetVal(), r.Unit())
|
||||||
r.logger.DebugContext(ctx, "Alert template data for rule", "rule_name", r.Name(), "formatter", valueFormatter.Name(), "value", value, "threshold", threshold)
|
r.logger.DebugContext(ctx, "Alert template data for rule", "rule_name", r.Name(), "formatter", valueFormatter.Name(), "value", value, "threshold", threshold)
|
||||||
|
|
||||||
@ -777,7 +790,7 @@ func (r *ThresholdRule) String() string {
|
|||||||
PreferredChannels: r.preferredChannels,
|
PreferredChannels: r.preferredChannels,
|
||||||
}
|
}
|
||||||
|
|
||||||
byt, err := yaml.Marshal(ar)
|
byt, err := json.Marshal(ar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Sprintf("error marshaling alerting rule: %s", err.Error())
|
return fmt.Sprintf("error marshaling alerting rule: %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@ -801,9 +801,16 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for idx, c := range cases {
|
for idx, c := range cases {
|
||||||
postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
postableRule.RuleCondition.Target = &c.target
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: &c.target,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -815,10 +822,24 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
|
|||||||
values.Points[i].Timestamp = time.Now().UnixMilli()
|
values.Points[i].Timestamp = time.Now().UnixMilli()
|
||||||
}
|
}
|
||||||
|
|
||||||
smpl, shoulAlert := rule.ShouldAlert(c.values)
|
resultVectors, err := rule.Threshold.ShouldAlert(c.values)
|
||||||
assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx)
|
assert.NoError(t, err, "Test case %d", idx)
|
||||||
if shoulAlert {
|
|
||||||
assert.Equal(t, c.expectedAlertSample.Value, smpl.V, "Test case %d", idx)
|
// Compare result vectors with expected behavior
|
||||||
|
if c.expectAlert {
|
||||||
|
assert.NotEmpty(t, resultVectors, "Expected alert but got no result vectors for case %d", idx)
|
||||||
|
if len(resultVectors) > 0 {
|
||||||
|
found := false
|
||||||
|
for _, sample := range resultVectors {
|
||||||
|
if sample.V == c.expectedAlertSample.Value {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.True(t, found, "Expected alert sample value %.2f not found in result vectors for case %d. Got values: %v", c.expectedAlertSample.Value, idx, getVectorValues(resultVectors))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
assert.Empty(t, resultVectors, "Expected no alert but got result vectors for case %d", idx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -894,7 +915,16 @@ func TestPrepareLinksToLogs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger := instrumentationtest.New().Logger()
|
logger := instrumentationtest.New().Logger()
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: postableRule.RuleCondition.Target,
|
||||||
|
MatchType: ruletypes.MatchType(postableRule.RuleCondition.MatchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(postableRule.RuleCondition.CompareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -944,7 +974,16 @@ func TestPrepareLinksToLogsV5(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger := instrumentationtest.New().Logger()
|
logger := instrumentationtest.New().Logger()
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: postableRule.RuleCondition.Target,
|
||||||
|
MatchType: ruletypes.MatchType(postableRule.RuleCondition.MatchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(postableRule.RuleCondition.CompareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -994,7 +1033,16 @@ func TestPrepareLinksToTracesV5(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger := instrumentationtest.New().Logger()
|
logger := instrumentationtest.New().Logger()
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: postableRule.RuleCondition.Target,
|
||||||
|
MatchType: ruletypes.MatchType(postableRule.RuleCondition.MatchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(postableRule.RuleCondition.CompareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -1037,7 +1085,16 @@ func TestPrepareLinksToTraces(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger := instrumentationtest.New().Logger()
|
logger := instrumentationtest.New().Logger()
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: postableRule.RuleCondition.Target,
|
||||||
|
MatchType: ruletypes.MatchType(postableRule.RuleCondition.MatchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(postableRule.RuleCondition.CompareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -1111,9 +1168,16 @@ func TestThresholdRuleLabelNormalization(t *testing.T) {
|
|||||||
logger := instrumentationtest.New().Logger()
|
logger := instrumentationtest.New().Logger()
|
||||||
|
|
||||||
for idx, c := range cases {
|
for idx, c := range cases {
|
||||||
postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
postableRule.RuleCondition.Target = &c.target
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: &c.target,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1125,12 +1189,26 @@ func TestThresholdRuleLabelNormalization(t *testing.T) {
|
|||||||
values.Points[i].Timestamp = time.Now().UnixMilli()
|
values.Points[i].Timestamp = time.Now().UnixMilli()
|
||||||
}
|
}
|
||||||
|
|
||||||
sample, shoulAlert := rule.ShouldAlert(c.values)
|
vector, err := rule.Threshold.ShouldAlert(c.values)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
for name, value := range c.values.Labels {
|
for name, value := range c.values.Labels {
|
||||||
|
for _, sample := range vector {
|
||||||
assert.Equal(t, value, sample.Metric.Get(name))
|
assert.Equal(t, value, sample.Metric.Get(name))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx)
|
// Get result vectors from threshold evaluation
|
||||||
|
resultVectors, err := rule.Threshold.ShouldAlert(c.values)
|
||||||
|
assert.NoError(t, err, "Test case %d", idx)
|
||||||
|
|
||||||
|
// Compare result vectors with expected behavior
|
||||||
|
if c.expectAlert {
|
||||||
|
assert.NotEmpty(t, resultVectors, "Expected alert but got no result vectors for case %d", idx)
|
||||||
|
// For this test, we don't have expectedAlertSample, so just verify vectors exist
|
||||||
|
} else {
|
||||||
|
assert.Empty(t, resultVectors, "Expected no alert but got result vectors for case %d", idx)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1152,6 +1230,16 @@ func TestThresholdRuleEvalDelay(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: postableRule.RuleCondition.Target,
|
||||||
|
MatchType: ruletypes.MatchType(postableRule.RuleCondition.MatchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(postableRule.RuleCondition.CompareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// 01:39:47
|
// 01:39:47
|
||||||
ts := time.Unix(1717205987, 0)
|
ts := time.Unix(1717205987, 0)
|
||||||
@ -1169,6 +1257,7 @@ func TestThresholdRuleEvalDelay(t *testing.T) {
|
|||||||
logger := instrumentationtest.New().Logger()
|
logger := instrumentationtest.New().Logger()
|
||||||
|
|
||||||
for idx, c := range cases {
|
for idx, c := range cases {
|
||||||
|
|
||||||
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger) // no eval delay
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger) // no eval delay
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -1202,6 +1291,16 @@ func TestThresholdRuleClickHouseTmpl(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: postableRule.RuleCondition.Target,
|
||||||
|
MatchType: ruletypes.MatchType(postableRule.RuleCondition.MatchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(postableRule.RuleCondition.CompareOp),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// 01:39:47
|
// 01:39:47
|
||||||
ts := time.Unix(1717205987, 0)
|
ts := time.Unix(1717205987, 0)
|
||||||
@ -1380,6 +1479,20 @@ func TestThresholdRuleUnitCombinations(t *testing.T) {
|
|||||||
postableRule.RuleCondition.Target = &c.target
|
postableRule.RuleCondition.Target = &c.target
|
||||||
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
||||||
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
Name: postableRule.AlertName,
|
||||||
|
TargetValue: &c.target,
|
||||||
|
TargetUnit: c.targetUnit,
|
||||||
|
RuleUnit: postableRule.RuleCondition.CompositeQuery.Unit,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
SelectedQuery: postableRule.RuleCondition.SelectedQuery,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
postableRule.Annotations = map[string]string{
|
postableRule.Annotations = map[string]string{
|
||||||
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
||||||
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
||||||
@ -1475,9 +1588,19 @@ func TestThresholdRuleNoData(t *testing.T) {
|
|||||||
ExpectQuery(queryString).
|
ExpectQuery(queryString).
|
||||||
WillReturnRows(rows)
|
WillReturnRows(rows)
|
||||||
var target float64 = 0
|
var target float64 = 0
|
||||||
postableRule.RuleCondition.CompareOp = ruletypes.ValueIsEq
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
postableRule.RuleCondition.MatchType = ruletypes.AtleastOnce
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
postableRule.RuleCondition.Target = &target
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
Name: postableRule.AlertName,
|
||||||
|
TargetValue: &target,
|
||||||
|
RuleUnit: postableRule.RuleCondition.CompositeQuery.Unit,
|
||||||
|
MatchType: ruletypes.AtleastOnce,
|
||||||
|
CompareOp: ruletypes.ValueIsEq,
|
||||||
|
SelectedQuery: postableRule.RuleCondition.SelectedQuery,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
postableRule.Annotations = map[string]string{
|
postableRule.Annotations = map[string]string{
|
||||||
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
||||||
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
||||||
@ -1585,6 +1708,20 @@ func TestThresholdRuleTracesLink(t *testing.T) {
|
|||||||
postableRule.RuleCondition.Target = &c.target
|
postableRule.RuleCondition.Target = &c.target
|
||||||
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
||||||
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
Name: postableRule.AlertName,
|
||||||
|
TargetValue: &c.target,
|
||||||
|
TargetUnit: c.targetUnit,
|
||||||
|
RuleUnit: postableRule.RuleCondition.CompositeQuery.Unit,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
SelectedQuery: postableRule.RuleCondition.SelectedQuery,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
postableRule.Annotations = map[string]string{
|
postableRule.Annotations = map[string]string{
|
||||||
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
||||||
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
||||||
@ -1708,6 +1845,20 @@ func TestThresholdRuleLogsLink(t *testing.T) {
|
|||||||
postableRule.RuleCondition.Target = &c.target
|
postableRule.RuleCondition.Target = &c.target
|
||||||
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
||||||
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
Name: postableRule.AlertName,
|
||||||
|
TargetValue: &c.target,
|
||||||
|
TargetUnit: c.targetUnit,
|
||||||
|
RuleUnit: postableRule.RuleCondition.CompositeQuery.Unit,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
SelectedQuery: postableRule.RuleCondition.SelectedQuery,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
postableRule.Annotations = map[string]string{
|
postableRule.Annotations = map[string]string{
|
||||||
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
||||||
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
||||||
@ -1756,6 +1907,16 @@ func TestThresholdRuleShiftBy(t *testing.T) {
|
|||||||
EvalWindow: ruletypes.Duration(5 * time.Minute),
|
EvalWindow: ruletypes.Duration(5 * time.Minute),
|
||||||
Frequency: ruletypes.Duration(1 * time.Minute),
|
Frequency: ruletypes.Duration(1 * time.Minute),
|
||||||
RuleCondition: &ruletypes.RuleCondition{
|
RuleCondition: &ruletypes.RuleCondition{
|
||||||
|
Thresholds: &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
TargetValue: &target,
|
||||||
|
CompareOp: ruletypes.ValueAboveOrEq,
|
||||||
|
MatchType: ruletypes.AtleastOnce,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
CompositeQuery: &v3.CompositeQuery{
|
CompositeQuery: &v3.CompositeQuery{
|
||||||
QueryType: v3.QueryTypeBuilder,
|
QueryType: v3.QueryTypeBuilder,
|
||||||
BuilderQueries: map[string]*v3.BuilderQuery{
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
@ -1787,8 +1948,6 @@ func TestThresholdRuleShiftBy(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Target: &target,
|
|
||||||
CompareOp: ruletypes.ValueAboveOrEq,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1811,3 +1970,188 @@ func TestThresholdRuleShiftBy(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(t, int64(10), params.CompositeQuery.BuilderQueries["A"].ShiftBy)
|
assert.Equal(t, int64(10), params.CompositeQuery.BuilderQueries["A"].ShiftBy)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMultipleThresholdRule(t *testing.T) {
|
||||||
|
postableRule := ruletypes.PostableRule{
|
||||||
|
AlertName: "Mulitple threshold test",
|
||||||
|
AlertType: ruletypes.AlertTypeMetric,
|
||||||
|
RuleType: ruletypes.RuleTypeThreshold,
|
||||||
|
EvalWindow: ruletypes.Duration(5 * time.Minute),
|
||||||
|
Frequency: ruletypes.Duration(1 * time.Minute),
|
||||||
|
RuleCondition: &ruletypes.RuleCondition{
|
||||||
|
CompositeQuery: &v3.CompositeQuery{
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
StepInterval: 60,
|
||||||
|
AggregateAttribute: v3.AttributeKey{
|
||||||
|
Key: "signoz_calls_total",
|
||||||
|
},
|
||||||
|
AggregateOperator: v3.AggregateOperatorSumRate,
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
|
||||||
|
|
||||||
|
cols := make([]cmock.ColumnType, 0)
|
||||||
|
cols = append(cols, cmock.ColumnType{Name: "value", Type: "Float64"})
|
||||||
|
cols = append(cols, cmock.ColumnType{Name: "attr", Type: "String"})
|
||||||
|
cols = append(cols, cmock.ColumnType{Name: "timestamp", Type: "String"})
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
targetUnit string
|
||||||
|
yAxisUnit string
|
||||||
|
values [][]interface{}
|
||||||
|
expectAlerts int
|
||||||
|
compareOp string
|
||||||
|
matchType string
|
||||||
|
target float64
|
||||||
|
secondTarget float64
|
||||||
|
summaryAny []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
targetUnit: "s",
|
||||||
|
yAxisUnit: "ns",
|
||||||
|
values: [][]interface{}{
|
||||||
|
{float64(572588400), "attr", time.Now()}, // 0.57 seconds
|
||||||
|
{float64(572386400), "attr", time.Now().Add(1 * time.Second)}, // 0.57 seconds
|
||||||
|
{float64(300947400), "attr", time.Now().Add(2 * time.Second)}, // 0.3 seconds
|
||||||
|
{float64(299316000), "attr", time.Now().Add(3 * time.Second)}, // 0.3 seconds
|
||||||
|
{float64(66640400.00000001), "attr", time.Now().Add(4 * time.Second)}, // 0.06 seconds
|
||||||
|
},
|
||||||
|
expectAlerts: 2,
|
||||||
|
compareOp: "1", // Above
|
||||||
|
matchType: "1", // Once
|
||||||
|
target: 1, // 1 second
|
||||||
|
secondTarget: .5,
|
||||||
|
summaryAny: []string{
|
||||||
|
"observed metric value is 573 ms",
|
||||||
|
"observed metric value is 572 ms",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
targetUnit: "ms",
|
||||||
|
yAxisUnit: "ns",
|
||||||
|
values: [][]interface{}{
|
||||||
|
{float64(572588400), "attr", time.Now()}, // 572.58 ms
|
||||||
|
{float64(572386400), "attr", time.Now().Add(1 * time.Second)}, // 572.38 ms
|
||||||
|
{float64(300947400), "attr", time.Now().Add(2 * time.Second)}, // 300.94 ms
|
||||||
|
{float64(299316000), "attr", time.Now().Add(3 * time.Second)}, // 299.31 ms
|
||||||
|
{float64(66640400.00000001), "attr", time.Now().Add(4 * time.Second)}, // 66.64 ms
|
||||||
|
},
|
||||||
|
expectAlerts: 6, // Expects 6 values exceed 200ms (572.58, 572.38, 300.94, 299.31) + 2 values exceed 500ms (572.58, 572.38)
|
||||||
|
compareOp: "1", // Above
|
||||||
|
matchType: "1", // Once
|
||||||
|
target: 200, // 200 ms
|
||||||
|
secondTarget: 500,
|
||||||
|
summaryAny: []string{
|
||||||
|
"observed metric value is 299 ms",
|
||||||
|
"the observed metric value is 573 ms",
|
||||||
|
"the observed metric value is 572 ms",
|
||||||
|
"the observed metric value is 301 ms",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
targetUnit: "decgbytes",
|
||||||
|
yAxisUnit: "bytes",
|
||||||
|
values: [][]interface{}{
|
||||||
|
{float64(2863284053), "attr", time.Now()}, // 2.86 GB
|
||||||
|
{float64(2863388842), "attr", time.Now().Add(1 * time.Second)}, // 2.86 GB
|
||||||
|
{float64(300947400), "attr", time.Now().Add(2 * time.Second)}, // 0.3 GB
|
||||||
|
{float64(299316000), "attr", time.Now().Add(3 * time.Second)}, // 0.3 GB
|
||||||
|
{float64(66640400.00000001), "attr", time.Now().Add(4 * time.Second)}, // 66.64 MB
|
||||||
|
},
|
||||||
|
expectAlerts: 2,
|
||||||
|
compareOp: "1", // Above
|
||||||
|
matchType: "1", // Once
|
||||||
|
target: 200, // 200 GB
|
||||||
|
secondTarget: 2, // 2GB
|
||||||
|
summaryAny: []string{
|
||||||
|
"observed metric value is 2.7 GiB",
|
||||||
|
"the observed metric value is 0.3 GB",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
logger := instrumentationtest.New().Logger()
|
||||||
|
|
||||||
|
for idx, c := range cases {
|
||||||
|
rows := cmock.NewRows(cols, c.values)
|
||||||
|
// We are testing the eval logic after the query is run
|
||||||
|
// so we don't care about the query string here
|
||||||
|
queryString := "SELECT any"
|
||||||
|
telemetryStore.Mock().
|
||||||
|
ExpectQuery(queryString).
|
||||||
|
WillReturnRows(rows)
|
||||||
|
postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
|
||||||
|
postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
|
||||||
|
postableRule.RuleCondition.Target = &c.target
|
||||||
|
postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit
|
||||||
|
postableRule.RuleCondition.TargetUnit = c.targetUnit
|
||||||
|
postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
|
||||||
|
Kind: ruletypes.BasicThresholdKind,
|
||||||
|
Spec: ruletypes.BasicRuleThresholds{
|
||||||
|
{
|
||||||
|
Name: "first_threshold",
|
||||||
|
TargetValue: &c.target,
|
||||||
|
TargetUnit: c.targetUnit,
|
||||||
|
RuleUnit: postableRule.RuleCondition.CompositeQuery.Unit,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
SelectedQuery: postableRule.RuleCondition.SelectedQuery,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "second_threshold",
|
||||||
|
TargetValue: &c.secondTarget,
|
||||||
|
TargetUnit: c.targetUnit,
|
||||||
|
RuleUnit: postableRule.RuleCondition.CompositeQuery.Unit,
|
||||||
|
MatchType: ruletypes.MatchType(c.matchType),
|
||||||
|
CompareOp: ruletypes.CompareOp(c.compareOp),
|
||||||
|
SelectedQuery: postableRule.RuleCondition.SelectedQuery,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
postableRule.Annotations = map[string]string{
|
||||||
|
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
|
||||||
|
"summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}",
|
||||||
|
}
|
||||||
|
|
||||||
|
options := clickhouseReader.NewOptions("", "", "archiveNamespace")
|
||||||
|
readerCache, err := cachetest.New(cache.Config{Provider: "memory", Memory: cache.Memory{TTL: DefaultFrequency}})
|
||||||
|
require.NoError(t, err)
|
||||||
|
reader := clickhouseReader.NewReaderFromClickhouseConnection(options, nil, telemetryStore, prometheustest.New(instrumentationtest.New().Logger(), prometheus.Config{}), "", time.Duration(time.Second), readerCache)
|
||||||
|
rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
|
||||||
|
rule.TemporalityMap = map[string]map[v3.Temporality]bool{
|
||||||
|
"signoz_calls_total": {
|
||||||
|
v3.Delta: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
retVal, err := rule.Eval(context.Background(), time.Now())
|
||||||
|
if err != nil {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, c.expectAlerts, retVal.(int), "case %d", idx)
|
||||||
|
if c.expectAlerts != 0 {
|
||||||
|
foundCount := 0
|
||||||
|
for _, item := range rule.Active {
|
||||||
|
for _, summary := range c.summaryAny {
|
||||||
|
if strings.Contains(item.Annotations.Get("summary"), summary) {
|
||||||
|
foundCount++
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.Equal(t, c.expectAlerts, foundCount, "case %d", idx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -11,7 +11,6 @@ import (
|
|||||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||||
"github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
"github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||||
|
|
||||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -106,18 +105,19 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type RuleCondition struct {
|
type RuleCondition struct {
|
||||||
CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"`
|
CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty"`
|
||||||
CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"`
|
CompareOp CompareOp `json:"op,omitempty"`
|
||||||
Target *float64 `yaml:"target,omitempty" json:"target,omitempty"`
|
Target *float64 `json:"target,omitempty"`
|
||||||
AlertOnAbsent bool `yaml:"alertOnAbsent,omitempty" json:"alertOnAbsent,omitempty"`
|
AlertOnAbsent bool `json:"alertOnAbsent,omitempty"`
|
||||||
AbsentFor uint64 `yaml:"absentFor,omitempty" json:"absentFor,omitempty"`
|
AbsentFor uint64 `json:"absentFor,omitempty"`
|
||||||
MatchType MatchType `json:"matchType,omitempty"`
|
MatchType MatchType `json:"matchType,omitempty"`
|
||||||
TargetUnit string `json:"targetUnit,omitempty"`
|
TargetUnit string `json:"targetUnit,omitempty"`
|
||||||
Algorithm string `json:"algorithm,omitempty"`
|
Algorithm string `json:"algorithm,omitempty"`
|
||||||
Seasonality string `json:"seasonality,omitempty"`
|
Seasonality string `json:"seasonality,omitempty"`
|
||||||
SelectedQuery string `json:"selectedQueryName,omitempty"`
|
SelectedQuery string `json:"selectedQueryName,omitempty"`
|
||||||
RequireMinPoints bool `yaml:"requireMinPoints,omitempty" json:"requireMinPoints,omitempty"`
|
RequireMinPoints bool `json:"requireMinPoints,omitempty"`
|
||||||
RequiredNumPoints int `yaml:"requiredNumPoints,omitempty" json:"requiredNumPoints,omitempty"`
|
RequiredNumPoints int `json:"requiredNumPoints,omitempty"`
|
||||||
|
Thresholds *RuleThresholdData `json:"thresholds,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RuleCondition) GetSelectedQueryName() string {
|
func (rc *RuleCondition) GetSelectedQueryName() string {
|
||||||
@ -188,10 +188,7 @@ func (rc *RuleCondition) IsValid() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if rc.QueryType() == v3.QueryTypeBuilder {
|
if rc.QueryType() == v3.QueryTypeBuilder {
|
||||||
if rc.Target == nil {
|
if rc.Thresholds == nil {
|
||||||
return false
|
|
||||||
}
|
|
||||||
if rc.CompareOp == "" {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,18 +3,15 @@ package ruletypes
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
signozError "github.com/SigNoz/signoz/pkg/errors"
|
||||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||||
"github.com/pkg/errors"
|
|
||||||
"go.uber.org/multierr"
|
|
||||||
|
|
||||||
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
||||||
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
||||||
yaml "gopkg.in/yaml.v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type AlertType string
|
type AlertType string
|
||||||
@ -30,30 +27,20 @@ type RuleDataKind string
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
RuleDataKindJson RuleDataKind = "json"
|
RuleDataKindJson RuleDataKind = "json"
|
||||||
RuleDataKindYaml RuleDataKind = "yaml"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
ErrFailedToParseJSON = errors.New("failed to parse json")
|
|
||||||
ErrFailedToParseYAML = errors.New("failed to parse yaml")
|
|
||||||
ErrInvalidDataType = errors.New("invalid data type")
|
|
||||||
)
|
|
||||||
|
|
||||||
// this file contains api request and responses to be
|
|
||||||
// served over http
|
|
||||||
|
|
||||||
// PostableRule is used to create alerting rule from HTTP api
|
// PostableRule is used to create alerting rule from HTTP api
|
||||||
type PostableRule struct {
|
type PostableRule struct {
|
||||||
AlertName string `yaml:"alert,omitempty" json:"alert,omitempty"`
|
AlertName string `json:"alert,omitempty"`
|
||||||
AlertType AlertType `yaml:"alertType,omitempty" json:"alertType,omitempty"`
|
AlertType AlertType `json:"alertType,omitempty"`
|
||||||
Description string `yaml:"description,omitempty" json:"description,omitempty"`
|
Description string `json:"description,omitempty"`
|
||||||
RuleType RuleType `yaml:"ruleType,omitempty" json:"ruleType,omitempty"`
|
RuleType RuleType `json:"ruleType,omitempty"`
|
||||||
EvalWindow Duration `yaml:"evalWindow,omitempty" json:"evalWindow,omitempty"`
|
EvalWindow Duration `json:"evalWindow,omitempty"`
|
||||||
Frequency Duration `yaml:"frequency,omitempty" json:"frequency,omitempty"`
|
Frequency Duration `json:"frequency,omitempty"`
|
||||||
|
|
||||||
RuleCondition *RuleCondition `yaml:"condition,omitempty" json:"condition,omitempty"`
|
RuleCondition *RuleCondition `json:"condition,omitempty"`
|
||||||
Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"`
|
Labels map[string]string `json:"labels,omitempty"`
|
||||||
Annotations map[string]string `yaml:"annotations,omitempty" json:"annotations,omitempty"`
|
Annotations map[string]string `json:"annotations,omitempty"`
|
||||||
|
|
||||||
Disabled bool `json:"disabled"`
|
Disabled bool `json:"disabled"`
|
||||||
|
|
||||||
@ -63,84 +50,65 @@ type PostableRule struct {
|
|||||||
PreferredChannels []string `json:"preferredChannels,omitempty"`
|
PreferredChannels []string `json:"preferredChannels,omitempty"`
|
||||||
|
|
||||||
Version string `json:"version,omitempty"`
|
Version string `json:"version,omitempty"`
|
||||||
|
|
||||||
// legacy
|
|
||||||
Expr string `yaml:"expr,omitempty" json:"expr,omitempty"`
|
|
||||||
OldYaml string `json:"yaml,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParsePostableRule(content []byte) (*PostableRule, error) {
|
func (r *PostableRule) processRuleDefaults() error {
|
||||||
return ParsePostableRuleWithKind(content, "json")
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParsePostableRuleWithKind(content []byte, kind RuleDataKind) (*PostableRule, error) {
|
if r.EvalWindow == 0 {
|
||||||
return ParseIntoRule(PostableRule{}, content, kind)
|
r.EvalWindow = Duration(5 * time.Minute)
|
||||||
}
|
|
||||||
|
|
||||||
// parseIntoRule loads the content (data) into PostableRule and also
|
|
||||||
// validates the end result
|
|
||||||
func ParseIntoRule(initRule PostableRule, content []byte, kind RuleDataKind) (*PostableRule, error) {
|
|
||||||
rule := &initRule
|
|
||||||
|
|
||||||
var err error
|
|
||||||
if kind == RuleDataKindJson {
|
|
||||||
if err = json.Unmarshal(content, rule); err != nil {
|
|
||||||
return nil, ErrFailedToParseJSON
|
|
||||||
}
|
|
||||||
} else if kind == RuleDataKindYaml {
|
|
||||||
if err = yaml.Unmarshal(content, rule); err != nil {
|
|
||||||
return nil, ErrFailedToParseYAML
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return nil, ErrInvalidDataType
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if rule.RuleCondition == nil && rule.Expr != "" {
|
if r.Frequency == 0 {
|
||||||
// account for legacy rules
|
r.Frequency = Duration(1 * time.Minute)
|
||||||
rule.RuleType = RuleTypeProm
|
|
||||||
rule.EvalWindow = Duration(5 * time.Minute)
|
|
||||||
rule.Frequency = Duration(1 * time.Minute)
|
|
||||||
rule.RuleCondition = &RuleCondition{
|
|
||||||
CompositeQuery: &v3.CompositeQuery{
|
|
||||||
QueryType: v3.QueryTypePromQL,
|
|
||||||
PromQueries: map[string]*v3.PromQuery{
|
|
||||||
"A": {
|
|
||||||
Query: rule.Expr,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if rule.EvalWindow == 0 {
|
if r.RuleCondition != nil {
|
||||||
rule.EvalWindow = Duration(5 * time.Minute)
|
if r.RuleCondition.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
||||||
|
if r.RuleType == "" {
|
||||||
|
r.RuleType = RuleTypeThreshold
|
||||||
|
}
|
||||||
|
} else if r.RuleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL {
|
||||||
|
r.RuleType = RuleTypeProm
|
||||||
}
|
}
|
||||||
|
|
||||||
if rule.Frequency == 0 {
|
for qLabel, q := range r.RuleCondition.CompositeQuery.BuilderQueries {
|
||||||
rule.Frequency = Duration(1 * time.Minute)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.RuleCondition != nil {
|
|
||||||
if rule.RuleCondition.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
|
||||||
if rule.RuleType == "" {
|
|
||||||
rule.RuleType = RuleTypeThreshold
|
|
||||||
}
|
|
||||||
} else if rule.RuleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL {
|
|
||||||
rule.RuleType = RuleTypeProm
|
|
||||||
}
|
|
||||||
|
|
||||||
for qLabel, q := range rule.RuleCondition.CompositeQuery.BuilderQueries {
|
|
||||||
if q.AggregateAttribute.Key != "" && q.Expression == "" {
|
if q.AggregateAttribute.Key != "" && q.Expression == "" {
|
||||||
q.Expression = qLabel
|
q.Expression = qLabel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
//added alerts v2 fields
|
||||||
|
if r.RuleCondition.Thresholds == nil {
|
||||||
|
thresholdName := CriticalThresholdName
|
||||||
|
if r.Labels != nil {
|
||||||
|
if severity, ok := r.Labels["severity"]; ok {
|
||||||
|
thresholdName = severity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
thresholdData := RuleThresholdData{
|
||||||
|
Kind: BasicThresholdKind,
|
||||||
|
Spec: BasicRuleThresholds{{
|
||||||
|
Name: thresholdName,
|
||||||
|
RuleUnit: r.RuleCondition.CompositeQuery.Unit,
|
||||||
|
TargetUnit: r.RuleCondition.TargetUnit,
|
||||||
|
TargetValue: r.RuleCondition.Target,
|
||||||
|
MatchType: r.RuleCondition.MatchType,
|
||||||
|
CompareOp: r.RuleCondition.CompareOp,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
r.RuleCondition.Thresholds = &thresholdData
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := rule.Validate(); err != nil {
|
return r.Validate()
|
||||||
return nil, err
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return rule, nil
|
func (r *PostableRule) UnmarshalJSON(bytes []byte) error {
|
||||||
|
type Alias PostableRule
|
||||||
|
aux := (*Alias)(r)
|
||||||
|
if err := json.Unmarshal(bytes, aux); err != nil {
|
||||||
|
return signozError.NewInvalidInputf(signozError.CodeInvalidInput, "failed to parse json: %v", err)
|
||||||
|
}
|
||||||
|
return r.processRuleDefaults()
|
||||||
}
|
}
|
||||||
|
|
||||||
func isValidLabelName(ln string) bool {
|
func isValidLabelName(ln string) bool {
|
||||||
@ -204,47 +172,35 @@ func (r *PostableRule) Validate() error {
|
|||||||
|
|
||||||
if r.RuleCondition == nil {
|
if r.RuleCondition == nil {
|
||||||
// will get panic if we try to access CompositeQuery, so return here
|
// will get panic if we try to access CompositeQuery, so return here
|
||||||
return errors.Errorf("rule condition is required")
|
return signozError.NewInvalidInputf(signozError.CodeInvalidInput, "rule condition is required")
|
||||||
} else {
|
} else {
|
||||||
if r.RuleCondition.CompositeQuery == nil {
|
if r.RuleCondition.CompositeQuery == nil {
|
||||||
errs = append(errs, errors.Errorf("composite metric query is required"))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "composite metric query is required"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if isAllQueriesDisabled(r.RuleCondition.CompositeQuery) {
|
if isAllQueriesDisabled(r.RuleCondition.CompositeQuery) {
|
||||||
errs = append(errs, errors.Errorf("all queries are disabled in rule condition"))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "all queries are disabled in rule condition"))
|
||||||
}
|
|
||||||
|
|
||||||
if r.RuleType == RuleTypeThreshold {
|
|
||||||
if r.RuleCondition.Target == nil {
|
|
||||||
errs = append(errs, errors.Errorf("rule condition missing the threshold"))
|
|
||||||
}
|
|
||||||
if r.RuleCondition.CompareOp == "" {
|
|
||||||
errs = append(errs, errors.Errorf("rule condition missing the compare op"))
|
|
||||||
}
|
|
||||||
if r.RuleCondition.MatchType == "" {
|
|
||||||
errs = append(errs, errors.Errorf("rule condition missing the match option"))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for k, v := range r.Labels {
|
for k, v := range r.Labels {
|
||||||
if !isValidLabelName(k) {
|
if !isValidLabelName(k) {
|
||||||
errs = append(errs, errors.Errorf("invalid label name: %s", k))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "invalid label name: %s", k))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isValidLabelValue(v) {
|
if !isValidLabelValue(v) {
|
||||||
errs = append(errs, errors.Errorf("invalid label value: %s", v))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "invalid label value: %s", v))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for k := range r.Annotations {
|
for k := range r.Annotations {
|
||||||
if !isValidLabelName(k) {
|
if !isValidLabelName(k) {
|
||||||
errs = append(errs, errors.Errorf("invalid annotation name: %s", k))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "invalid annotation name: %s", k))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
errs = append(errs, testTemplateParsing(r)...)
|
errs = append(errs, testTemplateParsing(r)...)
|
||||||
return multierr.Combine(errs...)
|
return signozError.Join(errs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testTemplateParsing(rl *PostableRule) (errs []error) {
|
func testTemplateParsing(rl *PostableRule) (errs []error) {
|
||||||
@ -272,7 +228,7 @@ func testTemplateParsing(rl *PostableRule) (errs []error) {
|
|||||||
for _, val := range rl.Labels {
|
for _, val := range rl.Labels {
|
||||||
err := parseTest(val)
|
err := parseTest(val)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = append(errs, fmt.Errorf("msg=%s", err.Error()))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "template parsing error: %s", err.Error()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -280,7 +236,7 @@ func testTemplateParsing(rl *PostableRule) (errs []error) {
|
|||||||
for _, val := range rl.Annotations {
|
for _, val := range rl.Annotations {
|
||||||
err := parseTest(val)
|
err := parseTest(val)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = append(errs, fmt.Errorf("msg=%s", err.Error()))
|
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "template parsing error: %s", err.Error()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,7 +1,10 @@
|
|||||||
package ruletypes
|
package ruletypes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||||
)
|
)
|
||||||
@ -84,3 +87,307 @@ func TestIsAllQueriesDisabled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseIntoRule(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
initRule PostableRule
|
||||||
|
content []byte
|
||||||
|
kind RuleDataKind
|
||||||
|
expectError bool
|
||||||
|
validate func(*testing.T, *PostableRule)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid JSON with complete rule",
|
||||||
|
initRule: PostableRule{},
|
||||||
|
content: []byte(`{
|
||||||
|
"alert": "TestAlert",
|
||||||
|
"alertType": "METRIC_BASED_ALERT",
|
||||||
|
"description": "Test description",
|
||||||
|
"ruleType": "threshold_rule",
|
||||||
|
"evalWindow": "5m",
|
||||||
|
"frequency": "1m",
|
||||||
|
"condition": {
|
||||||
|
"compositeQuery": {
|
||||||
|
"queryType": "builder",
|
||||||
|
"builderQueries": {
|
||||||
|
"A": {
|
||||||
|
"expression": "A",
|
||||||
|
"disabled": false,
|
||||||
|
"aggregateAttribute": {
|
||||||
|
"key": "test_metric"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"target": 10.0,
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A"
|
||||||
|
}
|
||||||
|
}`),
|
||||||
|
kind: RuleDataKindJson,
|
||||||
|
expectError: false,
|
||||||
|
validate: func(t *testing.T, rule *PostableRule) {
|
||||||
|
if rule.AlertName != "TestAlert" {
|
||||||
|
t.Errorf("Expected alert name 'TestAlert', got '%s'", rule.AlertName)
|
||||||
|
}
|
||||||
|
if rule.RuleType != RuleTypeThreshold {
|
||||||
|
t.Errorf("Expected rule type '%s', got '%s'", RuleTypeThreshold, rule.RuleType)
|
||||||
|
}
|
||||||
|
if rule.RuleCondition.Thresholds.Kind.IsZero() {
|
||||||
|
t.Error("Expected thresholds to be populated")
|
||||||
|
}
|
||||||
|
if rule.RuleCondition.Target == nil {
|
||||||
|
t.Error("Expected target to be populated")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "rule with default values applied",
|
||||||
|
initRule: PostableRule{},
|
||||||
|
content: []byte(`{
|
||||||
|
"alert": "DefaultsRule",
|
||||||
|
"ruleType": "threshold_rule",
|
||||||
|
"condition": {
|
||||||
|
"compositeQuery": {
|
||||||
|
"queryType": "builder",
|
||||||
|
"builderQueries": {
|
||||||
|
"A": {
|
||||||
|
"disabled": false,
|
||||||
|
"aggregateAttribute": {
|
||||||
|
"key": "test_metric"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"target": 5.0,
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A"
|
||||||
|
}
|
||||||
|
}`),
|
||||||
|
kind: RuleDataKindJson,
|
||||||
|
expectError: false,
|
||||||
|
validate: func(t *testing.T, rule *PostableRule) {
|
||||||
|
if rule.EvalWindow != Duration(5*time.Minute) {
|
||||||
|
t.Errorf("Expected default eval window '5m', got '%v'", rule.EvalWindow)
|
||||||
|
}
|
||||||
|
if rule.Frequency != Duration(1*time.Minute) {
|
||||||
|
t.Errorf("Expected default frequency '1m', got '%v'", rule.Frequency)
|
||||||
|
}
|
||||||
|
if rule.RuleCondition.CompositeQuery.BuilderQueries["A"].Expression != "A" {
|
||||||
|
t.Errorf("Expected expression 'A', got '%s'", rule.RuleCondition.CompositeQuery.BuilderQueries["A"].Expression)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PromQL rule type detection",
|
||||||
|
initRule: PostableRule{},
|
||||||
|
content: []byte(`{
|
||||||
|
"alert": "PromQLRule",
|
||||||
|
"condition": {
|
||||||
|
"compositeQuery": {
|
||||||
|
"queryType": "promql",
|
||||||
|
"promQueries": {
|
||||||
|
"A": {
|
||||||
|
"query": "rate(http_requests_total[5m])",
|
||||||
|
"disabled": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"target": 10.0,
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A"
|
||||||
|
}
|
||||||
|
}`),
|
||||||
|
kind: RuleDataKindJson,
|
||||||
|
expectError: false,
|
||||||
|
validate: func(t *testing.T, rule *PostableRule) {
|
||||||
|
if rule.RuleType != RuleTypeProm {
|
||||||
|
t.Errorf("Expected rule type 'PROM_QL_RULE', got '%s'", rule.RuleType)
|
||||||
|
}
|
||||||
|
if rule.RuleCondition.Thresholds.Kind.IsZero() {
|
||||||
|
t.Error("Expected thresholds to be populated")
|
||||||
|
}
|
||||||
|
if rule.RuleCondition.Target == nil {
|
||||||
|
t.Error("Expected target to be populated")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := json.Unmarshal(tt.content, &tt.initRule)
|
||||||
|
if tt.expectError {
|
||||||
|
if err == nil {
|
||||||
|
t.Error("Expected error but got none")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Unexpected error: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.validate != nil {
|
||||||
|
tt.validate(t, &tt.initRule)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseIntoRuleThresholdGeneration(t *testing.T) {
|
||||||
|
content := []byte(`{
|
||||||
|
"alert": "TestThresholds",
|
||||||
|
"condition": {
|
||||||
|
"compositeQuery": {
|
||||||
|
"queryType": "builder",
|
||||||
|
"builderQueries": {
|
||||||
|
"A": {
|
||||||
|
"expression": "A",
|
||||||
|
"disabled": false,
|
||||||
|
"aggregateAttribute": {
|
||||||
|
"key": "response_time"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"target": 100.0,
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A",
|
||||||
|
"targetUnit": "ms",
|
||||||
|
"thresholds": {
|
||||||
|
"kind": "basic",
|
||||||
|
"spec": [
|
||||||
|
{
|
||||||
|
"name": "CRITICAL",
|
||||||
|
"target": 100.0,
|
||||||
|
"targetUnit": "ms",
|
||||||
|
"ruleUnit": "s",
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
rule := PostableRule{}
|
||||||
|
err := json.Unmarshal(content, &rule)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that thresholds were parsed correctly
|
||||||
|
if rule.RuleCondition.Thresholds.Kind != BasicThresholdKind {
|
||||||
|
t.Errorf("Expected threshold kind 'basic', got '%s'", rule.RuleCondition.Thresholds.Kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the threshold and test functionality
|
||||||
|
threshold, err := rule.RuleCondition.Thresholds.GetRuleThreshold()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get threshold: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test that threshold can evaluate properly
|
||||||
|
vector, err := threshold.ShouldAlert(v3.Series{
|
||||||
|
Points: []v3.Point{{Value: 0.15, Timestamp: 1000}}, // 150ms in seconds
|
||||||
|
Labels: map[string]string{"test": "label"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error in ShouldAlert: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(vector) == 0 {
|
||||||
|
t.Error("Expected alert to be triggered for value above threshold")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseIntoRuleMultipleThresholds(t *testing.T) {
|
||||||
|
content := []byte(`{
|
||||||
|
"alert": "MultiThresholdAlert",
|
||||||
|
"ruleType": "threshold_rule",
|
||||||
|
"condition": {
|
||||||
|
"compositeQuery": {
|
||||||
|
"queryType": "builder",
|
||||||
|
"unit": "%",
|
||||||
|
"builderQueries": {
|
||||||
|
"A": {
|
||||||
|
"expression": "A",
|
||||||
|
"disabled": false,
|
||||||
|
"aggregateAttribute": {
|
||||||
|
"key": "cpu_usage"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"target": 90.0,
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A",
|
||||||
|
"thresholds": {
|
||||||
|
"kind": "basic",
|
||||||
|
"spec": [
|
||||||
|
{
|
||||||
|
"name": "WARNING",
|
||||||
|
"target": 70.0,
|
||||||
|
"targetUnit": "%",
|
||||||
|
"ruleUnit": "%",
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "CRITICAL",
|
||||||
|
"target": 90.0,
|
||||||
|
"targetUnit": "%",
|
||||||
|
"ruleUnit": "%",
|
||||||
|
"matchType": "1",
|
||||||
|
"op": "1",
|
||||||
|
"selectedQuery": "A"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
rule := PostableRule{}
|
||||||
|
err := json.Unmarshal(content, &rule)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.RuleCondition.Thresholds.Kind != BasicThresholdKind {
|
||||||
|
t.Errorf("Expected threshold kind 'basic', got '%s'", rule.RuleCondition.Thresholds.Kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
threshold, err := rule.RuleCondition.Thresholds.GetRuleThreshold()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get threshold: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test with a value that should trigger both WARNING and CRITICAL thresholds
|
||||||
|
vector, err := threshold.ShouldAlert(v3.Series{
|
||||||
|
Points: []v3.Point{{Value: 95.0, Timestamp: 1000}}, // 95% CPU usage
|
||||||
|
Labels: map[string]string{"service": "test"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error in ShouldAlert: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, 2, len(vector))
|
||||||
|
|
||||||
|
vector, err = threshold.ShouldAlert(v3.Series{
|
||||||
|
Points: []v3.Point{{Value: 75.0, Timestamp: 1000}}, // 75% CPU usage
|
||||||
|
Labels: map[string]string{"service": "test"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error in ShouldAlert: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, 1, len(vector))
|
||||||
|
}
|
||||||
|
|||||||
4
pkg/types/ruletypes/constants.go
Normal file
4
pkg/types/ruletypes/constants.go
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
package ruletypes
|
||||||
|
|
||||||
|
const CriticalThresholdName = "CRITICAL"
|
||||||
|
const LabelThresholdName = "threshold.name"
|
||||||
419
pkg/types/ruletypes/threshold.go
Normal file
419
pkg/types/ruletypes/threshold.go
Normal file
@ -0,0 +1,419 @@
|
|||||||
|
package ruletypes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"github.com/SigNoz/signoz/pkg/errors"
|
||||||
|
"github.com/SigNoz/signoz/pkg/query-service/converter"
|
||||||
|
"github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||||
|
"github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||||
|
"github.com/SigNoz/signoz/pkg/valuer"
|
||||||
|
"math"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ThresholdKind struct {
|
||||||
|
valuer.String
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
BasicThresholdKind = ThresholdKind{valuer.NewString("basic")}
|
||||||
|
)
|
||||||
|
|
||||||
|
type RuleThresholdData struct {
|
||||||
|
Kind ThresholdKind `json:"kind"`
|
||||||
|
Spec any `json:"spec"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RuleThresholdData) UnmarshalJSON(data []byte) error {
|
||||||
|
var raw map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &raw); err != nil {
|
||||||
|
return errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to unmarshal raw rule threshold json: %v", err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw["kind"], &r.Kind); err != nil {
|
||||||
|
return errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to unmarshal rule threshold kind: %v", err)
|
||||||
|
}
|
||||||
|
switch r.Kind {
|
||||||
|
case BasicThresholdKind:
|
||||||
|
var basicThresholds BasicRuleThresholds
|
||||||
|
if err := json.Unmarshal(raw["spec"], &basicThresholds); err != nil {
|
||||||
|
return errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to unmarshal rule threhsold spec: %v", err)
|
||||||
|
}
|
||||||
|
if err := basicThresholds.Validate(); err != nil {
|
||||||
|
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid rule threshold spec: %v", err)
|
||||||
|
}
|
||||||
|
r.Spec = basicThresholds
|
||||||
|
|
||||||
|
default:
|
||||||
|
return errors.NewInvalidInputf(errors.CodeUnsupported, "unknown threshold kind")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type RuleThreshold interface {
|
||||||
|
ShouldAlert(series v3.Series) (Vector, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type BasicRuleThreshold struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
TargetValue *float64 `json:"target"`
|
||||||
|
TargetUnit string `json:"targetUnit"`
|
||||||
|
RuleUnit string `json:"ruleUnit"`
|
||||||
|
RecoveryTarget *float64 `json:"recoveryTarget"`
|
||||||
|
MatchType MatchType `json:"matchType"`
|
||||||
|
CompareOp CompareOp `json:"op"`
|
||||||
|
SelectedQuery string `json:"selectedQuery"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BasicRuleThresholds []BasicRuleThreshold
|
||||||
|
|
||||||
|
func (r BasicRuleThresholds) Validate() error {
|
||||||
|
var errs []error
|
||||||
|
for _, basicThreshold := range r {
|
||||||
|
if err := basicThreshold.Validate(); err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r BasicRuleThresholds) ShouldAlert(series v3.Series) (Vector, error) {
|
||||||
|
var resultVector Vector
|
||||||
|
thresholds := []BasicRuleThreshold(r)
|
||||||
|
sort.Slice(thresholds, func(i, j int) bool {
|
||||||
|
compareOp := thresholds[i].GetCompareOp()
|
||||||
|
targetI := thresholds[i].Target()
|
||||||
|
targetJ := thresholds[j].Target()
|
||||||
|
|
||||||
|
switch compareOp {
|
||||||
|
case ValueIsAbove, ValueAboveOrEq, ValueOutsideBounds:
|
||||||
|
// For "above" operations, sort descending (higher values first)
|
||||||
|
return targetI > targetJ
|
||||||
|
case ValueIsBelow, ValueBelowOrEq:
|
||||||
|
// For "below" operations, sort ascending (lower values first)
|
||||||
|
return targetI < targetJ
|
||||||
|
default:
|
||||||
|
// For equal/not equal operations, use descending as default
|
||||||
|
return targetI > targetJ
|
||||||
|
}
|
||||||
|
})
|
||||||
|
for _, threshold := range thresholds {
|
||||||
|
smpl, shouldAlert := threshold.ShouldAlert(series)
|
||||||
|
if shouldAlert {
|
||||||
|
resultVector = append(resultVector, smpl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return resultVector, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) GetName() string {
|
||||||
|
return b.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) Target() float64 {
|
||||||
|
unitConverter := converter.FromUnit(converter.Unit(b.TargetUnit))
|
||||||
|
// convert the target value to the y-axis unit
|
||||||
|
value := unitConverter.Convert(converter.Value{
|
||||||
|
F: *b.TargetValue,
|
||||||
|
U: converter.Unit(b.TargetUnit),
|
||||||
|
}, converter.Unit(b.RuleUnit))
|
||||||
|
return value.F
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) GetRecoveryTarget() float64 {
|
||||||
|
if b.RecoveryTarget == nil {
|
||||||
|
return 0
|
||||||
|
} else {
|
||||||
|
return *b.RecoveryTarget
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) GetMatchType() MatchType {
|
||||||
|
return b.MatchType
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) GetCompareOp() CompareOp {
|
||||||
|
return b.CompareOp
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) GetSelectedQuery() string {
|
||||||
|
return b.SelectedQuery
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) Validate() error {
|
||||||
|
var errs []error
|
||||||
|
if b.Name == "" {
|
||||||
|
errs = append(errs, errors.NewInvalidInputf(errors.CodeInvalidInput, "threshold name cannot be empty"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if b.TargetValue == nil {
|
||||||
|
errs = append(errs, errors.NewInvalidInputf(errors.CodeInvalidInput, "target value cannot be nil"))
|
||||||
|
}
|
||||||
|
|
||||||
|
switch b.CompareOp {
|
||||||
|
case ValueIsAbove, ValueIsBelow, ValueIsEq, ValueIsNotEq, ValueAboveOrEq, ValueBelowOrEq, ValueOutsideBounds:
|
||||||
|
// valid compare operations
|
||||||
|
case CompareOpNone:
|
||||||
|
errs = append(errs, errors.NewInvalidInputf(errors.CodeInvalidInput, "compare operation cannot be none"))
|
||||||
|
default:
|
||||||
|
errs = append(errs, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid compare operation: %s", string(b.CompareOp)))
|
||||||
|
}
|
||||||
|
|
||||||
|
switch b.MatchType {
|
||||||
|
case AtleastOnce, AllTheTimes, OnAverage, InTotal, Last:
|
||||||
|
// valid match types
|
||||||
|
case MatchTypeNone:
|
||||||
|
errs = append(errs, errors.NewInvalidInputf(errors.CodeInvalidInput, "match type cannot be none"))
|
||||||
|
default:
|
||||||
|
errs = append(errs, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid match type: %s", string(b.MatchType)))
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeGroupinSetPoints(series v3.Series) []v3.Point {
|
||||||
|
var result []v3.Point
|
||||||
|
for _, s := range series.Points {
|
||||||
|
if s.Timestamp >= 0 && !math.IsNaN(s.Value) && !math.IsInf(s.Value, 0) {
|
||||||
|
result = append(result, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BasicRuleThreshold) ShouldAlert(series v3.Series) (Sample, bool) {
|
||||||
|
var shouldAlert bool
|
||||||
|
var alertSmpl Sample
|
||||||
|
var lbls labels.Labels
|
||||||
|
|
||||||
|
for name, value := range series.Labels {
|
||||||
|
lbls = append(lbls, labels.Label{Name: name, Value: value})
|
||||||
|
}
|
||||||
|
|
||||||
|
lbls = append(lbls, labels.Label{Name: LabelThresholdName, Value: b.Name})
|
||||||
|
|
||||||
|
series.Points = removeGroupinSetPoints(series)
|
||||||
|
|
||||||
|
// nothing to evaluate
|
||||||
|
if len(series.Points) == 0 {
|
||||||
|
return alertSmpl, false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch b.MatchType {
|
||||||
|
case AtleastOnce:
|
||||||
|
// If any sample matches the condition, the rule is firing.
|
||||||
|
if b.CompareOp == ValueIsAbove {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value > b.Target() {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
shouldAlert = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsBelow {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value < b.Target() {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
shouldAlert = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsEq {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value == b.Target() {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
shouldAlert = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsNotEq {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value != b.Target() {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
shouldAlert = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueOutsideBounds {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if math.Abs(smpl.Value) >= b.Target() {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
shouldAlert = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case AllTheTimes:
|
||||||
|
// If all samples match the condition, the rule is firing.
|
||||||
|
shouldAlert = true
|
||||||
|
alertSmpl = Sample{Point: Point{V: b.Target()}, Metric: lbls}
|
||||||
|
if b.CompareOp == ValueIsAbove {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value <= b.Target() {
|
||||||
|
shouldAlert = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// use min value from the series
|
||||||
|
if shouldAlert {
|
||||||
|
var minValue float64 = math.Inf(1)
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value < minValue {
|
||||||
|
minValue = smpl.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
alertSmpl = Sample{Point: Point{V: minValue}, Metric: lbls}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsBelow {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value >= b.Target() {
|
||||||
|
shouldAlert = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shouldAlert {
|
||||||
|
var maxValue float64 = math.Inf(-1)
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value > maxValue {
|
||||||
|
maxValue = smpl.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
alertSmpl = Sample{Point: Point{V: maxValue}, Metric: lbls}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsEq {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value != b.Target() {
|
||||||
|
shouldAlert = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsNotEq {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if smpl.Value == b.Target() {
|
||||||
|
shouldAlert = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// use any non-inf or nan value from the series
|
||||||
|
if shouldAlert {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if !math.IsInf(smpl.Value, 0) && !math.IsNaN(smpl.Value) {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueOutsideBounds {
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if math.Abs(smpl.Value) < b.Target() {
|
||||||
|
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||||
|
shouldAlert = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case OnAverage:
|
||||||
|
// If the average of all samples matches the condition, the rule is firing.
|
||||||
|
var sum, count float64
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if math.IsNaN(smpl.Value) || math.IsInf(smpl.Value, 0) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sum += smpl.Value
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
avg := sum / count
|
||||||
|
alertSmpl = Sample{Point: Point{V: avg}, Metric: lbls}
|
||||||
|
if b.CompareOp == ValueIsAbove {
|
||||||
|
if avg > b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsBelow {
|
||||||
|
if avg < b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsEq {
|
||||||
|
if avg == b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsNotEq {
|
||||||
|
if avg != b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueOutsideBounds {
|
||||||
|
if math.Abs(avg) >= b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case InTotal:
|
||||||
|
// If the sum of all samples matches the condition, the rule is firing.
|
||||||
|
var sum float64
|
||||||
|
|
||||||
|
for _, smpl := range series.Points {
|
||||||
|
if math.IsNaN(smpl.Value) || math.IsInf(smpl.Value, 0) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sum += smpl.Value
|
||||||
|
}
|
||||||
|
alertSmpl = Sample{Point: Point{V: sum}, Metric: lbls}
|
||||||
|
if b.CompareOp == ValueIsAbove {
|
||||||
|
if sum > b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsBelow {
|
||||||
|
if sum < b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsEq {
|
||||||
|
if sum == b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsNotEq {
|
||||||
|
if sum != b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueOutsideBounds {
|
||||||
|
if math.Abs(sum) >= b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case Last:
|
||||||
|
// If the last sample matches the condition, the rule is firing.
|
||||||
|
shouldAlert = false
|
||||||
|
alertSmpl = Sample{Point: Point{V: series.Points[len(series.Points)-1].Value}, Metric: lbls}
|
||||||
|
if b.CompareOp == ValueIsAbove {
|
||||||
|
if series.Points[len(series.Points)-1].Value > b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsBelow {
|
||||||
|
if series.Points[len(series.Points)-1].Value < b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsEq {
|
||||||
|
if series.Points[len(series.Points)-1].Value == b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
} else if b.CompareOp == ValueIsNotEq {
|
||||||
|
if series.Points[len(series.Points)-1].Value != b.Target() {
|
||||||
|
shouldAlert = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return alertSmpl, shouldAlert
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RuleThresholdData) GetRuleThreshold() (RuleThreshold, error) {
|
||||||
|
if r == nil {
|
||||||
|
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "rule threshold is nil")
|
||||||
|
}
|
||||||
|
switch r.Kind {
|
||||||
|
case BasicThresholdKind:
|
||||||
|
if thresholds, ok := r.Spec.(BasicRuleThresholds); ok {
|
||||||
|
basic := BasicRuleThresholds(thresholds)
|
||||||
|
return basic, nil
|
||||||
|
}
|
||||||
|
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid rule threshold spec")
|
||||||
|
default:
|
||||||
|
return nil, errors.NewInvalidInputf(errors.CodeUnsupported, "unknown threshold kind")
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user