diff --git a/ee/query-service/rules/anomaly.go b/ee/query-service/rules/anomaly.go index 3fbf1e32b1f2..60a2a6ddf175 100644 --- a/ee/query-service/rules/anomaly.go +++ b/ee/query-service/rules/anomaly.go @@ -253,9 +253,11 @@ func (r *AnomalyRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, t r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON)) for _, series := range queryResult.AnomalyScores { - smpl, shouldAlert := r.ShouldAlert(*series) - if shouldAlert { - resultVector = append(resultVector, smpl) + for _, threshold := range r.Thresholds() { + smpl, shouldAlert := threshold.ShouldAlert(*series) + if shouldAlert { + resultVector = append(resultVector, smpl) + } } } return resultVector, nil @@ -296,9 +298,11 @@ func (r *AnomalyRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUID, r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON)) for _, series := range queryResult.AnomalyScores { - smpl, shouldAlert := r.ShouldAlert(*series) - if shouldAlert { - resultVector = append(resultVector, smpl) + for _, threshold := range r.Thresholds() { + smpl, shouldAlert := threshold.ShouldAlert(*series) + if shouldAlert { + resultVector = append(resultVector, smpl) + } } } return resultVector, nil diff --git a/pkg/query-service/rules/base_rule.go b/pkg/query-service/rules/base_rule.go index 40952e4a5059..46aedb09b557 100644 --- a/pkg/query-service/rules/base_rule.go +++ b/pkg/query-service/rules/base_rule.go @@ -85,6 +85,9 @@ type BaseRule struct { TemporalityMap map[string]map[v3.Temporality]bool sqlstore sqlstore.SQLStore + + cronExpression string + cronEnabled bool } type RuleOption func(*BaseRule) @@ -210,6 +213,10 @@ func (r *BaseRule) TargetVal() float64 { return r.targetVal() } +func (r *BaseRule) Thresholds() []ruletypes.RuleThreshold { + return r.ruleCondition.Thresholds +} + func (r *ThresholdRule) hostFromSource() string { parsedUrl, err := url.Parse(r.source) if err != nil { diff --git a/pkg/query-service/rules/prom_rule.go b/pkg/query-service/rules/prom_rule.go index 90c9d4619c8e..4770a3e3d51b 100644 --- a/pkg/query-service/rules/prom_rule.go +++ b/pkg/query-service/rules/prom_rule.go @@ -151,84 +151,86 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error) var alerts = make(map[uint64]*ruletypes.Alert, len(res)) for _, series := range res { - l := make(map[string]string, len(series.Metric)) - for _, lbl := range series.Metric { - l[lbl.Name] = lbl.Value - } - - if len(series.Floats) == 0 { - continue - } - - alertSmpl, shouldAlert := r.ShouldAlert(toCommonSeries(series)) - if !shouldAlert { - continue - } - r.logger.DebugContext(ctx, "alerting for series", "rule_name", r.Name(), "series", series) - - threshold := valueFormatter.Format(r.targetVal(), r.Unit()) - - tmplData := ruletypes.AlertTemplateData(l, valueFormatter.Format(alertSmpl.V, r.Unit()), threshold) - // Inject some convenience variables that are easier to remember for users - // who are not used to Go's templating system. - defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}" - - expand := func(text string) string { - - tmpl := ruletypes.NewTemplateExpander( - ctx, - defs+text, - "__alert_"+r.Name(), - tmplData, - times.Time(timestamp.FromTime(ts)), - nil, - ) - result, err := tmpl.Expand() - if err != nil { - result = fmt.Sprintf("", err) - r.logger.WarnContext(ctx, "Expanding alert template failed", "rule_name", r.Name(), "error", err, "data", tmplData) + for _, ruleThreshold := range r.Thresholds() { + l := make(map[string]string, len(series.Metric)) + for _, lbl := range series.Metric { + l[lbl.Name] = lbl.Value } - return result - } - lb := qslabels.NewBuilder(alertSmpl.Metric).Del(qslabels.MetricNameLabel) - resultLabels := qslabels.NewBuilder(alertSmpl.Metric).Del(qslabels.MetricNameLabel).Labels() + if len(series.Floats) == 0 { + continue + } - for name, value := range r.labels.Map() { - lb.Set(name, expand(value)) - } + alertSmpl, shouldAlert := ruleThreshold.ShouldAlert(toCommonSeries(series)) + if !shouldAlert { + continue + } + r.logger.DebugContext(ctx, "alerting for series", "rule_name", r.Name(), "series", series) - lb.Set(qslabels.AlertNameLabel, r.Name()) - lb.Set(qslabels.AlertRuleIdLabel, r.ID()) - lb.Set(qslabels.RuleSourceLabel, r.GeneratorURL()) + threshold := valueFormatter.Format(r.targetVal(), r.Unit()) - annotations := make(qslabels.Labels, 0, len(r.annotations.Map())) - for name, value := range r.annotations.Map() { - annotations = append(annotations, qslabels.Label{Name: name, Value: expand(value)}) - } + tmplData := ruletypes.AlertTemplateData(l, valueFormatter.Format(alertSmpl.V, r.Unit()), threshold) + // Inject some convenience variables that are easier to remember for users + // who are not used to Go's templating system. + defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}" - lbs := lb.Labels() - h := lbs.Hash() - resultFPs[h] = struct{}{} + expand := func(text string) string { - if _, ok := alerts[h]; ok { - err = fmt.Errorf("vector contains metrics with the same labelset after applying alert labels") - // We have already acquired the lock above hence using SetHealth and - // SetLastError will deadlock. - r.health = ruletypes.HealthBad - r.lastError = err - return nil, err - } + tmpl := ruletypes.NewTemplateExpander( + ctx, + defs+text, + "__alert_"+r.Name(), + tmplData, + times.Time(timestamp.FromTime(ts)), + nil, + ) + result, err := tmpl.Expand() + if err != nil { + result = fmt.Sprintf("", err) + r.logger.WarnContext(ctx, "Expanding alert template failed", "rule_name", r.Name(), "error", err, "data", tmplData) + } + return result + } - alerts[h] = &ruletypes.Alert{ - Labels: lbs, - QueryResultLables: resultLabels, - Annotations: annotations, - ActiveAt: ts, - State: model.StatePending, - Value: alertSmpl.V, - GeneratorURL: r.GeneratorURL(), - Receivers: r.preferredChannels, + lb := qslabels.NewBuilder(alertSmpl.Metric).Del(qslabels.MetricNameLabel) + resultLabels := qslabels.NewBuilder(alertSmpl.Metric).Del(qslabels.MetricNameLabel).Labels() + + for name, value := range r.labels.Map() { + lb.Set(name, expand(value)) + } + + lb.Set(qslabels.AlertNameLabel, r.Name()) + lb.Set(qslabels.AlertRuleIdLabel, r.ID()) + lb.Set(qslabels.RuleSourceLabel, r.GeneratorURL()) + + annotations := make(qslabels.Labels, 0, len(r.annotations.Map())) + for name, value := range r.annotations.Map() { + annotations = append(annotations, qslabels.Label{Name: name, Value: expand(value)}) + } + + lbs := lb.Labels() + h := lbs.Hash() + resultFPs[h] = struct{}{} + + if _, ok := alerts[h]; ok { + err = fmt.Errorf("vector contains metrics with the same labelset after applying alert labels") + // We have already acquired the lock above hence using SetHealth and + // SetLastError will deadlock. + r.health = ruletypes.HealthBad + r.lastError = err + return nil, err + } + + alerts[h] = &ruletypes.Alert{ + Labels: lbs, + QueryResultLables: resultLabels, + Annotations: annotations, + ActiveAt: ts, + State: model.StatePending, + Value: alertSmpl.V, + GeneratorURL: r.GeneratorURL(), + Receivers: r.preferredChannels, + } } } diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index a2dc1db1e176..c7731822fca7 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -479,9 +479,11 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, } for _, series := range queryResult.Series { - smpl, shouldAlert := r.ShouldAlert(*series) - if shouldAlert { - resultVector = append(resultVector, smpl) + for _, threshold := range r.Thresholds() { + smpl, shouldAlert := threshold.ShouldAlert(*series) + if shouldAlert { + resultVector = append(resultVector, smpl) + } } } @@ -544,9 +546,11 @@ func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUI } for _, series := range queryResult.Series { - smpl, shouldAlert := r.ShouldAlert(*series) - if shouldAlert { - resultVector = append(resultVector, smpl) + for _, threshold := range r.Thresholds() { + smpl, shouldAlert := threshold.ShouldAlert(*series) + if shouldAlert { + resultVector = append(resultVector, smpl) + } } } @@ -587,6 +591,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er } value := valueFormatter.Format(smpl.V, r.Unit()) + //todo(aniket): handle different threshold threshold := valueFormatter.Format(r.targetVal(), r.Unit()) r.logger.DebugContext(ctx, "Alert template data for rule", "rule_name", r.Name(), "formatter", valueFormatter.Name(), "value", value, "threshold", threshold) diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 9735fe985238..3cb749a8d997 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -1351,6 +1351,7 @@ func TestThresholdRuleUnitCombinations(t *testing.T) { postableRule.RuleCondition.Target = &c.target postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit postableRule.RuleCondition.TargetUnit = c.targetUnit + postableRule.RuleCondition.Thresholds = []ruletypes.RuleThreshold{ruletypes.NewBasicRuleThreshold(postableRule.AlertName, &c.target, nil, ruletypes.MatchType(c.matchType), ruletypes.CompareOp(c.compareOp), postableRule.RuleCondition.SelectedQuery, c.targetUnit, postableRule.RuleCondition.CompositeQuery.Unit)} postableRule.Annotations = map[string]string{ "description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})", "summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}", @@ -1556,6 +1557,7 @@ func TestThresholdRuleTracesLink(t *testing.T) { postableRule.RuleCondition.Target = &c.target postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit postableRule.RuleCondition.TargetUnit = c.targetUnit + postableRule.RuleCondition.Thresholds = []ruletypes.RuleThreshold{ruletypes.NewBasicRuleThreshold(postableRule.AlertName, &c.target, nil, ruletypes.MatchType(c.matchType), ruletypes.CompareOp(c.compareOp), postableRule.RuleCondition.SelectedQuery, c.targetUnit, postableRule.RuleCondition.CompositeQuery.Unit)} postableRule.Annotations = map[string]string{ "description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})", "summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}", @@ -1679,6 +1681,7 @@ func TestThresholdRuleLogsLink(t *testing.T) { postableRule.RuleCondition.Target = &c.target postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit postableRule.RuleCondition.TargetUnit = c.targetUnit + postableRule.RuleCondition.Thresholds = []ruletypes.RuleThreshold{ruletypes.NewBasicRuleThreshold(postableRule.AlertName, &c.target, nil, ruletypes.MatchType(c.matchType), ruletypes.CompareOp(c.compareOp), postableRule.RuleCondition.SelectedQuery, c.targetUnit, postableRule.RuleCondition.CompositeQuery.Unit)} postableRule.Annotations = map[string]string{ "description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})", "summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}", @@ -1782,3 +1785,168 @@ func TestThresholdRuleShiftBy(t *testing.T) { assert.Equal(t, int64(10), params.CompositeQuery.BuilderQueries["A"].ShiftBy) } + +func TestMultipleThresholdRule(t *testing.T) { + postableRule := ruletypes.PostableRule{ + AlertName: "Mulitple threshold test", + AlertType: ruletypes.AlertTypeMetric, + RuleType: ruletypes.RuleTypeThreshold, + EvalWindow: ruletypes.Duration(5 * time.Minute), + Frequency: ruletypes.Duration(1 * time.Minute), + RuleCondition: &ruletypes.RuleCondition{ + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{ + Key: "signoz_calls_total", + }, + AggregateOperator: v3.AggregateOperatorSumRate, + DataSource: v3.DataSourceMetrics, + Expression: "A", + }, + }, + }, + }, + } + telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{}) + + cols := make([]cmock.ColumnType, 0) + cols = append(cols, cmock.ColumnType{Name: "value", Type: "Float64"}) + cols = append(cols, cmock.ColumnType{Name: "attr", Type: "String"}) + cols = append(cols, cmock.ColumnType{Name: "timestamp", Type: "String"}) + + cases := []struct { + targetUnit string + yAxisUnit string + values [][]interface{} + expectAlerts int + compareOp string + matchType string + target float64 + secondTarget float64 + summaryAny []string + }{ + { + targetUnit: "s", + yAxisUnit: "ns", + values: [][]interface{}{ + {float64(572588400), "attr", time.Now()}, // 0.57 seconds + {float64(572386400), "attr", time.Now().Add(1 * time.Second)}, // 0.57 seconds + {float64(300947400), "attr", time.Now().Add(2 * time.Second)}, // 0.3 seconds + {float64(299316000), "attr", time.Now().Add(3 * time.Second)}, // 0.3 seconds + {float64(66640400.00000001), "attr", time.Now().Add(4 * time.Second)}, // 0.06 seconds + }, + expectAlerts: 2, + compareOp: "1", // Above + matchType: "1", // Once + target: 1, // 1 second + secondTarget: .5, + summaryAny: []string{ + "observed metric value is 573 ms", + "observed metric value is 572 ms", + }, + }, + { + targetUnit: "ms", + yAxisUnit: "ns", + values: [][]interface{}{ + {float64(572588400), "attr", time.Now()}, // 572.58 ms + {float64(572386400), "attr", time.Now().Add(1 * time.Second)}, // 572.38 ms + {float64(300947400), "attr", time.Now().Add(2 * time.Second)}, // 300.94 ms + {float64(299316000), "attr", time.Now().Add(3 * time.Second)}, // 299.31 ms + {float64(66640400.00000001), "attr", time.Now().Add(4 * time.Second)}, // 66.64 ms + }, + expectAlerts: 6, + compareOp: "1", // Above + matchType: "1", // Once + target: 200, // 200 ms + secondTarget: 500, + summaryAny: []string{ + "observed metric value is 299 ms", + "the observed metric value is 573 ms", + "the observed metric value is 572 ms", + "the observed metric value is 301 ms", + }, + }, + { + targetUnit: "decgbytes", + yAxisUnit: "bytes", + values: [][]interface{}{ + {float64(2863284053), "attr", time.Now()}, // 2.86 GB + {float64(2863388842), "attr", time.Now().Add(1 * time.Second)}, // 2.86 GB + {float64(300947400), "attr", time.Now().Add(2 * time.Second)}, // 0.3 GB + {float64(299316000), "attr", time.Now().Add(3 * time.Second)}, // 0.3 GB + {float64(66640400.00000001), "attr", time.Now().Add(4 * time.Second)}, // 66.64 MB + }, + expectAlerts: 2, + compareOp: "1", // Above + matchType: "1", // Once + target: 200, // 200 GB + secondTarget: 2, // 2GB + summaryAny: []string{ + "observed metric value is 2.7 GiB", + "the observed metric value is 0.3 GB", + }, + }, + } + + logger := instrumentationtest.New().Logger() + + for idx, c := range cases { + rows := cmock.NewRows(cols, c.values) + // We are testing the eval logic after the query is run + // so we don't care about the query string here + queryString := "SELECT any" + telemetryStore.Mock(). + ExpectQuery(queryString). + WillReturnRows(rows) + postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp) + postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType) + postableRule.RuleCondition.Target = &c.target + postableRule.RuleCondition.CompositeQuery.Unit = c.yAxisUnit + postableRule.RuleCondition.TargetUnit = c.targetUnit + postableRule.RuleCondition.Thresholds = []ruletypes.RuleThreshold{ruletypes.NewBasicRuleThreshold("first_threshold", &c.target, nil, ruletypes.MatchType(c.matchType), ruletypes.CompareOp(c.compareOp), postableRule.RuleCondition.SelectedQuery, c.targetUnit, postableRule.RuleCondition.CompositeQuery.Unit), + ruletypes.NewBasicRuleThreshold("second_threshold", &c.secondTarget, nil, ruletypes.MatchType(c.matchType), ruletypes.CompareOp(c.compareOp), postableRule.RuleCondition.SelectedQuery, c.targetUnit, postableRule.RuleCondition.CompositeQuery.Unit), + } + postableRule.Annotations = map[string]string{ + "description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})", + "summary": "The rule threshold is set to {{$threshold}}, and the observed metric value is {{$value}}", + } + + options := clickhouseReader.NewOptions("", "", "archiveNamespace") + readerCache, err := cachetest.New(cache.Config{Provider: "memory", Memory: cache.Memory{TTL: DefaultFrequency}}) + require.NoError(t, err) + reader := clickhouseReader.NewReaderFromClickhouseConnection(options, nil, telemetryStore, prometheustest.New(instrumentationtest.New().Logger(), prometheus.Config{}), "", time.Duration(time.Second), readerCache) + rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger) + rule.TemporalityMap = map[string]map[v3.Temporality]bool{ + "signoz_calls_total": { + v3.Delta: true, + }, + } + if err != nil { + assert.NoError(t, err) + } + + retVal, err := rule.Eval(context.Background(), time.Now()) + if err != nil { + assert.NoError(t, err) + } + + assert.Equal(t, c.expectAlerts, retVal.(int), "case %d", idx) + if c.expectAlerts != 0 { + foundCount := 0 + for _, item := range rule.Active { + for _, summary := range c.summaryAny { + if strings.Contains(item.Annotations.Get("summary"), summary) { + foundCount++ + break + } + } + } + assert.Equal(t, c.expectAlerts, foundCount, "case %d", idx) + } + } +} diff --git a/pkg/types/ruletypes/alerting.go b/pkg/types/ruletypes/alerting.go index f76d5a3ec79d..2f1141bb83ef 100644 --- a/pkg/types/ruletypes/alerting.go +++ b/pkg/types/ruletypes/alerting.go @@ -3,6 +3,8 @@ package ruletypes import ( "encoding/json" "fmt" + "github.com/SigNoz/signoz/pkg/query-service/converter" + "math" "net/url" "sort" "strings" @@ -11,6 +13,7 @@ import ( "github.com/SigNoz/signoz/pkg/query-service/model" v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" "github.com/SigNoz/signoz/pkg/query-service/utils/labels" + qslabels "github.com/SigNoz/signoz/pkg/query-service/utils/labels" ) // this file contains common structs and methods used by @@ -103,6 +106,294 @@ const ( Last MatchType = "5" ) +type RuleThreshold interface { + Name() string + Target() float64 + RecoveryTarget() float64 + + MatchType() MatchType + CompareOp() CompareOp + + SelectedQuery() string + ShouldAlert(series v3.Series) (Sample, bool) +} + +type BasicRuleThreshold struct { + name string + target *float64 + targetUnit string + ruleUnit string + recoveryTarget *float64 + matchType MatchType + compareOp CompareOp + selectedQuery string +} + +func NewBasicRuleThreshold(name string, target *float64, recoveryTarget *float64, matchType MatchType, op CompareOp, selectedQuery string, targetUnit string, ruleUnit string) *BasicRuleThreshold { + return &BasicRuleThreshold{name: name, target: target, recoveryTarget: recoveryTarget, matchType: matchType, selectedQuery: selectedQuery, compareOp: op, targetUnit: targetUnit, ruleUnit: ruleUnit} +} + +func (b BasicRuleThreshold) Name() string { + return b.name +} + +func (b BasicRuleThreshold) Target() float64 { + unitConverter := converter.FromUnit(converter.Unit(b.targetUnit)) + // convert the target value to the y-axis unit + value := unitConverter.Convert(converter.Value{ + F: *b.target, + U: converter.Unit(b.targetUnit), + }, converter.Unit(b.ruleUnit)) + return value.F +} + +func (b BasicRuleThreshold) RecoveryTarget() float64 { + return *b.recoveryTarget +} + +func (b BasicRuleThreshold) MatchType() MatchType { + return b.matchType +} + +func (b BasicRuleThreshold) CompareOp() CompareOp { + return b.compareOp +} + +func (b BasicRuleThreshold) SelectedQuery() string { + return b.selectedQuery +} + +func removeGroupinSetPoints(series v3.Series) []v3.Point { + var result []v3.Point + for _, s := range series.Points { + if s.Timestamp >= 0 && !math.IsNaN(s.Value) && !math.IsInf(s.Value, 0) { + result = append(result, s) + } + } + return result +} + +func (b BasicRuleThreshold) ShouldAlert(series v3.Series) (Sample, bool) { + var shouldAlert bool + var alertSmpl Sample + var lbls qslabels.Labels + + for name, value := range series.Labels { + lbls = append(lbls, qslabels.Label{Name: name, Value: value}) + } + + lbls = append(lbls, qslabels.Label{Name: "threshold", Value: b.name}) + + series.Points = removeGroupinSetPoints(series) + + // nothing to evaluate + if len(series.Points) == 0 { + return alertSmpl, false + } + + switch b.MatchType() { + case AtleastOnce: + // If any sample matches the condition, the rule is firing. + if b.CompareOp() == ValueIsAbove { + for _, smpl := range series.Points { + if smpl.Value > b.Target() { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + shouldAlert = true + break + } + } + } else if b.CompareOp() == ValueIsBelow { + for _, smpl := range series.Points { + if smpl.Value < b.Target() { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + shouldAlert = true + break + } + } + } else if b.CompareOp() == ValueIsEq { + for _, smpl := range series.Points { + if smpl.Value == b.Target() { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + shouldAlert = true + break + } + } + } else if b.CompareOp() == ValueIsNotEq { + for _, smpl := range series.Points { + if smpl.Value != b.Target() { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + shouldAlert = true + break + } + } + } else if b.CompareOp() == ValueOutsideBounds { + for _, smpl := range series.Points { + if math.Abs(smpl.Value) >= b.Target() { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + shouldAlert = true + break + } + } + } + case AllTheTimes: + // If all samples match the condition, the rule is firing. + shouldAlert = true + alertSmpl = Sample{Point: Point{V: b.Target()}, Metric: lbls} + if b.CompareOp() == ValueIsAbove { + for _, smpl := range series.Points { + if smpl.Value <= b.Target() { + shouldAlert = false + break + } + } + // use min value from the series + if shouldAlert { + var minValue float64 = math.Inf(1) + for _, smpl := range series.Points { + if smpl.Value < minValue { + minValue = smpl.Value + } + } + alertSmpl = Sample{Point: Point{V: minValue}, Metric: lbls} + } + } else if b.CompareOp() == ValueIsBelow { + for _, smpl := range series.Points { + if smpl.Value >= b.Target() { + shouldAlert = false + break + } + } + if shouldAlert { + var maxValue float64 = math.Inf(-1) + for _, smpl := range series.Points { + if smpl.Value > maxValue { + maxValue = smpl.Value + } + } + alertSmpl = Sample{Point: Point{V: maxValue}, Metric: lbls} + } + } else if b.CompareOp() == ValueIsEq { + for _, smpl := range series.Points { + if smpl.Value != b.Target() { + shouldAlert = false + break + } + } + } else if b.CompareOp() == ValueIsNotEq { + for _, smpl := range series.Points { + if smpl.Value == b.Target() { + shouldAlert = false + break + } + } + // use any non-inf or nan value from the series + if shouldAlert { + for _, smpl := range series.Points { + if !math.IsInf(smpl.Value, 0) && !math.IsNaN(smpl.Value) { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + break + } + } + } + } else if b.CompareOp() == ValueOutsideBounds { + for _, smpl := range series.Points { + if math.Abs(smpl.Value) < b.Target() { + alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls} + shouldAlert = false + break + } + } + } + case OnAverage: + // If the average of all samples matches the condition, the rule is firing. + var sum, count float64 + for _, smpl := range series.Points { + if math.IsNaN(smpl.Value) || math.IsInf(smpl.Value, 0) { + continue + } + sum += smpl.Value + count++ + } + avg := sum / count + alertSmpl = Sample{Point: Point{V: avg}, Metric: lbls} + if b.CompareOp() == ValueIsAbove { + if avg > b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsBelow { + if avg < b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsEq { + if avg == b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsNotEq { + if avg != b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueOutsideBounds { + if math.Abs(avg) >= b.Target() { + shouldAlert = true + } + } + case InTotal: + // If the sum of all samples matches the condition, the rule is firing. + var sum float64 + + for _, smpl := range series.Points { + if math.IsNaN(smpl.Value) || math.IsInf(smpl.Value, 0) { + continue + } + sum += smpl.Value + } + alertSmpl = Sample{Point: Point{V: sum}, Metric: lbls} + if b.CompareOp() == ValueIsAbove { + if sum > b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsBelow { + if sum < b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsEq { + if sum == b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsNotEq { + if sum != b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueOutsideBounds { + if math.Abs(sum) >= b.Target() { + shouldAlert = true + } + } + case Last: + // If the last sample matches the condition, the rule is firing. + shouldAlert = false + alertSmpl = Sample{Point: Point{V: series.Points[len(series.Points)-1].Value}, Metric: lbls} + if b.CompareOp() == ValueIsAbove { + if series.Points[len(series.Points)-1].Value > b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsBelow { + if series.Points[len(series.Points)-1].Value < b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsEq { + if series.Points[len(series.Points)-1].Value == b.Target() { + shouldAlert = true + } + } else if b.CompareOp() == ValueIsNotEq { + if series.Points[len(series.Points)-1].Value != b.Target() { + shouldAlert = true + } + } + } + return alertSmpl, shouldAlert +} + type RuleCondition struct { CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"` CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"` @@ -116,6 +407,7 @@ type RuleCondition struct { SelectedQuery string `json:"selectedQueryName,omitempty"` RequireMinPoints bool `yaml:"requireMinPoints,omitempty" json:"requireMinPoints,omitempty"` RequiredNumPoints int `yaml:"requiredNumPoints,omitempty" json:"requiredNumPoints,omitempty"` + Thresholds []RuleThreshold `yaml:"thresholds,omitempty" json:"thresholds,omitempty"` } func (rc *RuleCondition) GetSelectedQueryName() string { diff --git a/pkg/types/ruletypes/api_params.go b/pkg/types/ruletypes/api_params.go index b6be66d93149..2ed3bb07b6a7 100644 --- a/pkg/types/ruletypes/api_params.go +++ b/pkg/types/ruletypes/api_params.go @@ -140,6 +140,10 @@ func ParseIntoRule(initRule PostableRule, content []byte, kind RuleDataKind) (*P return nil, err } + //added alerts v2 fields + rule.RuleCondition.Thresholds = append(rule.RuleCondition.Thresholds, + NewBasicRuleThreshold(rule.AlertName, rule.RuleCondition.Target, nil, rule.RuleCondition.MatchType, rule.RuleCondition.CompareOp, rule.RuleCondition.SelectedQuery, rule.RuleCondition.TargetUnit, rule.RuleCondition.CompositeQuery.Unit)) + return rule, nil }