fix: do not crash service on panic during rule eval (#7514)

This commit is contained in:
Srikanth Chekuri 2025-04-03 13:13:58 +05:30 committed by GitHub
parent 8df77c9221
commit b40ca4baf3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 0 deletions

View File

@ -315,6 +315,13 @@ func (g *PromRuleTask) CopyState(fromTask Task) error {
// Eval runs a single evaluation cycle in which all rules are evaluated sequentially.
func (g *PromRuleTask) Eval(ctx context.Context, ts time.Time) {
defer func() {
if r := recover(); r != nil {
zap.L().Error("panic during promql rule evaluation", zap.Any("panic", r))
}
}()
zap.L().Info("promql rule task", zap.String("name", g.name), zap.Time("eval started at", ts))
maintenance, err := g.ruleDB.GetAllPlannedMaintenance(ctx)

View File

@ -297,6 +297,12 @@ func (g *RuleTask) CopyState(fromTask Task) error {
// Eval runs a single evaluation cycle in which all rules are evaluated sequentially.
func (g *RuleTask) Eval(ctx context.Context, ts time.Time) {
defer func() {
if r := recover(); r != nil {
zap.L().Error("panic during threshold rule evaluation", zap.Any("panic", r))
}
}()
zap.L().Debug("rule task eval started", zap.String("name", g.name), zap.Time("start time", ts))
maintenance, err := g.ruleDB.GetAllPlannedMaintenance(ctx)