diff --git a/src/models/alert_cur_event.go b/src/models/alert_cur_event.go index 26c7f1f3a7ae325cada749155984766b51f346d6..138d4bad808f98b90ba1b3ca15f441e0117abba6 100644 --- a/src/models/alert_cur_event.go +++ b/src/models/alert_cur_event.go @@ -37,7 +37,8 @@ type AlertCurEvent struct { TagsMap map[string]string `json:"-" gorm:"-"` // for internal usage IsRecovered bool `json:"is_recovered" gorm:"-"` // for notify.py NotifyUsersObj []*User `json:"notify_users_obj" gorm:"-"` // for notify.py - LastEvalTime int64 `json:"last_eval_time" gorm:"-"` // for notify.py + LastEvalTime int64 `json:"last_eval_time" gorm:"-"` // for notify.py 上次计算的时间 + LastSentTime int64 `json:"last_sent_time" gorm:"-"` // 上次发送时间 } func (e *AlertCurEvent) TableName() string { diff --git a/src/server/engine/worker.go b/src/server/engine/worker.go index f67ec7b040043ff215d16982ae894113d951bce1..d119e21216b56d49ef22cc7844a5ed447fad7517 100644 --- a/src/server/engine/worker.go +++ b/src/server/engine/worker.go @@ -256,7 +256,6 @@ func (r RuleEval) judge(vectors []Vector) { event.NotifyChannelsJSON = r.rule.NotifyChannelsJSON event.NotifyGroups = r.rule.NotifyGroups event.NotifyGroupsJSON = r.rule.NotifyGroupsJSON - event.NotifyRepeatNext = now + int64(r.rule.NotifyRepeatStep*60) event.TargetIdent = string(targetIdent) event.TargetNote = targetNote event.TriggerValue = readableValue(vectors[i].Value) @@ -301,7 +300,7 @@ func (r RuleEval) handleNewEvent(event *models.AlertCurEvent) { _, has := r.pendings[event.Hash] if has { - r.pendings[event.Hash].LastEvalTime = event.TriggerTime + r.pendings[event.Hash].LastEvalTime = event.LastEvalTime } else { r.pendings[event.Hash] = event } @@ -313,19 +312,19 @@ func (r RuleEval) handleNewEvent(event *models.AlertCurEvent) { func (r RuleEval) fireEvent(event *models.AlertCurEvent) { if fired, has := r.fires[event.Hash]; has { + r.fires[event.Hash].LastEvalTime = event.LastEvalTime + if r.rule.NotifyRepeatStep == 0 { // 说明不想重复通知,那就直接返回了,nothing to do return } // 之前发送过告警了,这次是否要继续发送,要看是否过了通道静默时间 - if event.LastEvalTime > fired.LastEvalTime+int64(r.rule.NotifyRepeatStep)*60 { - r.fires[event.Hash] = event - pushEventToQueue(event) + if event.LastEvalTime > fired.LastSentTime+int64(r.rule.NotifyRepeatStep)*60 { + r.pushEventToQueue(event) } } else { - r.fires[event.Hash] = event - pushEventToQueue(event) + r.pushEventToQueue(event) } } @@ -372,12 +371,17 @@ func (r RuleEval) recoverRule(alertingKeys map[string]struct{}, now int64) { event.NotifyChannelsJSON = r.rule.NotifyChannelsJSON event.NotifyGroups = r.rule.NotifyGroups event.NotifyGroupsJSON = r.rule.NotifyGroupsJSON - pushEventToQueue(event) + r.pushEventToQueue(event) } } } -func pushEventToQueue(event *models.AlertCurEvent) { +func (r RuleEval) pushEventToQueue(event *models.AlertCurEvent) { + if !event.IsRecovered { + event.LastSentTime = event.LastEvalTime + r.fires[event.Hash] = event + } + promstat.CounterAlertsTotal.WithLabelValues(config.C.ClusterName).Inc() logEvent(event, "push_queue") if !EventQueue.PushFront(event) {