refactor: sort event tags & change extra

src/dataobj/event.go

@@ -20,6 +20,24 @@ type History struct {
 	Metric      string            `json:"metric"`         // 指标名
 	Tags        map[string]string `json:"tags,omitempty"` // endpoint/counter
 	Granularity int               `json:"-"`              // alarm补齐数据时需要
-	Points      []*RRDData        `json:"points"`         // 现场值
+	Points      []*HistoryData    `json:"points"`         // 现场值
+}
+
+type HistoryData struct {
+	Timestamp int64     `json:"timestamp"`
+	Value     JsonFloat `json:"value"`
 	Extra     string    `json:"extra"`
 }
+
+func RRDData2HistoryData(datas []*RRDData) []*HistoryData {
+	historyDatas := make([]*HistoryData, len(datas))
+
+	for i := range datas {
+		historyData := &HistoryData{
+			Timestamp: datas[i].Timestamp,
+			Value:     datas[i].Value,
+		}
+		historyDatas[i] = historyData
+	}
+	return historyDatas
+}

src/dataobj/judge.go

@@ -28,9 +28,3 @@ func (j *JudgeItem) PrimaryKey() string {
 func (j *JudgeItem) MD5() string {
 	return gstr.MD5(str.PK(strconv.FormatInt(j.Sid, 16), j.Endpoint, j.Metric, str.SortedTags(j.TagsMap)))
 }
-
-//告警现场的值
-type HistoryData struct {
-	Timestamp int64   `json:"timestamp"`
-	Value     float64 `json:"value"`
-}

src/model/event.go

@@ -39,12 +39,12 @@ type EventDetail struct {
 	Tags       map[string]string   `json:"tags"`
 	Points     []*EventDetailPoint `json:"points"`
 	PredPoints []*EventDetailPoint `json:"pred_points,omitempty"` // 预测值, 预测值不为空时, 现场值对应的是实际值
-	Extra      string              `json:"extra"`
 }
 
 type EventDetailPoint struct {
 	Timestamp int64   `json:"timestamp"`
 	Value     float64 `json:"value"`
+	Extra     string  `json:"extra"`
 }
 
 type EventAlertUpgrade struct {

src/model/event_cur.go

@@ -232,7 +232,7 @@ func EventCurGet(col string, value interface{}) (*EventCur, error) {
 }
 
 func (e *EventCur) EventIgnore() error {
-	_, err := DB["mon"].Exec("update event_cur set ignore_alert=1 where id=?", e.Id)
+	_, err := DB["mon"].Where("id=?", e.Id).Delete(new(EventCur))
 	return err
 }
 

src/modules/judge/cache/index.go

@@ -71,7 +71,7 @@ func (i *IndexMap) Clean() {
 		}
 
 		for key, series := range index {
-			if now-series.TS > 3600 {
+			if now-series.TS > 300 {
 				delete(i.Data[id], key)
 			}
 		}

src/modules/judge/cache/linkedlist.go

@@ -54,22 +54,22 @@ func (ll *SafeLinkedList) PushFrontAndMaintain(v *dataobj.JudgeItem, maxCount in
 
 // @param limit 至多返回这些，如果不够，有多少返回多少
 // @return bool isEnough
-func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) {
+func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.HistoryData, bool) {
 	if limit < 1 {
 		// 其实limit不合法，此处也返回false吧，上层代码要注意
 		// 因为false通常使上层代码进入异常分支，这样就统一了
-		return []*dataobj.RRDData{}, false
+		return []*dataobj.HistoryData{}, false
 	}
 
 	size := ll.Len()
 	if size == 0 {
-		return []*dataobj.RRDData{}, false
+		return []*dataobj.HistoryData{}, false
 	}
 
 	firstElement := ll.Front()
 	firstItem := firstElement.Value.(*dataobj.JudgeItem)
 
-	var vs []*dataobj.RRDData
+	var vs []*dataobj.HistoryData
 	isEnough := true
 
 	judgeType := firstItem.DsType[0]
@@ -79,15 +79,21 @@ func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) {
 			limit = size
 			isEnough = false
 		}
-		vs = make([]*dataobj.RRDData, limit)
-		vs[0] = &dataobj.RRDData{Timestamp: firstItem.Timestamp, Value: dataobj.JsonFloat(firstItem.Value)}
+		vs = make([]*dataobj.HistoryData, limit)
+		vs[0] = &dataobj.HistoryData{
+			Timestamp: firstItem.Timestamp,
+			Value:     dataobj.JsonFloat(firstItem.Value),
+			Extra:     firstItem.Extra,
+		}
+
 		i := 1
 		currentElement := firstElement
 		for i < limit {
 			nextElement := currentElement.Next()
-			vs[i] = &dataobj.RRDData{
+			vs[i] = &dataobj.HistoryData{
 				Timestamp: nextElement.Value.(*dataobj.JudgeItem).Timestamp,
 				Value:     dataobj.JsonFloat(nextElement.Value.(*dataobj.JudgeItem).Value),
+				Extra:     nextElement.Value.(*dataobj.JudgeItem).Extra,
 			}
 			i++
 			currentElement = nextElement

src/modules/judge/judge/func.go

@@ -8,7 +8,7 @@ import (
 )
 
 type Function interface {
-	Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool)
+	Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool)
 }
 
 type MaxFunction struct {
@@ -18,7 +18,7 @@ type MaxFunction struct {
 	RightValue float64
 }
 
-func (this MaxFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this MaxFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -42,7 +42,7 @@ type MinFunction struct {
 	RightValue float64
 }
 
-func (this MinFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this MinFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -66,7 +66,7 @@ type AllFunction struct {
 	RightValue float64
 }
 
-func (this AllFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this AllFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -90,7 +90,7 @@ type SumFunction struct {
 	RightValue float64
 }
 
-func (this SumFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this SumFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -112,7 +112,7 @@ type AvgFunction struct {
 	RightValue float64
 }
 
-func (this AvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this AvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -135,7 +135,7 @@ type DiffFunction struct {
 }
 
 // 只要有一个点的diff触发阈值，就报警
-func (this DiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this DiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -163,7 +163,7 @@ type PDiffFunction struct {
 	RightValue float64
 }
 
-func (this PDiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this PDiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -193,7 +193,7 @@ type HappenFunction struct {
 	RightValue float64
 }
 
-func (this HappenFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this HappenFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	for n, i := 0, 0; i < len(vs); i++ {
 		if checkIsTriggered(vs[i].Value, this.Operator, this.RightValue) {
 			n++
@@ -211,7 +211,7 @@ type NodataFunction struct {
 	Function
 }
 
-func (this NodataFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this NodataFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	for _, value := range vs {
 		if !math.IsNaN(float64(value.Value)) {
 			return value.Value, false
@@ -228,7 +228,7 @@ type CAvgAbsFunction struct {
 	CompareValue float64
 }
 
-func (this CAvgAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this CAvgAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -253,7 +253,7 @@ type CAvgFunction struct {
 	CompareValue float64
 }
 
-func (this CAvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this CAvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -277,7 +277,7 @@ type CAvgRateAbsFunction struct {
 	CompareValue float64
 }
 
-func (this CAvgRateAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this CAvgRateAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}
@@ -302,7 +302,7 @@ type CAvgRateFunction struct {
 	CompareValue float64
 }
 
-func (this CAvgRateFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func (this CAvgRateFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	if len(vs) < this.Limit {
 		return
 	}

src/modules/judge/judge/judge.go

@@ -62,7 +62,7 @@ func ToJudge(historyMap *cache.JudgeItemMap, key string, val *dataobj.JudgeItem,
 	Judge(stra, stra.Exprs, historyData, val, now, history, "", "", "", []bool{})
 }
 
-func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
+func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
 	stats.Counter.Set("running", 1)
 
 	if len(exps) < 1 {
@@ -86,10 +86,6 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
 		Granularity: int(firstItem.Step),
 		Points:      historyData,
 	}
-	if len(history) == 0 {
-		//只有第一个指标是push的模式，可以获取到extra字段
-		h.Extra = firstItem.Extra
-	}
 	history = append(history, h)
 
 	defer func() {
@@ -135,7 +131,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
 					Tags:     "",
 					DsType:   "GAUGE",
 				}
-				Judge(stra, exps[1:], []*dataobj.RRDData{}, judgeItem, now, history, info, value, extra, status)
+				Judge(stra, exps[1:], []*dataobj.HistoryData{}, judgeItem, now, history, info, value, extra, status)
 				return
 			}
 
@@ -143,7 +139,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
 				firstItem.Endpoint = respData[i].Endpoint
 				firstItem.Tags = getTags(respData[i].Counter)
 				firstItem.Step = respData[i].Step
-				Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status)
+				Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status)
 			}
 
 		} else {
@@ -162,13 +158,13 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
 				firstItem.Endpoint = respData[i].Endpoint
 				firstItem.Tags = getTags(respData[i].Counter)
 				firstItem.Step = respData[i].Step
-				Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status)
+				Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status)
 			}
 		}
 	}
 }
 
-func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.RRDData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) {
+func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.HistoryData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) {
 	straFunc := exp.Func
 
 	straParam := []interface{}{}
@@ -298,7 +294,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
 	stats.Counter.Set("get.index", 1)
 	indexsData, err := query.Xclude(req)
 	if err != nil {
-		return reqs, err
+		logger.Warning("get index err:", err)
 	}
 
 	lostSeries := []cache.Series{}
@@ -345,6 +341,10 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
 	}
 
 	seriess := cache.SeriesMap.Get(stra.Id)
+	if len(seriess) == 0 && err != nil {
+		return reqs, err
+	}
+
 	step := 0
 	if len(seriess) > 1 {
 		step = seriess[0].Step
@@ -389,7 +389,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
 	return reqs, nil
 }
 
-func sendEventIfNeed(historyData []*dataobj.RRDData, status []bool, event *dataobj.Event, stra *model.Stra) {
+func sendEventIfNeed(historyData []*dataobj.HistoryData, status []bool, event *dataobj.Event, stra *model.Stra) {
 	isTriggered := true
 	for _, s := range status {
 		isTriggered = isTriggered && s

src/modules/judge/judge/nodata.go

@@ -51,7 +51,7 @@ func nodataJudge() {
 				}
 
 				nodataJob.Acquire()
-				go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.RRDData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
+				go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.HistoryData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
 			}
 			return
 		}
@@ -80,12 +80,12 @@ func nodataJudge() {
 			}
 
 			nodataJob.Acquire()
-			go AsyncJudge(nodataJob, stra, stra.Exprs, data.Values, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
+			go AsyncJudge(nodataJob, stra, stra.Exprs, dataobj.RRDData2HistoryData(data.Values), judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
 		}
 	}
 }
 
-func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
+func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
 	defer sema.Release()
 	Judge(stra, exps, historyData, firstItem, now, history, info, value, extra, status)
 }

src/modules/monapi/cron/event_consumer.go

@@ -220,10 +220,9 @@ func isInConverge(event *model.Event) bool {
 	return false
 }
 
-// 三种情况，不需要升级报警
+// 两种情况，不需要升级报警
 // 1，认领的报警不需要升级
-// 2，忽略的报警不需要升级
-// 3，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注
+// 2，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注
 func needUpgrade(event *model.Event) bool {
 	alertUpgradeKey := PrefixAlertUpgrade + fmt.Sprint(event.HashId)
 	eventAlertKey := PrefixAlertTime + fmt.Sprint(event.HashId)
@@ -282,11 +281,6 @@ func needUpgrade(event *model.Event) bool {
 		return false
 	}
 
-	// 告警已经忽略了
-	if eventCur.IgnoreAlert == 1 {
-		return false
-	}
-
 	// 告警之后，比如30分钟没有处理，就需要升级，那首先得知道首次告警时间
 	if !redisc.HasKey(eventAlertKey) {
 		err := redisc.SetWithTTL(eventAlertKey, now, 30*24*3600)

src/modules/monapi/http/routes/event.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/toolkits/pkg/errors"
 
+	"github.com/didi/nightingale/src/dataobj"
 	"github.com/didi/nightingale/src/model"
 	jsoniter "github.com/json-iterator/go"
 )
@@ -79,9 +80,9 @@ func eventCurGets(c *gin.Context) {
 		err = json.Unmarshal([]byte(events[i].Detail), &detail)
 		errors.Dangerous(err)
 
-		tagsList := []string{}
-		for k, v := range detail[0].Tags {
-			tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v))
+		var tags string
+		if len(detail) > 0 {
+			tags = dataobj.SortedTags(detail[0].Tags)
 		}
 
 		alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade)
@@ -106,7 +107,7 @@ func eventCurGets(c *gin.Context) {
 			Etime:       events[i].Etime,
 			Value:       events[i].Value,
 			Info:        events[i].Info,
-			Tags:        strings.Join(tagsList, ","),
+			Tags:        tags,
 			Created:     events[i].Created,
 			Nid:         events[i].Nid,
 			Users:       users,
@@ -164,9 +165,9 @@ func eventHisGets(c *gin.Context) {
 		err = json.Unmarshal([]byte(events[i].Detail), &detail)
 		errors.Dangerous(err)
 
-		tagsList := []string{}
-		for k, v := range detail[0].Tags {
-			tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v))
+		var tags string
+		if len(detail) > 0 {
+			tags = dataobj.SortedTags(detail[0].Tags)
 		}
 
 		alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade)
@@ -191,7 +192,7 @@ func eventHisGets(c *gin.Context) {
 			Etime:       events[i].Etime,
 			Value:       events[i].Value,
 			Info:        events[i].Info,
-			Tags:        strings.Join(tagsList, ","),
+			Tags:        tags,
 			Created:     events[i].Created,
 			Nid:         events[i].Nid,
 			Users:       users,