package notify
import (
"bytes"
"fmt"
"html/template"
"math/rand"
"path"
"strconv"
"strings"
"time"
"github.com/didi/nightingale/v4/src/common/address"
"github.com/didi/nightingale/v4/src/common/dataobj"
"github.com/didi/nightingale/v4/src/common/slice"
"github.com/didi/nightingale/v4/src/models"
"github.com/didi/nightingale/v4/src/modules/server/config"
"github.com/didi/nightingale/v4/src/modules/server/cron"
"github.com/didi/nightingale/v4/src/modules/server/redisc"
"github.com/toolkits/pkg/file"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/net/httplib"
)
func DoNotify(isUpgrade bool, events ...*models.Event) {
cnt := len(events)
if cnt == 0 {
return
}
userIds := events[cnt-1].RecvUserIDs
prio := fmt.Sprintf("p%v", events[cnt-1].Priority)
eventType := events[cnt-1].EventType
hashId := strconv.FormatUint(events[cnt-1].HashId, 10)
workGroups := events[cnt-1].WorkGroups
content, mailContent := genContent(isUpgrade, events)
subject := genSubject(isUpgrade, events)
if len(workGroups) > 0 {
go send2Ticket(content, subject, hashId, events[cnt-1].Priority, eventType, workGroups)
}
if len(userIds) == 0 {
return
}
users, err := models.UserGetByIds(userIds)
if err != nil {
logger.Errorf("notify failed, get user by id failed, events: %+v, err: %v", events, err)
return
}
notifyTypes := config.Config.Monapi.Notify[prio]
for i := 0; i < len(notifyTypes); i++ {
var err error
switch notifyTypes[i] {
case "voice":
if events[0].EventType == models.ALERT {
tos := []string{}
for j := 0; j < len(users); j++ {
tos = append(tos, users[j].Phone)
}
err = send(slice.Set(tos), events[0].Sname, "", "voice")
}
case "sms":
tos := []string{}
for j := 0; j < len(users); j++ {
tos = append(tos, users[j].Phone)
}
err = send(slice.Set(tos), content, "", "sms")
case "mail":
tos := []string{}
for j := 0; j < len(users); j++ {
tos = append(tos, users[j].Email)
}
err = send(slice.Set(tos), mailContent, subject, "mail")
case "im":
tos := []string{}
for j := 0; j < len(users); j++ {
tos = append(tos, users[j].Im)
}
err = send(slice.Set(tos), content, "", "im")
default:
logger.Errorf("not support %s to send notify, events: %+v", notifyTypes[i], events)
}
if err != nil {
logger.Errorf("send %s users:%+v content:%s err:%v", notifyTypes[i], users, content, err)
}
}
}
func genContent(isUpgrade bool, events []*models.Event) (string, string) {
cnt := len(events)
if cnt == 0 {
return "", ""
}
cfg := config.Config.Monapi
metricList := []string{}
detail, err := events[cnt-1].GetEventDetail()
if err != nil {
logger.Errorf("get event detail failed, event: %+v, err: %v", events[cnt-1], err)
} else {
for i := 0; i < len(detail); i++ {
metricList = append(metricList, detail[i].Metric)
}
}
resources := getResources(events)
metric := strings.Join(metricList, ",")
status := genStatus(events)
sname := events[cnt-1].Sname
endpoint := genEndpoint(events)
name, note := genNameAndNoteByResources(resources)
tags := genTags(events)
value := events[cnt-1].Value
info := events[cnt-1].Info
etime := genEtime(events)
slink := fmt.Sprintf(cfg.Link.Stra, events[cnt-1].Sid)
elink := fmt.Sprintf(cfg.Link.Event, events[cnt-1].Id)
clink := ""
curNodePath := events[cnt-1].CurNodePath
if events[0].EventType == models.ALERT {
clink = genClaimLink(events)
}
smsContent := ""
mailContent := ""
// 获取设备挂载的节点,放到告警信息里发出来,这样可以方便知道设备是属于哪个服务的
bindings, err := HostBindingsForMon(getEndpoint(events))
if err != nil {
bindings = []string{err.Error()}
}
isAlert := false
hasClaim := false
isMachineDep := false
if events[0].EventType == models.ALERT {
isAlert = true
}
if clink != "" {
hasClaim = true
}
if events[0].Category == 1 {
isMachineDep = true
}
values := map[string]interface{}{
"IsAlert": isAlert,
"IsMachineDep": isMachineDep,
"Status": status,
"Sname": sname,
"Endpoint": endpoint,
"Name": name,
"Note": note,
"CurNodePath": curNodePath,
"Metric": metric,
"Tags": tags,
"Value": value,
"Info": info,
"Etime": etime,
"Elink": elink,
"Slink": slink,
"HasClaim": hasClaim,
"Clink": clink,
"IsUpgrade": isUpgrade,
"Bindings": bindings,
}
// 生成告警邮件
fp := path.Join(file.SelfDir(), "etc", "mail.tpl")
t, err := template.ParseFiles(fp)
if err != nil {
logger.Errorf("InternalServerError: cannot parse %s %v", fp, err)
mailContent = fmt.Sprintf("InternalServerError: cannot parse %s %v", fp, err)
} else {
var body bytes.Buffer
err = t.Execute(&body, values)
if err != nil {
logger.Errorf("InternalServerError: %v", err)
mailContent = fmt.Sprintf("InternalServerError: %v", err)
} else {
mailContent += body.String()
}
}
// 生成告警短信,短信和IM复用一个内容模板
fp = path.Join(file.SelfDir(), "etc", "sms.tpl")
t, err = template.New("sms.tpl").Funcs(template.FuncMap{
"unescaped": func(str string) interface{} { return template.HTML(str) },
"urlconvert": func(str string) interface{} { return template.URL(str) },
}).ParseFiles(fp)
if err != nil {
logger.Errorf("InternalServerError: cannot parse %s %v", fp, err)
smsContent = fmt.Sprintf("InternalServerError: cannot parse %s %v", fp, err)
} else {
var body bytes.Buffer
err = t.Execute(&body, values)
if err != nil {
logger.Errorf("InternalServerError: %v", err)
smsContent = fmt.Sprintf("InternalServerError: %v", err)
} else {
smsContent += body.String()
}
}
return smsContent, mailContent
}
func genClaimLink(events []*models.Event) string {
for i := 0; i < len(events); i++ {
eventCur, err := models.EventCurGet("hashid", events[i].HashId)
if err != nil {
logger.Errorf("get event_cur failed, err: %v, event: %+v", err, events[i])
continue
}
if eventCur == nil {
continue
}
return fmt.Sprintf(config.Config.Monapi.Link.Claim, eventCur.Id)
}
return ""
}
func genSubject(isUpgrade bool, events []*models.Event) string {
cnt := len(events)
subject := ""
if isUpgrade {
subject = "[报警已升级]" + subject
}
if cnt > 1 {
subject += fmt.Sprintf("[P%d 聚合%s]%s", events[cnt-1].Priority, models.EventTypeMap[events[cnt-1].EventType], events[cnt-1].Sname)
} else {
subject += fmt.Sprintf("[P%d %s]%s", events[cnt-1].Priority, models.EventTypeMap[events[cnt-1].EventType], events[cnt-1].Sname)
}
return subject + " - " + genEndpoint(events)
}
func genStatus(events []*models.Event) string {
cnt := len(events)
status := fmt.Sprintf("P%d %s", events[cnt-1].Priority, models.EventTypeMap[events[cnt-1].EventType])
if cnt > 1 {
status += "(聚合)"
}
return status
}
func HostBindingsForMon(endpointList []string) ([]string, error) {
var list []string
resouceIds, err := models.ResourceIdsByIdents(endpointList)
if err != nil {
return list, err
}
nodeIds, err := models.NodeIdsGetByResIds(resouceIds)
if err != nil {
return list, err
}
for _, id := range nodeIds {
node, err := models.NodeGet("id=?", id)
if err != nil {
return list, err
}
if node == nil {
continue
}
list = append(list, node.Path)
}
return list, nil
}
func getResources(events []*models.Event) []models.Resource {
idents := []string{}
for i := 0; i < len(events); i++ {
idents = append(idents, events[i].Endpoint)
}
resources, err := models.ResourcesByIdents(idents)
if err != nil {
logger.Errorf("get resources by idents failed : %v", err)
}
return resources
}
func genNameAndNoteByResources(resources []models.Resource) (name, note string) {
names := []string{}
notes := []string{}
for i := 0; i < len(resources); i++ {
names = append(names, resources[i].Name)
notes = append(notes, resources[i].Note)
}
names = slice.Set(names)
notes = slice.Set(notes)
if len(resources) == 1 {
if len(names) > 0 {
name = names[0]
}
if len(notes) > 0 {
note = notes[0]
}
return
}
name = fmt.Sprintf("%s(%v)", strings.Join(names, ","), len(names))
note = fmt.Sprintf("%s(%v)", strings.Join(notes, ","), len(notes))
return
}
func getEndpoint(events []*models.Event) []string {
endpointList := []string{}
for i := 0; i < len(events); i++ {
endpointList = append(endpointList, events[i].Endpoint)
}
endpointList = slice.Set(endpointList)
return endpointList
}
func genEndpoint(events []*models.Event) string {
endpointList := []string{}
for i := 0; i < len(events); i++ {
endpointList = append(endpointList, events[i].Endpoint)
}
endpointList = slice.Set(endpointList)
if len(endpointList) == 1 {
return endpointList[0]
}
return fmt.Sprintf("%s(%v)", strings.Join(endpointList, ","), len(endpointList))
}
func genTags(events []*models.Event) string {
tagsMap := make(map[string][]string)
for i := 0; i < len(events); i++ {
detail, err := events[i].GetEventDetail()
if err != nil {
continue
}
if len(detail) > 0 {
for k, v := range detail[0].Tags {
if !slice.InSlice(v, tagsMap[k]) {
tagsMap[k] = append(tagsMap[k], v)
}
}
}
}
tagsList := []string{}
for k, v := range tagsMap {
valueString := strings.Join(v, ",")
if len(v) > 1 {
valueString = "[" + valueString + "]"
}
tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, valueString))
}
return strings.Join(tagsList, ",")
}
func genEtime(events []*models.Event) string {
if len(events) == 1 {
return models.ParseEtime(events[0].Etime)
}
stime := events[0].Etime
etime := events[0].Etime
for i := 1; i < len(events); i++ {
if events[i].Etime < stime {
stime = events[i].Etime
}
if events[i].Etime > etime {
etime = events[i].Etime
}
}
if stime == etime {
return models.ParseEtime(stime)
}
return models.ParseEtime(stime) + "~" + models.ParseEtime(etime)
}
func send(tos []string, content, subject, notifyType string) error {
var message dataobj.Message
if tos == nil || len(tos) == 0 {
return fmt.Errorf("tos is empty")
}
message.Tos = tos
message.Content = strings.TrimSpace(content)
if message.Content == "" {
return fmt.Errorf("content is blank")
}
if notifyType == "email" {
message.Subject = strings.TrimSpace(message.Subject)
if message.Subject == "" {
return fmt.Errorf("subject is blank")
}
}
switch notifyType {
case "voice":
redisc.Write(&message, cron.VOICE_QUEUE_NAME)
case "sms":
redisc.Write(&message, cron.SMS_QUEUE_NAME)
case "email":
redisc.Write(&message, cron.MAIL_QUEUE_NAME)
case "im":
redisc.Write(&message, cron.IM_QUEUE_NAME)
}
logger.Infof("write %s message:%+v", message, notifyType)
return nil
}
type TicketInfo struct {
Title string `json:"title"`
Typ string `json:"typ"`
MonitorTrace string `json:"monitorTrace"`
Level int `json:"level"`
Content string `json:"content"`
MultiQueueIds []int `json:"multiQueueIds"`
AlertType int `json:"alertType"`
}
type TicketReq struct {
Info TicketInfo `json:"ticketInfo"`
}
func send2Ticket(content, subject, hashId string, prio int, eventType string, workGroups []int) {
if !config.Config.Monapi.TicketEnabled {
return
}
addrs := address.GetHTTPAddresses("ticket")
perm := rand.Perm(len(addrs))
for i := range perm {
url := fmt.Sprintf("%s/v1/ticket/monitor/event?systemName=monitor", addrs[perm[i]])
if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) {
url = "http://" + url
}
alertType := 2
if eventType == "alert" {
alertType = 1
}
info := TicketInfo{
Title: subject,
Typ: "extern_sync",
MonitorTrace: hashId,
Level: prio,
Content: content,
MultiQueueIds: workGroups,
AlertType: alertType,
}
req := TicketReq{
Info: info,
}
res, code, err := httplib.PostJSON(url, time.Second*5, req, map[string]string{"X-Srv-Token": "ticket-builtin-token"})
if err != nil {
logger.Errorf("call ticket api failed, server: %v, data: %+v, err: %v, resp:%v, status code:%d", url, req, err, string(res), code)
return
}
if code != 200 {
logger.Errorf("call ticket api failed, server: %v, data: %+v, resp:%v, code:%d", url, req, string(res), code)
return
}
if err == nil {
break
}
}
return
}