From 403cb5a6ad5f9d13a13200707f248ae932d82bdf Mon Sep 17 00:00:00 2001 From: Ulric Qin Date: Mon, 28 Feb 2022 23:50:02 +0800 Subject: [PATCH] not stable version --- etc/server.conf | 19 ++++++-- src/server/config/config.go | 35 ++++++++++----- src/server/engine/callback.go | 40 +---------------- src/server/engine/notify.go | 81 +++++++++++++++++++++-------------- src/server/poster/post.go | 42 ++++++++++++++++++ src/server/sender/dingtalk.go | 55 ++++++++++++++++++++++++ src/server/sender/feishu.go | 52 ++++++++++++++++++++++ src/server/sender/wecom.go | 41 ++++++++++++++++++ 8 files changed, 283 insertions(+), 82 deletions(-) create mode 100644 src/server/poster/post.go create mode 100644 src/server/sender/dingtalk.go create mode 100644 src/server/sender/feishu.go create mode 100644 src/server/sender/wecom.go diff --git a/etc/server.conf b/etc/server.conf index 6e96e65b..12daddd5 100644 --- a/etc/server.conf +++ b/etc/server.conf @@ -54,17 +54,30 @@ IP = "" # unit ms Interval = 1000 +[SMTP] +Host = "smtp.163.com" +Port = 994 +User = "username" +Pass = "password" +From = "username@163.com" +InsecureSkipVerify = true + [Alerting] -NotifyScriptPath = "./etc/script/notify.py" -NotifyConcurrency = 100 TemplatesDir = "./etc/template" +NotifyConcurrency = 100 + +[Alerting.CallScript] +# built in sending capability in go code +# so, no need enable script sender +Enable = false +ScriptPath = "./etc/script/notify.py" [Alerting.RedisPub] Enable = false # complete redis key: ${ChannelPrefix} + ${Cluster} ChannelPrefix = "/alerts/" -[Alerting.GlobalCallback] +[Alerting.Webhook] Enable = false Url = "http://a.com/n9e/callback" BasicAuthUser = "" diff --git a/src/server/config/config.go b/src/server/config/config.go index 07b7a7e4..4ba151fe 100644 --- a/src/server/config/config.go +++ b/src/server/config/config.go @@ -79,16 +79,16 @@ func MustLoad(fpaths ...string) { C.Heartbeat.Endpoint = fmt.Sprintf("%s:%d", C.Heartbeat.IP, C.HTTP.Port) C.Alerting.RedisPub.ChannelKey = C.Alerting.RedisPub.ChannelPrefix + C.ClusterName - if C.Alerting.GlobalCallback.Enable { - if C.Alerting.GlobalCallback.Timeout == "" { - C.Alerting.GlobalCallback.TimeoutDuration = time.Second * 5 + if C.Alerting.Webhook.Enable { + if C.Alerting.Webhook.Timeout == "" { + C.Alerting.Webhook.TimeoutDuration = time.Second * 5 } else { - dur, err := time.ParseDuration(C.Alerting.GlobalCallback.Timeout) + dur, err := time.ParseDuration(C.Alerting.Webhook.Timeout) if err != nil { - fmt.Println("failed to parse Alerting.GlobalCallback.Timeout") + fmt.Println("failed to parse Alerting.Webhook.Timeout") os.Exit(1) } - C.Alerting.GlobalCallback.TimeoutDuration = dur + C.Alerting.Webhook.TimeoutDuration = dur } } @@ -104,6 +104,7 @@ type Config struct { Log logx.Config HTTP httpx.Config BasicAuth gin.Accounts + SMTP SMTPConfig Heartbeat HeartbeatConfig Alerting Alerting NoData NoData @@ -123,12 +124,26 @@ type HeartbeatConfig struct { Endpoint string } +type SMTPConfig struct { + Host string + Port int + User string + Pass string + From string + InsecureSkipVerify bool +} + type Alerting struct { - NotifyScriptPath string - NotifyConcurrency int TemplatesDir string + NotifyConcurrency int + CallScript CallScript RedisPub RedisPub - GlobalCallback GlobalCallback + Webhook Webhook +} + +type CallScript struct { + Enable bool + ScriptPath string } type RedisPub struct { @@ -137,7 +152,7 @@ type RedisPub struct { ChannelKey string } -type GlobalCallback struct { +type Webhook struct { Enable bool Url string BasicAuthUser string diff --git a/src/server/engine/callback.go b/src/server/engine/callback.go index c77d0a03..17028037 100644 --- a/src/server/engine/callback.go +++ b/src/server/engine/callback.go @@ -1,10 +1,6 @@ package engine import ( - "bytes" - "encoding/json" - "io/ioutil" - "net/http" "strconv" "strings" "time" @@ -15,41 +11,9 @@ import ( "github.com/didi/nightingale/v5/src/pkg/ibex" "github.com/didi/nightingale/v5/src/server/config" "github.com/didi/nightingale/v5/src/server/memsto" + "github.com/didi/nightingale/v5/src/server/poster" ) -func PostJSON(url string, timeout time.Duration, v interface{}) (response []byte, code int, err error) { - var bs []byte - - bs, err = json.Marshal(v) - if err != nil { - return - } - - bf := bytes.NewBuffer(bs) - - client := http.Client{ - Timeout: timeout, - } - - req, err := http.NewRequest("POST", url, bf) - req.Header.Set("Content-Type", "application/json") - - var resp *http.Response - resp, err = client.Do(req) - if err != nil { - return - } - - code = resp.StatusCode - - if resp.Body != nil { - defer resp.Body.Close() - response, err = ioutil.ReadAll(resp.Body) - } - - return -} - func callback(event *models.AlertCurEvent) { urls := strings.Fields(event.Callbacks) for _, url := range urls { @@ -68,7 +32,7 @@ func callback(event *models.AlertCurEvent) { url = "http://" + url } - resp, code, err := PostJSON(url, 5*time.Second, event) + resp, code, err := poster.PostJSON(url, 5*time.Second, event) if err != nil { logger.Errorf("event_callback(rule_id=%d url=%s) fail, resp: %s, err: %v, code: %d", event.RuleId, url, string(resp), err, code) } else { diff --git a/src/server/engine/notify.go b/src/server/engine/notify.go index c921ebb0..bba59842 100644 --- a/src/server/engine/notify.go +++ b/src/server/engine/notify.go @@ -31,14 +31,14 @@ var fns = template.FuncMap{ "urlconvert": func(str string) interface{} { return template.URL(str) }, "timeformat": func(ts int64, pattern ...string) string { defp := "2006-01-02 15:04:05" - if pattern != nil && len(pattern) > 0 { + if len(pattern) > 0 { defp = pattern[0] } return time.Unix(ts, 0).Format(defp) }, "timestamp": func(pattern ...string) string { defp := "2006-01-02 15:04:05" - if pattern != nil && len(pattern) > 0 { + if len(pattern) > 0 { defp = pattern[0] } return time.Now().Format(defp) @@ -89,7 +89,7 @@ type Notice struct { Tpls map[string]string `json:"tpls"` } -func buildStdin(event *models.AlertCurEvent) ([]byte, error) { +func genNotice(event *models.AlertCurEvent) Notice { // build notice body with templates ntpls := make(map[string]string) for filename, tpl := range tpls { @@ -101,36 +101,40 @@ func buildStdin(event *models.AlertCurEvent) ([]byte, error) { } } - return json.Marshal(Notice{Event: event, Tpls: ntpls}) + return Notice{Event: event, Tpls: ntpls} } -func notify(event *models.AlertCurEvent) { - logEvent(event, "notify") - - stdin, err := buildStdin(event) - if err != nil { - logger.Errorf("event_notify: build stdin failed: %v", err) - return - } - +func alertingRedisPub(bs []byte) { // pub all alerts to redis if config.C.Alerting.RedisPub.Enable { - err = storage.Redis.Publish(context.Background(), config.C.Alerting.RedisPub.ChannelKey, stdin).Err() + err := storage.Redis.Publish(context.Background(), config.C.Alerting.RedisPub.ChannelKey, bs).Err() if err != nil { logger.Errorf("event_notify: redis publish %s err: %v", config.C.Alerting.RedisPub.ChannelKey, err) } } +} - if config.C.Alerting.GlobalCallback.Enable { - DoGlobalCallback(event) - } +func handleNotice(notice Notice, bs []byte) { + alertingCallScript(bs) - // no notify.py? do nothing - if config.C.Alerting.NotifyScriptPath == "" { + // TODO 弄个channel发邮件，学习daemon写法 + // 收集tokens、phones，发呗 +} + +func notify(event *models.AlertCurEvent) { + logEvent(event, "notify") + + notice := genNotice(event) + stdinBytes, err := json.Marshal(notice) + if err != nil { + logger.Errorf("event_notify: failed to marshal notice: %v", err) return } - callScript(stdin) + alertingRedisPub(stdinBytes) + alertingWebhook(event) + + handleNotice(notice, stdinBytes) // handle alert subscribes subs, has := memsto.AlertSubscribeCache.Get(event.RuleId) @@ -144,8 +148,13 @@ func notify(event *models.AlertCurEvent) { } } -func DoGlobalCallback(event *models.AlertCurEvent) { - conf := config.C.Alerting.GlobalCallback +func alertingWebhook(event *models.AlertCurEvent) { + conf := config.C.Alerting.Webhook + + if !conf.Enable { + return + } + if conf.Url == "" { return } @@ -159,7 +168,7 @@ func DoGlobalCallback(event *models.AlertCurEvent) { req, err := http.NewRequest("POST", conf.Url, bf) if err != nil { - logger.Warning("DoGlobalCallback failed to new request", err) + logger.Warning("alertingWebhook failed to new request", err) return } @@ -180,17 +189,17 @@ func DoGlobalCallback(event *models.AlertCurEvent) { var resp *http.Response resp, err = client.Do(req) if err != nil { - logger.Warning("DoGlobalCallback failed to call url, error: ", err) + logger.Warning("alertingWebhook failed to call url, error: ", err) return } var body []byte if resp.Body != nil { defer resp.Body.Close() - body, err = ioutil.ReadAll(resp.Body) + body, _ = ioutil.ReadAll(resp.Body) } - logger.Debugf("DoGlobalCallback done, url: %s, response code: %d, body: %s", conf.Url, resp.StatusCode, string(body)) + logger.Debugf("alertingWebhook done, url: %s, response code: %d, body: %s", conf.Url, resp.StatusCode, string(body)) } func handleSubscribes(event models.AlertCurEvent, subs []*models.AlertSubscribe) { @@ -223,17 +232,27 @@ func handleSubscribe(event models.AlertCurEvent, sub *models.AlertSubscribe) { fillUsers(&event) - stdin, err := buildStdin(&event) + notice := genNotice(&event) + stdinBytes, err := json.Marshal(notice) if err != nil { - logger.Errorf("event_notify: build stdin failed when handle subscribe: %v", err) + logger.Errorf("event_notify: failed to marshal notice: %v", err) return } - callScript(stdin) + handleNotice(notice, stdinBytes) } -func callScript(stdinBytes []byte) { - fpath := config.C.Alerting.NotifyScriptPath +func alertingCallScript(stdinBytes []byte) { + if !config.C.Alerting.CallScript.Enable { + return + } + + // no notify.py? do nothing + if config.C.Alerting.CallScript.ScriptPath == "" { + return + } + + fpath := config.C.Alerting.CallScript.ScriptPath cmd := exec.Command(fpath) cmd.Stdin = bytes.NewReader(stdinBytes) diff --git a/src/server/poster/post.go b/src/server/poster/post.go new file mode 100644 index 00000000..158df3a6 --- /dev/null +++ b/src/server/poster/post.go @@ -0,0 +1,42 @@ +package poster + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "time" +) + +func PostJSON(url string, timeout time.Duration, v interface{}) (response []byte, code int, err error) { + var bs []byte + + bs, err = json.Marshal(v) + if err != nil { + return + } + + bf := bytes.NewBuffer(bs) + + client := http.Client{ + Timeout: timeout, + } + + req, err := http.NewRequest("POST", url, bf) + req.Header.Set("Content-Type", "application/json") + + var resp *http.Response + resp, err = client.Do(req) + if err != nil { + return + } + + code = resp.StatusCode + + if resp.Body != nil { + defer resp.Body.Close() + response, err = ioutil.ReadAll(resp.Body) + } + + return +} diff --git a/src/server/sender/dingtalk.go b/src/server/sender/dingtalk.go new file mode 100644 index 00000000..c8325c85 --- /dev/null +++ b/src/server/sender/dingtalk.go @@ -0,0 +1,55 @@ +package sender + +import ( + "time" + + "github.com/didi/nightingale/v5/src/server/poster" + "github.com/toolkits/pkg/logger" +) + +type DingtalkMessage struct { + Title string + Text string + AtMobiles []string + Tokens []string +} + +type dingtalkMarkdown struct { + Title string `json:"title"` + Text string `json:"text"` +} + +type dingtalkAt struct { + AtMobiles []string `json:"atMobiles"` + IsAtAll bool `json:"isAtAll"` +} + +type dingtalk struct { + Msgtype string `json:"msgtype"` + Markdown dingtalkMarkdown `json:"markdown"` + At dingtalkAt `json:"at"` +} + +func SendDingtalk(message DingtalkMessage) { + for i := 0; i < len(message.Tokens); i++ { + url := "https://oapi.dingtalk.com/robot/send?access_token=" + message.Tokens[i] + body := dingtalk{ + Msgtype: "markdown", + Markdown: dingtalkMarkdown{ + Title: message.Title, + Text: message.Text, + }, + At: dingtalkAt{ + AtMobiles: message.AtMobiles, + IsAtAll: false, + }, + } + + res, code, err := poster.PostJSON(url, time.Second*5, body) + if err != nil { + logger.Errorf("dingtalk_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res)) + } else { + logger.Infof("dingtalk_sender: result=succ url=%s code=%d response=%s", url, code, string(res)) + } + } +} diff --git a/src/server/sender/feishu.go b/src/server/sender/feishu.go new file mode 100644 index 00000000..41693ac5 --- /dev/null +++ b/src/server/sender/feishu.go @@ -0,0 +1,52 @@ +package sender + +import ( + "time" + + "github.com/didi/nightingale/v5/src/server/poster" + "github.com/toolkits/pkg/logger" +) + +type FeishuMessage struct { + Text string + AtMobiles []string + Tokens []string +} + +type feishuContent struct { + Text string `json:"text"` +} + +type feishuAt struct { + AtMobiles []string `json:"atMobiles"` + IsAtAll bool `json:"isAtAll"` +} + +type feishu struct { + Msgtype string `json:"msg_type"` + Content feishuContent `json:"content"` + At feishuAt `json:"at"` +} + +func SendFeishu(message FeishuMessage) { + for i := 0; i < len(message.Tokens); i++ { + url := "https://open.feishu.cn/open-apis/bot/v2/hook/" + message.Tokens[i] + body := feishu{ + Msgtype: "text", + Content: feishuContent{ + Text: message.Text, + }, + At: feishuAt{ + AtMobiles: message.AtMobiles, + IsAtAll: false, + }, + } + + res, code, err := poster.PostJSON(url, time.Second*5, body) + if err != nil { + logger.Errorf("feishu_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res)) + } else { + logger.Infof("feishu_sender: result=succ url=%s code=%d response=%s", url, code, string(res)) + } + } +} diff --git a/src/server/sender/wecom.go b/src/server/sender/wecom.go new file mode 100644 index 00000000..2cf9769d --- /dev/null +++ b/src/server/sender/wecom.go @@ -0,0 +1,41 @@ +package sender + +import ( + "time" + + "github.com/didi/nightingale/v5/src/server/poster" + "github.com/toolkits/pkg/logger" +) + +type WecomMessage struct { + Text string + Tokens []string +} + +type wecomMarkdown struct { + Content string `json:"content"` +} + +type wecom struct { + Msgtype string `json:"msgtype"` + Markdown wecomMarkdown `json:"markdown"` +} + +func SendWecom(message WecomMessage) { + for i := 0; i < len(message.Tokens); i++ { + url := "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=" + message.Tokens[i] + body := wecom{ + Msgtype: "markdown", + Markdown: wecomMarkdown{ + Content: message.Text, + }, + } + + res, code, err := poster.PostJSON(url, time.Second*5, body) + if err != nil { + logger.Errorf("wecom_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res)) + } else { + logger.Infof("wecom_sender: result=succ url=%s code=%d response=%s", url, code, string(res)) + } + } +} -- GitLab