not stable version

etc/server.conf

@@ -54,17 +54,30 @@ IP = ""
 # unit ms
 Interval = 1000
 
+[SMTP]
+Host = "smtp.163.com"
+Port = 994
+User = "username"
+Pass = "password"
+From = "username@163.com"
+InsecureSkipVerify = true
+
 [Alerting]
-NotifyScriptPath = "./etc/script/notify.py"
-NotifyConcurrency = 100
 TemplatesDir = "./etc/template"
+NotifyConcurrency = 100
+
+[Alerting.CallScript]
+# built in sending capability in go code
+# so, no need enable script sender
+Enable = false
+ScriptPath = "./etc/script/notify.py"
 
 [Alerting.RedisPub]
 Enable = false
 # complete redis key: ${ChannelPrefix} + ${Cluster}
 ChannelPrefix = "/alerts/"
 
-[Alerting.GlobalCallback]
+[Alerting.Webhook]
 Enable = false
 Url = "http://a.com/n9e/callback"
 BasicAuthUser = ""

src/server/config/config.go

@@ -79,16 +79,16 @@ func MustLoad(fpaths ...string) {
 		C.Heartbeat.Endpoint = fmt.Sprintf("%s:%d", C.Heartbeat.IP, C.HTTP.Port)
 		C.Alerting.RedisPub.ChannelKey = C.Alerting.RedisPub.ChannelPrefix + C.ClusterName
 
-		if C.Alerting.GlobalCallback.Enable {
-			if C.Alerting.GlobalCallback.Timeout == "" {
-				C.Alerting.GlobalCallback.TimeoutDuration = time.Second * 5
+		if C.Alerting.Webhook.Enable {
+			if C.Alerting.Webhook.Timeout == "" {
+				C.Alerting.Webhook.TimeoutDuration = time.Second * 5
 			} else {
-				dur, err := time.ParseDuration(C.Alerting.GlobalCallback.Timeout)
+				dur, err := time.ParseDuration(C.Alerting.Webhook.Timeout)
 				if err != nil {
-					fmt.Println("failed to parse Alerting.GlobalCallback.Timeout")
+					fmt.Println("failed to parse Alerting.Webhook.Timeout")
 					os.Exit(1)
 				}
-				C.Alerting.GlobalCallback.TimeoutDuration = dur
+				C.Alerting.Webhook.TimeoutDuration = dur
 			}
 		}
 
@@ -104,6 +104,7 @@ type Config struct {
 	Log         logx.Config
 	HTTP        httpx.Config
 	BasicAuth   gin.Accounts
+	SMTP        SMTPConfig
 	Heartbeat   HeartbeatConfig
 	Alerting    Alerting
 	NoData      NoData
@@ -123,12 +124,26 @@ type HeartbeatConfig struct {
 	Endpoint string
 }
 
+type SMTPConfig struct {
+	Host               string
+	Port               int
+	User               string
+	Pass               string
+	From               string
+	InsecureSkipVerify bool
+}
+
 type Alerting struct {
-	NotifyScriptPath  string
-	NotifyConcurrency int
 	TemplatesDir      string
+	NotifyConcurrency int
+	CallScript        CallScript
 	RedisPub          RedisPub
-	GlobalCallback    GlobalCallback
+	Webhook           Webhook
+}
+
+type CallScript struct {
+	Enable     bool
+	ScriptPath string
 }
 
 type RedisPub struct {
@@ -137,7 +152,7 @@ type RedisPub struct {
 	ChannelKey    string
 }
 
-type GlobalCallback struct {
+type Webhook struct {
 	Enable          bool
 	Url             string
 	BasicAuthUser   string

src/server/engine/callback.go

 package engine
 
 import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
 	"strconv"
 	"strings"
 	"time"
@@ -15,41 +11,9 @@ import (
 	"github.com/didi/nightingale/v5/src/pkg/ibex"
 	"github.com/didi/nightingale/v5/src/server/config"
 	"github.com/didi/nightingale/v5/src/server/memsto"
+	"github.com/didi/nightingale/v5/src/server/poster"
 )
 
-func PostJSON(url string, timeout time.Duration, v interface{}) (response []byte, code int, err error) {
-	var bs []byte
-
-	bs, err = json.Marshal(v)
-	if err != nil {
-		return
-	}
-
-	bf := bytes.NewBuffer(bs)
-
-	client := http.Client{
-		Timeout: timeout,
-	}
-
-	req, err := http.NewRequest("POST", url, bf)
-	req.Header.Set("Content-Type", "application/json")
-
-	var resp *http.Response
-	resp, err = client.Do(req)
-	if err != nil {
-		return
-	}
-
-	code = resp.StatusCode
-
-	if resp.Body != nil {
-		defer resp.Body.Close()
-		response, err = ioutil.ReadAll(resp.Body)
-	}
-
-	return
-}
-
 func callback(event *models.AlertCurEvent) {
 	urls := strings.Fields(event.Callbacks)
 	for _, url := range urls {
@@ -68,7 +32,7 @@ func callback(event *models.AlertCurEvent) {
 			url = "http://" + url
 		}
 
-		resp, code, err := PostJSON(url, 5*time.Second, event)
+		resp, code, err := poster.PostJSON(url, 5*time.Second, event)
 		if err != nil {
 			logger.Errorf("event_callback(rule_id=%d url=%s) fail, resp: %s, err: %v, code: %d", event.RuleId, url, string(resp), err, code)
 		} else {

src/server/engine/notify.go

@@ -31,14 +31,14 @@ var fns = template.FuncMap{
 	"urlconvert": func(str string) interface{} { return template.URL(str) },
 	"timeformat": func(ts int64, pattern ...string) string {
 		defp := "2006-01-02 15:04:05"
-		if pattern != nil && len(pattern) > 0 {
+		if len(pattern) > 0 {
 			defp = pattern[0]
 		}
 		return time.Unix(ts, 0).Format(defp)
 	},
 	"timestamp": func(pattern ...string) string {
 		defp := "2006-01-02 15:04:05"
-		if pattern != nil && len(pattern) > 0 {
+		if len(pattern) > 0 {
 			defp = pattern[0]
 		}
 		return time.Now().Format(defp)
@@ -89,7 +89,7 @@ type Notice struct {
 	Tpls  map[string]string     `json:"tpls"`
 }
 
-func buildStdin(event *models.AlertCurEvent) ([]byte, error) {
+func genNotice(event *models.AlertCurEvent) Notice {
 	// build notice body with templates
 	ntpls := make(map[string]string)
 	for filename, tpl := range tpls {
@@ -101,36 +101,40 @@ func buildStdin(event *models.AlertCurEvent) ([]byte, error) {
 		}
 	}
 
-	return json.Marshal(Notice{Event: event, Tpls: ntpls})
+	return Notice{Event: event, Tpls: ntpls}
 }
 
-func notify(event *models.AlertCurEvent) {
-	logEvent(event, "notify")
-
-	stdin, err := buildStdin(event)
-	if err != nil {
-		logger.Errorf("event_notify: build stdin failed: %v", err)
-		return
-	}
-
+func alertingRedisPub(bs []byte) {
 	// pub all alerts to redis
 	if config.C.Alerting.RedisPub.Enable {
-		err = storage.Redis.Publish(context.Background(), config.C.Alerting.RedisPub.ChannelKey, stdin).Err()
+		err := storage.Redis.Publish(context.Background(), config.C.Alerting.RedisPub.ChannelKey, bs).Err()
 		if err != nil {
 			logger.Errorf("event_notify: redis publish %s err: %v", config.C.Alerting.RedisPub.ChannelKey, err)
 		}
 	}
+}
 
-	if config.C.Alerting.GlobalCallback.Enable {
-		DoGlobalCallback(event)
-	}
+func handleNotice(notice Notice, bs []byte) {
+	alertingCallScript(bs)
 
-	// no notify.py? do nothing
-	if config.C.Alerting.NotifyScriptPath == "" {
+	// TODO 弄个channel发邮件，学习daemon写法
+	// 收集tokens、phones，发呗
+}
+
+func notify(event *models.AlertCurEvent) {
+	logEvent(event, "notify")
+
+	notice := genNotice(event)
+	stdinBytes, err := json.Marshal(notice)
+	if err != nil {
+		logger.Errorf("event_notify: failed to marshal notice: %v", err)
 		return
 	}
 
-	callScript(stdin)
+	alertingRedisPub(stdinBytes)
+	alertingWebhook(event)
+
+	handleNotice(notice, stdinBytes)
 
 	// handle alert subscribes
 	subs, has := memsto.AlertSubscribeCache.Get(event.RuleId)
@@ -144,8 +148,13 @@ func notify(event *models.AlertCurEvent) {
 	}
 }
 
-func DoGlobalCallback(event *models.AlertCurEvent) {
-	conf := config.C.Alerting.GlobalCallback
+func alertingWebhook(event *models.AlertCurEvent) {
+	conf := config.C.Alerting.Webhook
+
+	if !conf.Enable {
+		return
+	}
+
 	if conf.Url == "" {
 		return
 	}
@@ -159,7 +168,7 @@ func DoGlobalCallback(event *models.AlertCurEvent) {
 
 	req, err := http.NewRequest("POST", conf.Url, bf)
 	if err != nil {
-		logger.Warning("DoGlobalCallback failed to new request", err)
+		logger.Warning("alertingWebhook failed to new request", err)
 		return
 	}
 
@@ -180,17 +189,17 @@ func DoGlobalCallback(event *models.AlertCurEvent) {
 	var resp *http.Response
 	resp, err = client.Do(req)
 	if err != nil {
-		logger.Warning("DoGlobalCallback failed to call url, error: ", err)
+		logger.Warning("alertingWebhook failed to call url, error: ", err)
 		return
 	}
 
 	var body []byte
 	if resp.Body != nil {
 		defer resp.Body.Close()
-		body, err = ioutil.ReadAll(resp.Body)
+		body, _ = ioutil.ReadAll(resp.Body)
 	}
 
-	logger.Debugf("DoGlobalCallback done, url: %s, response code: %d, body: %s", conf.Url, resp.StatusCode, string(body))
+	logger.Debugf("alertingWebhook done, url: %s, response code: %d, body: %s", conf.Url, resp.StatusCode, string(body))
 }
 
 func handleSubscribes(event models.AlertCurEvent, subs []*models.AlertSubscribe) {
@@ -223,17 +232,27 @@ func handleSubscribe(event models.AlertCurEvent, sub *models.AlertSubscribe) {
 
 	fillUsers(&event)
 
-	stdin, err := buildStdin(&event)
+	notice := genNotice(&event)
+	stdinBytes, err := json.Marshal(notice)
 	if err != nil {
-		logger.Errorf("event_notify: build stdin failed when handle subscribe: %v", err)
+		logger.Errorf("event_notify: failed to marshal notice: %v", err)
 		return
 	}
 
-	callScript(stdin)
+	handleNotice(notice, stdinBytes)
 }
 
-func callScript(stdinBytes []byte) {
-	fpath := config.C.Alerting.NotifyScriptPath
+func alertingCallScript(stdinBytes []byte) {
+	if !config.C.Alerting.CallScript.Enable {
+		return
+	}
+
+	// no notify.py? do nothing
+	if config.C.Alerting.CallScript.ScriptPath == "" {
+		return
+	}
+
+	fpath := config.C.Alerting.CallScript.ScriptPath
 	cmd := exec.Command(fpath)
 	cmd.Stdin = bytes.NewReader(stdinBytes)
 

src/server/poster/post.go

+package poster
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+func PostJSON(url string, timeout time.Duration, v interface{}) (response []byte, code int, err error) {
+	var bs []byte
+
+	bs, err = json.Marshal(v)
+	if err != nil {
+		return
+	}
+
+	bf := bytes.NewBuffer(bs)
+
+	client := http.Client{
+		Timeout: timeout,
+	}
+
+	req, err := http.NewRequest("POST", url, bf)
+	req.Header.Set("Content-Type", "application/json")
+
+	var resp *http.Response
+	resp, err = client.Do(req)
+	if err != nil {
+		return
+	}
+
+	code = resp.StatusCode
+
+	if resp.Body != nil {
+		defer resp.Body.Close()
+		response, err = ioutil.ReadAll(resp.Body)
+	}
+
+	return
+}

src/server/sender/dingtalk.go

+package sender
+
+import (
+	"time"
+
+	"github.com/didi/nightingale/v5/src/server/poster"
+	"github.com/toolkits/pkg/logger"
+)
+
+type DingtalkMessage struct {
+	Title     string
+	Text      string
+	AtMobiles []string
+	Tokens    []string
+}
+
+type dingtalkMarkdown struct {
+	Title string `json:"title"`
+	Text  string `json:"text"`
+}
+
+type dingtalkAt struct {
+	AtMobiles []string `json:"atMobiles"`
+	IsAtAll   bool     `json:"isAtAll"`
+}
+
+type dingtalk struct {
+	Msgtype  string           `json:"msgtype"`
+	Markdown dingtalkMarkdown `json:"markdown"`
+	At       dingtalkAt       `json:"at"`
+}
+
+func SendDingtalk(message DingtalkMessage) {
+	for i := 0; i < len(message.Tokens); i++ {
+		url := "https://oapi.dingtalk.com/robot/send?access_token=" + message.Tokens[i]
+		body := dingtalk{
+			Msgtype: "markdown",
+			Markdown: dingtalkMarkdown{
+				Title: message.Title,
+				Text:  message.Text,
+			},
+			At: dingtalkAt{
+				AtMobiles: message.AtMobiles,
+				IsAtAll:   false,
+			},
+		}
+
+		res, code, err := poster.PostJSON(url, time.Second*5, body)
+		if err != nil {
+			logger.Errorf("dingtalk_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res))
+		} else {
+			logger.Infof("dingtalk_sender: result=succ url=%s code=%d response=%s", url, code, string(res))
+		}
+	}
+}

src/server/sender/feishu.go

+package sender
+
+import (
+	"time"
+
+	"github.com/didi/nightingale/v5/src/server/poster"
+	"github.com/toolkits/pkg/logger"
+)
+
+type FeishuMessage struct {
+	Text      string
+	AtMobiles []string
+	Tokens    []string
+}
+
+type feishuContent struct {
+	Text string `json:"text"`
+}
+
+type feishuAt struct {
+	AtMobiles []string `json:"atMobiles"`
+	IsAtAll   bool     `json:"isAtAll"`
+}
+
+type feishu struct {
+	Msgtype string        `json:"msg_type"`
+	Content feishuContent `json:"content"`
+	At      feishuAt      `json:"at"`
+}
+
+func SendFeishu(message FeishuMessage) {
+	for i := 0; i < len(message.Tokens); i++ {
+		url := "https://open.feishu.cn/open-apis/bot/v2/hook/" + message.Tokens[i]
+		body := feishu{
+			Msgtype: "text",
+			Content: feishuContent{
+				Text: message.Text,
+			},
+			At: feishuAt{
+				AtMobiles: message.AtMobiles,
+				IsAtAll:   false,
+			},
+		}
+
+		res, code, err := poster.PostJSON(url, time.Second*5, body)
+		if err != nil {
+			logger.Errorf("feishu_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res))
+		} else {
+			logger.Infof("feishu_sender: result=succ url=%s code=%d response=%s", url, code, string(res))
+		}
+	}
+}

src/server/sender/wecom.go

+package sender
+
+import (
+	"time"
+
+	"github.com/didi/nightingale/v5/src/server/poster"
+	"github.com/toolkits/pkg/logger"
+)
+
+type WecomMessage struct {
+	Text   string
+	Tokens []string
+}
+
+type wecomMarkdown struct {
+	Content string `json:"content"`
+}
+
+type wecom struct {
+	Msgtype  string        `json:"msgtype"`
+	Markdown wecomMarkdown `json:"markdown"`
+}
+
+func SendWecom(message WecomMessage) {
+	for i := 0; i < len(message.Tokens); i++ {
+		url := "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=" + message.Tokens[i]
+		body := wecom{
+			Msgtype: "markdown",
+			Markdown: wecomMarkdown{
+				Content: message.Text,
+			},
+		}
+
+		res, code, err := poster.PostJSON(url, time.Second*5, body)
+		if err != nil {
+			logger.Errorf("wecom_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res))
+		} else {
+			logger.Infof("wecom_sender: result=succ url=%s code=%d response=%s", url, code, string(res))
+		}
+	}
+}