package http import ( "encoding/json" "fmt" "regexp" "strings" "github.com/didi/nightingale/src/models" "github.com/didi/nightingale/src/modules/monapi/collector" "github.com/didi/nightingale/src/modules/monapi/scache" "github.com/gin-gonic/gin" "github.com/toolkits/pkg/errors" "github.com/toolkits/pkg/logger" ) type CollectRecv struct { Type string `json:"type"` Data json.RawMessage `json:"data"` } func collectRulePost(c *gin.Context) { var recv []CollectRecv errors.Dangerous(c.ShouldBind(&recv)) creator := loginUsername(c) for _, obj := range recv { cl, err := collector.GetCollector(obj.Type) errors.Dangerous(err) if err := cl.Create([]byte(obj.Data), creator); err != nil { errors.Bomb("%s add rule err %s", obj.Type, err) } } renderData(c, "ok", nil) } func collectRulesGetByLocalEndpoint(c *gin.Context) { collect := scache.CollectCache.GetBy(urlParamStr(c, "endpoint")) renderData(c, collect, nil) } func collectRuleGet(c *gin.Context) { t := mustQueryStr(c, "type") id := mustQueryInt64(c, "id") cl, err := collector.GetCollector(t) errors.Dangerous(err) ret, err := cl.Get(id) renderData(c, ret, err) } func collectRulesGet(c *gin.Context) { nid := queryInt64(c, "nid", -1) tp := queryStr(c, "type", "") var resp []interface{} var types []string if tp == "" { types = []string{"port", "proc", "log", "plugin"} } else { types = []string{tp} } nids := []int64{nid} for _, t := range types { cl, err := collector.GetCollector(t) if err != nil { logger.Warning(t, err) continue } ret, err := cl.Gets(nids) if err != nil { logger.Warning(t, err) continue } resp = append(resp, ret...) } renderData(c, resp, nil) } func collectRulesGetV2(c *gin.Context) { nid := queryInt64(c, "nid", 0) limit := queryInt(c, "limit", 20) typ := queryStr(c, "type", "") total, list, err := models.GetCollectRules(typ, nid, limit, offset(c, limit, 0)) renderData(c, map[string]interface{}{ "total": total, "list": list, }, err) } func collectRulePut(c *gin.Context) { var recv CollectRecv errors.Dangerous(c.ShouldBind(&recv)) cl, err := collector.GetCollector(recv.Type) errors.Dangerous(err) creator := loginUsername(c) if err := cl.Update([]byte(recv.Data), creator); err != nil { errors.Bomb("%s update rule err %s", recv.Type, err) } renderData(c, "ok", nil) } type CollectsDelRev struct { Type string `json:"type"` Ids []int64 `json:"ids"` } func collectsRuleDel(c *gin.Context) { var recv []CollectsDelRev errors.Dangerous(c.ShouldBind(&recv)) username := loginUsername(c) for _, obj := range recv { for i := 0; i < len(obj.Ids); i++ { cl, err := collector.GetCollector(obj.Type) errors.Dangerous(err) if err := cl.Delete(obj.Ids[i], username); err != nil { errors.Dangerous(err) } } } renderData(c, "ok", nil) } func collectRuleTypesGet(c *gin.Context) { category := mustQueryStr(c, "category") switch category { case "remote": renderData(c, collector.GetRemoteCollectors(), nil) case "local": renderData(c, collector.GetLocalCollectors(), nil) default: renderData(c, nil, nil) } } func collectRuleTemplateGet(c *gin.Context) { t := urlParamStr(c, "type") collector, err := collector.GetCollector(t) errors.Dangerous(err) tpl, err := collector.Template() renderData(c, tpl, err) } type RegExpCheckDto struct { Success bool `json:"success"` Data []map[string]string `json:"tags"` } var RegExpExcludePatition string = "```EXCLUDE```" func regExpCheck(c *gin.Context) { param := make(map[string]string, 0) errors.Dangerous(c.ShouldBind(¶m)) ret := &RegExpCheckDto{ Success: true, Data: make([]map[string]string, 0), } // 处理时间格式 if t, ok := param["time"]; !ok || t == "" { tmp := map[string]string{"time": "time参数不存在或为空"} ret.Data = append(ret.Data, tmp) } else { timePat, _ := GetPatAndTimeFormat(param["time"]) if timePat == "" { tmp := map[string]string{"time": genErrMsg("时间格式")} ret.Data = append(ret.Data, tmp) } else { suc, tRes, _ := checkRegPat(timePat, param["log"], true) if !suc { ret.Success = false tRes = genErrMsg("时间格式") } tmp := map[string]string{"time": tRes} ret.Data = append(ret.Data, tmp) } } // 计算方式 calc_method, _ := param["calc_method"] // 处理主正则(with exclude) if re, ok := param["re"]; !ok || re == "" { tmp := map[string]string{"re": "re参数不存在或为空"} ret.Data = append(ret.Data, tmp) } else { // 处理exclude的情况 exclude := "" if strings.Contains(re, RegExpExcludePatition) { l := strings.Split(re, RegExpExcludePatition) if len(l) >= 2 { param["re"] = l[0] exclude = l[1] } } // 匹配主正则 suc, reRes, isSub := checkRegPat(param["re"], param["log"], false) if !suc { ret.Success = false reRes = genErrMsg("主正则") } if calc_method != "" && calc_method != "cnt" && !isSub { ret.Success = false reRes = genSubErrMsg("主正则") } tmp := map[string]string{"主正则": reRes} ret.Data = append(ret.Data, tmp) // 匹配exclude, 这个不影响失败 if exclude != "" { suc, exRes, _ := checkRegPat(exclude, param["log"], false) if !suc { //ret.Success = false exRes = "未匹配到排除串,请检查是否符合预期" } tmp := map[string]string{"排除串": exRes} ret.Data = append(ret.Data, tmp) } } // 处理tags var nonTagKey = map[string]bool{ "re": true, "log": true, "time": true, "calc_method": true, } for tagk, pat := range param { // 如果不是tag,就继续循环 if _, ok := nonTagKey[tagk]; ok { continue } suc, tagRes, isSub := checkRegPat(pat, param["log"], false) if !suc { // 正则错误 ret.Success = false tagRes = genErrMsg(tagk) } else if !isSub { // 未匹配出子串 ret.Success = false tagRes = genSubErrMsg(tagk) } else if includeIllegalChar(tagRes) || includeIllegalChar(tagk) { // 保留字报错 ret.Success = false tagRes = genIllegalCharErrMsg() } tmp := map[string]string{tagk: tagRes} ret.Data = append(ret.Data, tmp) } renderData(c, ret, nil) } //根据配置的时间格式,获取对应的正则匹配pattern和time包用的时间格式 func GetPatAndTimeFormat(tf string) (string, string) { var pat, timeFormat string switch tf { case "dd/mmm/yyyy:HH:MM:SS": pat = `([012][0-9]|3[01])/[JFMASOND][a-z]{2}/(2[0-9]{3}):([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "02/Jan/2006:15:04:05" case "dd/mmm/yyyy HH:MM:SS": pat = `([012][0-9]|3[01])/[JFMASOND][a-z]{2}/(2[0-9]{3})\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "02/Jan/2006 15:04:05" case "yyyy-mm-ddTHH:MM:SS": pat = `(2[0-9]{3})-(0[1-9]|1[012])-([012][0-9]|3[01])T([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "2006-01-02T15:04:05" case "dd-mmm-yyyy HH:MM:SS": pat = `([012][0-9]|3[01])-[JFMASOND][a-z]{2}-(2[0-9]{3})\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "02-Jan-2006 15:04:05" case "yyyy-mm-dd HH:MM:SS": pat = `(2[0-9]{3})-(0[1-9]|1[012])-([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "2006-01-02 15:04:05" case "yyyy/mm/dd HH:MM:SS": pat = `(2[0-9]{3})/(0[1-9]|1[012])/([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "2006/01/02 15:04:05" case "yyyymmdd HH:MM:SS": pat = `(2[0-9]{3})(0[1-9]|1[012])([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "20060102 15:04:05" case "mmm dd HH:MM:SS": pat = `[JFMASOND][a-z]{2}\s+([1-9]|[1-2][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "Jan 2 15:04:05" case "mmdd HH:MM:SS": pat = `(0[1-9]|1[012])([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` timeFormat = "0102 15:04:05" default: logger.Errorf("match time pac failed : [timeFormat:%s]", tf) return "", "" } return pat, timeFormat } // 出错信息直接放在body里 func checkRegPat(pat string, log string, origin bool) (succ bool, result string, isSub bool) { if pat == "" { return false, "", false } reg, err := regexp.Compile(pat) if err != nil { return false, "", false } res := reg.FindStringSubmatch(log) switch len(res) { // 没查到 case 0: return false, "", false // 没查到括号内的串,返回整个匹配串 case 1: return true, res[0], false // 查到了,默认取第一个串 default: var msg string if origin { msg = res[0] isSub = false } else { msg = res[1] isSub = true } return true, msg, isSub } } func includeIllegalChar(s string) bool { illegalChars := ":,=\r\n\t" return strings.ContainsAny(s, illegalChars) } // 生成返回错误信息 func genErrMsg(sign string) string { return fmt.Sprintf("正则匹配失败,请认真检查您[%s]的配置", sign) } // 生成子串匹配错误信息 func genSubErrMsg(sign string) string { return fmt.Sprintf("正则匹配成功。但根据配置,并没有获取到()内的子串,请认真检查您[%s]的配置", sign) } // 生成子串匹配错误信息 func genIllegalCharErrMsg() string { return fmt.Sprintf(`正则匹配成功。但是tag的key或者value包含非法字符:[:,/=\r\n\t], 请重新调整`) } func collectRulesGetByRemoteEndpoint(c *gin.Context) { rules := scache.CollectRuleCache.GetBy(urlParamStr(c, "endpoint")) renderData(c, rules, nil) }