Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
jobily
Nightingale
提交
fd9d7806
N
Nightingale
项目概览
jobily
/
Nightingale
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
N
Nightingale
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
fd9d7806
编写于
6月 29, 2021
作者:
ning1875
提交者:
GitHub
6月 29, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
feat: notify support mail and dingding
上级
b03d57f4
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
235 addition
and
17 deletion
+235
-17
etc/script/notify.py
etc/script/notify.py
+235
-17
未找到文件。
etc/script/notify.py
浏览文件 @
fd9d7806
...
...
@@ -4,6 +4,7 @@ import sys
import
json
import
os
import
smtplib
import
time
from
email.mime.text
import
MIMEText
from
email.header
import
Header
...
...
@@ -12,6 +13,24 @@ from email.header import Header
# 2. 文件路径和名字是.alerts/${timestamp}_${ruleid}
# 3. 调用SMTP服务器发送告警,微信、钉钉、飞书、slack、jira、短信、电话等等留给社区实现
# 脚本二开指南
# 1. 可以根据下面的TEST_ALERT_JSON 中的结构修改脚本发送逻辑,定制化告警格式格式如下
"""
告警类型:prometheus
规则名称:pull_promql
是否已恢复:已触发
告警级别:1
触发时间:2021-06-28 11:46:35
可读表达式: go_gc_duration_seconds>0
当前值:[vector={__name__="go_gc_duration_seconds", instance="172.20.70.205:9100", job="node-targets", quantile="1"}]: [value=0.022498]
标签组:a=b c=d instance=172.20.70.205:9100 job=node-targets quantile=1
"""
# 2. 每个告警会以json文件的格式存储在LOCAL_EVENT_FILE_DIR 下面,文件名为 filename = '%d_%d_%d' % (rule_id, event_id, trigger_time)
# 3. 告警通道需要自行定义Send类中的send_xxx同名方法,反射调用:举例 event.notify_channels = [qq dingding]
# 则需要Send类中 有 send_qq send_dingding方法
import
requests
mail_host
=
"smtp.163.com"
mail_port
=
994
mail_user
=
"ulricqin"
...
...
@@ -24,40 +43,239 @@ mail_body = """
<p><a href="https://www.baidu.com">baidu</a></p>
"""
# 本地告警event json存储目录
LOCAL_EVENT_FILE_DIR
=
".alerts"
NOTIFY_CHANNELS_SPLIT_STR
=
" "
# stdin 告警json实例
TEST_ALERT_JSON
=
{
"event"
:
{
"alert_duration"
:
30
,
"hash_id"
:
"0c6cbe29df08bb6f9c26af518638128e"
,
"history_points"
:
[
{
"metric"
:
"go_gc_duration_seconds"
,
"points"
:
[
{
"t"
:
1624851995
,
"v"
:
0.000175
}
],
"tags"
:
{
"instance"
:
"172.20.70.205:9100"
,
"job"
:
"node-targets"
,
"quantile"
:
"0.25"
}
}
],
"id"
:
190
,
"is_prome_pull"
:
1
,
"is_recovery"
:
0
,
"last_sent"
:
True
,
"notify_channels"
:
"qq dingding "
,
"notify_group_objs"
:
None
,
"notify_groups"
:
""
,
"notify_user_objs"
:
None
,
"notify_users"
:
"1"
,
"priority"
:
1
,
"readable_expression"
:
" go_gc_duration_seconds>0"
,
"res_classpaths"
:
"all"
,
"res_ident"
:
""
,
"rule_id"
:
10
,
"rule_name"
:
"pull_promql"
,
"rule_note"
:
"note"
,
"runbook_url"
:
"qq.com"
,
"status"
:
0
,
"tag_map"
:
{
"a"
:
"b"
,
"c"
:
"d"
,
"instance"
:
"172.20.70.205:9100"
,
"job"
:
"node-targets"
,
"quantile"
:
"0.25"
},
"tags"
:
"a=b c=d instance=172.20.70.205:9100 job=node-targets quantile=0.25"
,
"trigger_time"
:
1624851995
,
"values"
:
"[vector={__name__=
\"
go_gc_duration_seconds
\"
, instance=
\"
172.20.70.205:9100
\"
, job=
\"
node-targets
\"
, quantile=
\"
0.25
\"
}]: [value=0.000175]"
},
"rule"
:
{
"alert_duration"
:
30
,
"append_tags"
:
"a=b c=d"
,
"callbacks"
:
"localhost:8881"
,
"create_at"
:
1624851512
,
"create_by"
:
"root"
,
"enable_days_of_week"
:
"1 2 3 4 5 6 7"
,
"enable_etime"
:
"23:59"
,
"enable_stime"
:
"00:00"
,
"expression"
:
{
"evaluation_interval"
:
3
,
"promql"
:
" go_gc_duration_seconds>0"
,
"resolve_timeout"
:
40
},
"group_id"
:
1
,
"id"
:
10
,
"name"
:
"pull_promql"
,
"note"
:
"note"
,
"notify_channels"
:
"qq dingding "
,
"notify_groups"
:
""
,
"notify_users"
:
"1"
,
"priority"
:
1
,
"recovery_notify"
:
0
,
"runbook_url"
:
"qq.com"
,
"status"
:
0
,
"type"
:
1
,
"update_at"
:
1624851512
,
"update_by"
:
"root"
},
"users"
:
[
{
"contacts"
:
None
,
"create_at"
:
1624258550
,
"create_by"
:
"system"
,
"email"
:
""
,
"id"
:
1
,
"nickname"
:
"
\u8d85\u7ba1
"
,
"phone"
:
""
,
"portrait"
:
""
,
"role"
:
"Admin"
,
"status"
:
0
,
"update_at"
:
1624258550
,
"update_by"
:
"system"
,
"username"
:
"root"
}
]
}
def
main
():
payload
=
json
.
load
(
sys
.
stdin
)
persist
(
payload
)
trigger_time
=
payload
[
'event'
][
'trigger_time'
]
event_id
=
payload
[
'event'
][
'id'
]
rule_id
=
payload
[
'rule'
][
'id'
]
notify_channels
=
payload
[
'event'
].
get
(
'notify_channels'
).
strip
().
split
(
NOTIFY_CHANNELS_SPLIT_STR
)
if
len
(
notify_channels
)
==
0
:
msg
=
"notify_channels_empty"
print
(
msg
)
return
# 持久化到本地json文件
persist
(
payload
,
rule_id
,
event_id
,
trigger_time
)
# 生成告警内容
alert_content
=
content_gen
(
payload
)
for
ch
in
notify_channels
:
send_func_name
=
"send_{}"
.
format
(
ch
.
strip
())
has_func
=
hasattr
(
Send
,
send_func_name
)
if
not
has_func
:
msg
=
"[send_func_name_err][func_not_found_in_Send_class:{}]"
.
format
(
send_func_name
)
print
(
msg
)
continue
send_func
=
getattr
(
Send
,
send_func_name
)
send_func
(
alert_content
)
def
content_gen
(
payload
):
# 生成格式化告警内容
text
=
""
event_obj
=
payload
.
get
(
"event"
)
rule_type
=
event_obj
.
get
(
"is_prome_pull"
)
type_str_m
=
{
1
:
"prometheus"
,
0
:
"n9e"
}
rule_type
=
type_str_m
.
get
(
rule_type
)
text
+=
"告警类型:{}
\n
"
.
format
(
rule_type
)
def
persist
(
payload
):
if
not
os
.
path
.
exists
(
".alerts"
):
os
.
makedirs
(
".alerts"
)
rule_name
=
event_obj
.
get
(
"rule_name"
)
text
+=
"规则名称:{}
\n
"
.
format
(
rule_name
)
filename
=
'%d_%d'
%
(
payload
[
'event'
][
'trigger_time'
],
payload
[
'rule'
][
'id'
])
filepath
=
os
.
path
.
join
(
".alerts"
,
filename
)
is_recovery
=
event_obj
.
get
(
"is_recovery"
)
is_recovery_str_m
=
{
1
:
"已恢复"
,
0
:
"已触发"
}
is_recovery
=
is_recovery_str_m
.
get
(
is_recovery
)
text
+=
"是否已恢复:{}
\n
"
.
format
(
is_recovery
)
f
=
open
(
filepath
,
'w'
)
print
(
json
.
dumps
(
payload
,
indent
=
4
),
file
=
f
)
f
.
close
()
priority
=
event_obj
.
get
(
"priority"
)
text
+=
"告警级别:{}
\n
"
.
format
(
priority
)
trigger_time
=
event_obj
.
get
(
"trigger_time"
)
text
+=
"触发时间:{}
\n
"
.
format
(
time
.
strftime
(
"%Y-%m-%d %H:%M:%S"
,
time
.
localtime
(
int
(
trigger_time
))))
def
send_mail
(
payload
):
print
(
"send_mail"
)
readable_expression
=
event_obj
.
get
(
"readable_expression"
)
text
+=
"可读表达式:{}
\n
"
.
format
(
readable_expression
)
values
=
event_obj
.
get
(
"values"
)
text
+=
"当前值:{}
\n
"
.
format
(
values
)
def
send_wecom
(
payload
):
print
(
"send_wecom"
)
tags
=
event_obj
.
get
(
"tags"
)
text
+=
"标签组:{}
\n
"
.
format
(
tags
)
print
(
text
)
return
text
def
send_dingtalk
(
payload
):
print
(
"send_dingtalk"
)
def
persist
(
payload
,
rule_id
,
event_id
,
trigger_time
):
if
not
os
.
path
.
exists
(
LOCAL_EVENT_FILE_DIR
):
os
.
makedirs
(
LOCAL_EVENT_FILE_DIR
)
filename
=
'%d_%d_%d'
%
(
rule_id
,
event_id
,
trigger_time
)
filepath
=
os
.
path
.
join
(
LOCAL_EVENT_FILE_DIR
,
filename
)
with
open
(
filepath
,
'w'
)
as
f
:
f
.
write
(
json
.
dumps
(
payload
,
indent
=
4
))
class
Send
(
object
):
@
classmethod
def
send_mail
(
cls
,
payload
):
users
=
payload
.
get
(
"event"
).
get
(
"users"
)
emails
=
[
x
.
get
(
"email"
)
for
x
in
users
]
if
not
emails
:
print
(
"[emails_empty]"
)
return
recipients
=
emails
message
=
MIMEText
(
mail_body
,
'html'
,
'utf-8'
)
message
[
'From'
]
=
mail_from
message
[
'To'
]
=
", "
.
join
(
recipients
)
message
[
"Subject"
]
=
"n9e alert"
smtp
=
smtplib
.
SMTP_SSL
(
mail_host
,
mail_port
)
smtp
.
login
(
mail_user
,
mail_pass
)
smtp
.
sendmail
(
mail_from
,
recipients
,
message
.
as_string
())
smtp
.
close
()
print
(
"send_mail_success"
)
@
classmethod
def
send_wecom
(
cls
,
payload
):
print
(
"send_wecom"
)
@
classmethod
def
send_dingtalk
(
cls
,
payload
):
# TODO 钉钉发群信息需要群的webhook机器人 token,这个信息可以写在告警策略中的附加字段中
dingtalk_api_url
=
"https://oapi.dingtalk.com/robot/send?access_token=xxxx"
users
=
payload
.
get
(
"event"
).
get
(
"users"
)
atMobiles
=
[
x
.
get
(
"phone"
)
for
x
in
users
]
headers
=
{
'Content-Type'
:
'application/json;charset=utf-8'
}
payload
=
{
"msgtype"
:
"text"
,
"text"
:
{
"content"
:
payload
},
"at"
:
{
"atMobiles"
:
atMobiles
,
"isAtAll"
:
False
}
}
res
=
requests
.
post
(
dingtalk_api_url
,
json
.
dumps
(
payload
),
headers
=
headers
)
print
(
res
.
status_code
)
print
(
res
.
text
)
print
(
"send_dingtalk"
)
def
mail_test
():
print
(
"mail_test_todo"
)
recipients
=
[
"ulricqin@qq.com"
,
"ulric@163.com"
]
message
=
MIMEText
(
mail_body
,
'html'
,
'utf-8'
)
message
[
'From'
]
=
mail_from
message
[
'To'
]
=
", "
.
join
(
recipients
)
...
...
@@ -77,4 +295,4 @@ if __name__ == "__main__":
elif
sys
.
argv
[
1
]
==
"mail"
:
mail_test
()
else
:
print
(
"I am confused"
)
\ No newline at end of file
print
(
"I am confused"
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录