Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
jobily
Nightingale
提交
178acfb4
N
Nightingale
项目概览
jobily
/
Nightingale
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
N
Nightingale
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
178acfb4
编写于
3月 21, 2020
作者:
U
Ulric Qin
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'master' of github.com:didi/nightingale
上级
7ef118c9
3092b4b6
变更
18
隐藏空白更改
内联
并排
Showing
18 changed file
with
50 addition
and
27 deletion
+50
-27
src/modules/collector/stra/cron.go
src/modules/collector/stra/cron.go
+1
-1
src/modules/collector/sys/funcs/push.go
src/modules/collector/sys/funcs/push.go
+3
-1
src/modules/index/cache/endpoints.go
src/modules/index/cache/endpoints.go
+4
-2
src/modules/index/cache/indexdb.go
src/modules/index/cache/indexdb.go
+2
-1
src/modules/judge/backend/redi/funcs.go
src/modules/judge/backend/redi/funcs.go
+0
-1
src/modules/judge/backend/redi/redis.go
src/modules/judge/backend/redi/redis.go
+4
-0
src/modules/judge/judge/judge.go
src/modules/judge/judge/judge.go
+6
-5
src/modules/judge/stra/stra.go
src/modules/judge/stra/stra.go
+2
-0
src/modules/monapi/cron/checker_judge.go
src/modules/monapi/cron/checker_judge.go
+2
-0
src/modules/monapi/redisc/redis.go
src/modules/monapi/redisc/redis.go
+4
-0
src/modules/transfer/backend/query.go
src/modules/transfer/backend/query.go
+3
-0
src/modules/transfer/backend/sender.go
src/modules/transfer/backend/sender.go
+12
-9
src/modules/transfer/rpc/push.go
src/modules/transfer/rpc/push.go
+3
-2
src/modules/tsdb/index/index.go
src/modules/tsdb/index/index.go
+1
-1
src/modules/tsdb/rpc/push.go
src/modules/tsdb/rpc/push.go
+0
-1
src/modules/tsdb/rrdtool/sync_disk.go
src/modules/tsdb/rrdtool/sync_disk.go
+1
-0
src/toolkits/identity/identity.go
src/toolkits/identity/identity.go
+2
-1
src/toolkits/stats/init.go
src/toolkits/stats/init.go
+0
-2
未找到文件。
src/modules/collector/stra/cron.go
浏览文件 @
178acfb4
...
...
@@ -74,7 +74,7 @@ func getCollects() (CollectResp, error) {
url
:=
fmt
.
Sprintf
(
"http://%s%s%s"
,
addr
,
StraConfig
.
Api
,
identity
.
Identity
)
err
=
httplib
.
Get
(
url
)
.
SetTimeout
(
time
.
Duration
(
StraConfig
.
Timeout
)
*
time
.
Millisecond
)
.
ToJSON
(
&
res
)
if
err
!=
nil
{
err
=
fmt
.
Errorf
(
"get collects from remote
failed, error:%v"
,
err
)
err
=
fmt
.
Errorf
(
"get collects from remote
:%s failed, error:%v"
,
url
,
err
)
}
return
res
,
err
...
...
src/modules/collector/sys/funcs/push.go
浏览文件 @
178acfb4
...
...
@@ -48,7 +48,9 @@ func Push(items []*dataobj.MetricValue) {
logger
.
Error
(
err
)
continue
}
else
{
logger
.
Info
(
"push succ, reply: "
,
reply
)
if
reply
.
Msg
!=
"ok"
{
logger
.
Error
(
"some item push err"
,
reply
)
}
return
}
}
...
...
src/modules/index/cache/endpoints.go
浏览文件 @
178acfb4
...
...
@@ -6,6 +6,7 @@ import (
"time"
"github.com/didi/nightingale/src/toolkits/address"
"github.com/didi/nightingale/src/toolkits/stats"
"github.com/toolkits/pkg/concurrent/semaphore"
"github.com/toolkits/pkg/logger"
...
...
@@ -49,11 +50,12 @@ func reportEndpoint(endpoints []interface{}) {
err
:=
httplib
.
Post
(
url
)
.
JSONBodyQuiet
(
m
)
.
SetTimeout
(
3
*
time
.
Second
)
.
Header
(
"x-srv-token"
,
"monapi-builtin-token"
)
.
ToJSON
(
&
body
)
if
err
!=
nil
{
logger
.
Warningf
(
"curl %s fail: %v. retry"
,
url
,
err
)
stats
.
Counter
.
Set
(
"report.endpoint.err"
,
1
)
continue
}
if
body
.
Err
!=
""
{
if
body
.
Err
!=
""
{
//数据库连接出错会出现此情况
logger
.
Warningf
(
"curl %s fail: %s. retry"
,
url
,
body
.
Err
)
stats
.
Counter
.
Set
(
"report.endpoint.err"
,
1
)
continue
}
...
...
src/modules/index/cache/indexdb.go
浏览文件 @
178acfb4
...
...
@@ -19,6 +19,7 @@ import (
"github.com/didi/nightingale/src/toolkits/compress"
"github.com/didi/nightingale/src/toolkits/identity"
"github.com/didi/nightingale/src/toolkits/report"
"github.com/didi/nightingale/src/toolkits/stats"
)
type
CacheSection
struct
{
...
...
@@ -72,8 +73,8 @@ func StartPersist(interval int) {
err
:=
Persist
(
"normal"
)
if
err
!=
nil
{
logger
.
Error
(
"Persist err:"
,
err
)
stats
.
Counter
.
Set
(
"persist.err"
,
1
)
}
//logger.Infof("clean %+v, took %.2f ms\n", cleanRet, float64(time.Since(start).Nanoseconds())*1e-6)
}
}
...
...
src/modules/judge/backend/redi/funcs.go
浏览文件 @
178acfb4
...
...
@@ -42,6 +42,5 @@ func Push(event *dataobj.Event) error {
return
nil
}
stats
.
Counter
.
Set
(
"redis.failed"
,
1
)
return
fmt
.
Errorf
(
"redis publish failed finally:%v"
,
err
)
}
src/modules/judge/backend/redi/redis.go
浏览文件 @
178acfb4
...
...
@@ -4,6 +4,7 @@ import (
"log"
"time"
"github.com/didi/nightingale/src/toolkits/stats"
"github.com/garyburd/redigo/redis"
"github.com/toolkits/pkg/logger"
)
...
...
@@ -44,6 +45,7 @@ func Init(cfg RedisSection) {
c
,
err
:=
redis
.
Dial
(
"tcp"
,
addr
,
redis
.
DialConnectTimeout
(
connTimeout
),
redis
.
DialReadTimeout
(
readTimeout
),
redis
.
DialWriteTimeout
(
writeTimeout
))
if
err
!=
nil
{
logger
.
Errorf
(
"conn redis err:%v"
,
err
)
stats
.
Counter
.
Set
(
"redis.conn.failed"
,
1
)
return
nil
,
err
}
...
...
@@ -51,6 +53,8 @@ func Init(cfg RedisSection) {
if
_
,
err
:=
c
.
Do
(
"AUTH"
,
pass
);
err
!=
nil
{
c
.
Close
()
logger
.
Errorf
(
"ERR: redis auth fail:%v"
,
err
)
stats
.
Counter
.
Set
(
"redis.conn.failed"
,
1
)
return
nil
,
err
}
}
...
...
src/modules/judge/judge/judge.go
浏览文件 @
178acfb4
...
...
@@ -65,7 +65,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
stats
.
Counter
.
Set
(
"running"
,
1
)
if
len
(
exps
)
<
1
{
stats
.
Counter
.
Set
(
"stra.
err
"
,
1
)
stats
.
Counter
.
Set
(
"stra.
illegal
"
,
1
)
logger
.
Warningf
(
"stra:%v exp is null"
,
stra
)
return
}
...
...
@@ -110,16 +110,16 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
}()
leftValue
,
isTriggered
=
judgeItemWithStrategy
(
stra
,
historyData
,
exps
[
0
],
firstItem
,
now
)
if
!
isTriggered
{
return
}
if
value
==
""
{
value
=
fmt
.
Sprintf
(
"%s: %v"
,
exp
.
Metric
,
leftValue
)
}
else
{
value
+=
fmt
.
Sprintf
(
"; %s: %v"
,
exp
.
Metric
,
leftValue
)
}
if
!
isTriggered
{
return
}
//与条件情况下执行
if
len
(
exps
)
>
1
{
if
exps
[
1
]
.
Func
==
"nodata"
{
//nodata重新查询索引来进行告警判断
...
...
@@ -421,6 +421,7 @@ func sendEvent(event *dataobj.Event) {
err
:=
redi
.
Push
(
event
)
if
err
!=
nil
{
stats
.
Counter
.
Set
(
"redis.push.failed"
,
1
)
logger
.
Errorf
(
"push event:%v err:%v"
,
event
,
err
)
}
}
...
...
src/modules/judge/stra/stra.go
浏览文件 @
178acfb4
...
...
@@ -56,11 +56,13 @@ func getStrategy(opts StrategySection) {
if
err
!=
nil
{
logger
.
Warningf
(
"get strategy from remote failed, error:%v"
,
err
)
stats
.
Counter
.
Set
(
"stra.get.err"
,
1
)
continue
}
if
resp
.
Err
!=
""
{
logger
.
Warningf
(
"get strategy from remote failed, error:%v"
,
resp
.
Err
)
stats
.
Counter
.
Set
(
"stra.get.err"
,
1
)
continue
}
...
...
src/modules/monapi/cron/checker_judge.go
浏览文件 @
178acfb4
...
...
@@ -11,6 +11,7 @@ import (
"github.com/didi/nightingale/src/model"
"github.com/didi/nightingale/src/modules/monapi/config"
"github.com/didi/nightingale/src/modules/monapi/scache"
"github.com/didi/nightingale/src/toolkits/stats"
)
func
CheckJudgeLoop
()
{
...
...
@@ -19,6 +20,7 @@ func CheckJudgeLoop() {
time
.
Sleep
(
duration
)
err
:=
CheckJudge
()
if
err
!=
nil
{
stats
.
Counter
.
Set
(
"get.judge.err"
,
1
)
logger
.
Error
(
"check judge fail: "
,
err
)
}
}
...
...
src/modules/monapi/redisc/redis.go
浏览文件 @
178acfb4
...
...
@@ -7,6 +7,7 @@ import (
"github.com/toolkits/pkg/logger"
"github.com/didi/nightingale/src/modules/monapi/config"
"github.com/didi/nightingale/src/toolkits/stats"
)
var
RedisConnPool
*
redis
.
Pool
...
...
@@ -29,6 +30,8 @@ func InitRedis() {
Dial
:
func
()
(
redis
.
Conn
,
error
)
{
c
,
err
:=
redis
.
Dial
(
"tcp"
,
addr
,
redis
.
DialConnectTimeout
(
connTimeout
),
redis
.
DialReadTimeout
(
readTimeout
),
redis
.
DialWriteTimeout
(
writeTimeout
))
if
err
!=
nil
{
logger
.
Errorf
(
"conn redis err:%v"
,
err
)
stats
.
Counter
.
Set
(
"redis.conn.failed"
,
1
)
return
nil
,
err
}
...
...
@@ -36,6 +39,7 @@ func InitRedis() {
if
_
,
err
:=
c
.
Do
(
"AUTH"
,
pass
);
err
!=
nil
{
c
.
Close
()
logger
.
Error
(
"redis auth fail, pass: "
,
pass
)
stats
.
Counter
.
Set
(
"redis.conn.failed"
,
1
)
return
nil
,
err
}
}
...
...
src/modules/transfer/backend/query.go
浏览文件 @
178acfb4
...
...
@@ -12,6 +12,7 @@ import (
"github.com/didi/nightingale/src/dataobj"
"github.com/didi/nightingale/src/modules/transfer/calc"
"github.com/didi/nightingale/src/toolkits/address"
"github.com/didi/nightingale/src/toolkits/stats"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/net/httplib"
...
...
@@ -161,10 +162,12 @@ func fetchDataSync(start, end int64, consolFun, endpoint, counter string, step i
defer
func
()
{
<-
worker
}()
stats
.
Counter
.
Set
(
"query.tsdb"
,
1
)
data
,
err
:=
fetchData
(
start
,
end
,
consolFun
,
endpoint
,
counter
,
step
)
if
err
!=
nil
{
logger
.
Warning
(
err
)
stats
.
Counter
.
Set
(
"query.data.err"
,
1
)
}
dataChan
<-
data
return
...
...
src/modules/transfer/backend/sender.go
浏览文件 @
178acfb4
...
...
@@ -107,6 +107,7 @@ func Send2TsdbTask(Q *list.SafeListLimited, node string, addr string, concurrent
// 将数据 打入 某个Tsdb的发送缓存队列, 具体是哪一个Tsdb 由一致性哈希 决定
func
Push2TsdbSendQueue
(
items
[]
*
dataobj
.
MetricValue
)
{
errCnt
:=
0
for
_
,
item
:=
range
items
{
tsdbItem
:=
convert2TsdbItem
(
item
)
stats
.
Counter
.
Set
(
"tsdb.queue.push"
,
1
)
...
...
@@ -118,19 +119,18 @@ func Push2TsdbSendQueue(items []*dataobj.MetricValue) {
}
cnode
:=
Config
.
ClusterList
[
node
]
errCnt
:=
0
for
_
,
addr
:=
range
cnode
.
Addrs
{
Q
:=
TsdbQueues
[
node
+
addr
]
if
!
Q
.
PushFront
(
tsdbItem
)
{
errCnt
+=
1
}
}
}
// statistics
if
errCnt
>
0
{
stats
.
Counter
.
Set
(
"tsdb.queue.err"
,
errCnt
)
logger
.
Error
(
"Push2TsdbSendQueue err num: "
,
errCnt
)
}
// statistics
if
errCnt
>
0
{
stats
.
Counter
.
Set
(
"tsdb.queue.err"
,
errCnt
)
logger
.
Error
(
"Push2TsdbSendQueue err num: "
,
errCnt
)
}
}
...
...
@@ -172,7 +172,7 @@ func Send2JudgeTask(Q *list.SafeListLimited, addr string, concurrent int) {
if
!
sendOk
{
stats
.
Counter
.
Set
(
"points.out.judge.err"
,
1
)
logger
.
Errorf
(
"send
judge %s fail: %v"
,
addr
,
err
)
logger
.
Errorf
(
"send
%v to judge %s fail: %v"
,
judgeItems
,
addr
,
err
)
}
}(
addr
,
judgeItems
,
count
)
...
...
@@ -180,6 +180,7 @@ func Send2JudgeTask(Q *list.SafeListLimited, addr string, concurrent int) {
}
func
Push2JudgeSendQueue
(
items
[]
*
dataobj
.
MetricValue
)
{
errCnt
:=
0
for
_
,
item
:=
range
items
{
key
:=
str
.
PK
(
item
.
Metric
,
item
.
Endpoint
)
stras
:=
cache
.
StraMap
.
GetByKey
(
key
)
...
...
@@ -203,11 +204,13 @@ func Push2JudgeSendQueue(items []*dataobj.MetricValue) {
q
,
exists
:=
JudgeQueues
.
Get
(
stra
.
JudgeInstance
)
if
exists
{
q
.
PushFront
(
judgeItem
)
if
!
q
.
PushFront
(
judgeItem
)
{
errCnt
+=
1
}
}
}
}
stats
.
Counter
.
Set
(
"judge.queue.err"
,
errCnt
)
}
// 打到Tsdb的数据,要根据rrdtool的特定 来限制 step、counterType、timestamp
...
...
src/modules/transfer/rpc/push.go
浏览文件 @
178acfb4
...
...
@@ -27,9 +27,10 @@ func (t *Transfer) Push(args []*dataobj.MetricValue, reply *dataobj.TransferResp
err
:=
v
.
CheckValidity
()
if
err
!=
nil
{
stats
.
Counter
.
Set
(
"points.in.err"
,
1
)
logger
.
Warningf
(
"item is illegal item:%s err:%v"
,
v
,
err
)
msg
:=
fmt
.
Sprintf
(
"item is illegal item:%s err:%v"
,
v
,
err
)
logger
.
Warningf
(
msg
)
reply
.
Invalid
+=
1
reply
.
Msg
+=
fmt
.
Sprintf
(
"%v
\n
"
,
err
)
reply
.
Msg
+=
msg
continue
}
...
...
src/modules/tsdb/index/index.go
浏览文件 @
178acfb4
...
...
@@ -47,7 +47,7 @@ func GetIndexLoop() {
func
GetIndex
()
{
instances
,
err
:=
report
.
GetAlive
(
"index"
,
Config
.
HbsMod
)
if
err
!=
nil
{
stats
.
Counter
.
Set
(
"
index.get
.err"
,
1
)
stats
.
Counter
.
Set
(
"
get.index
.err"
,
1
)
logger
.
Warningf
(
"get index list err:%v"
,
err
)
return
}
...
...
src/modules/tsdb/rpc/push.go
浏览文件 @
178acfb4
...
...
@@ -58,7 +58,6 @@ func handleItems(items []*dataobj.TsdbItem) {
//todo hash冲突问题需要解决
if
err
:=
cache
.
Caches
.
Push
(
item
.
Key
,
item
.
Timestamp
,
item
.
Value
);
err
!=
nil
{
stats
.
Counter
.
Set
(
"points.in.err"
,
1
)
logger
.
Warningf
(
"push obj error, obj: %v, error: %v
\n
"
,
items
[
i
],
err
)
fail
++
}
...
...
src/modules/tsdb/rrdtool/sync_disk.go
浏览文件 @
178acfb4
...
...
@@ -231,6 +231,7 @@ func FlushRRD(flushChunks map[interface{}][]*cache.Chunk) {
err
:=
FlushFile
(
seriesID
,
items
)
if
err
!=
nil
{
stats
.
Counter
.
Set
(
"flush.rrd.err"
,
1
)
logger
.
Errorf
(
"flush %v data to rrd err:%v"
,
seriesID
,
err
)
continue
}
...
...
src/toolkits/identity/identity.go
浏览文件 @
178acfb4
...
...
@@ -18,11 +18,12 @@ type IdentitySection struct {
func
Init
(
identity
IdentitySection
)
{
if
identity
.
Specify
!=
""
{
Identity
=
identity
.
Specify
return
}
var
err
error
Identity
,
err
=
sys
.
CmdOutTrim
(
"bash"
,
"-c"
,
identity
.
Shell
)
if
err
!=
nil
{
log
.
Fatalln
(
"[F] cannot get
hostname
"
)
log
.
Fatalln
(
"[F] cannot get
identity
"
)
}
}
src/toolkits/stats/init.go
浏览文件 @
178acfb4
...
...
@@ -7,7 +7,6 @@ import (
"time"
"github.com/didi/nightingale/src/dataobj"
"github.com/didi/nightingale/src/toolkits/identity"
"github.com/toolkits/pkg/logger"
)
...
...
@@ -42,7 +41,6 @@ func Push() {
func
NewMetricValue
(
metric
string
,
value
int64
)
*
dataobj
.
MetricValue
{
item
:=
&
dataobj
.
MetricValue
{
Metric
:
metric
,
Endpoint
:
identity
.
Identity
,
Timestamp
:
time
.
Now
()
.
Unix
(),
ValueUntyped
:
value
,
CounterType
:
"GAUGE"
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录