Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
74fd28df
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
74fd28df
编写于
7月 21, 2022
作者:
S
SimFG
提交者:
GitHub
7月 21, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Refine some datanode code (#18351)
Signed-off-by:
N
SimFG
<
bang.fu@zilliz.com
>
上级
a6e1ddb5
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
33 addition
and
72 deletion
+33
-72
internal/datanode/compactor.go
internal/datanode/compactor.go
+13
-13
internal/datanode/data_node.go
internal/datanode/data_node.go
+11
-23
internal/datanode/data_sync_service.go
internal/datanode/data_sync_service.go
+4
-6
internal/datanode/meta_service.go
internal/datanode/meta_service.go
+4
-22
internal/datanode/segment_replica.go
internal/datanode/segment_replica.go
+1
-8
未找到文件。
internal/datanode/compactor.go
浏览文件 @
74fd28df
...
...
@@ -168,12 +168,12 @@ func (t *compactionTask) mergeDeltalogs(dBlobs map[UniqueID][]*Blob, timetravelT
dbuff
.
delData
.
Append
(
pk
,
ts
)
if
Timestamp
(
ts
)
<
dbuff
.
TimestampFrom
{
dbuff
.
TimestampFrom
=
Timestamp
(
ts
)
if
ts
<
dbuff
.
TimestampFrom
{
dbuff
.
TimestampFrom
=
ts
}
if
Timestamp
(
ts
)
>
dbuff
.
TimestampTo
{
dbuff
.
TimestampTo
=
Timestamp
(
ts
)
if
ts
>
dbuff
.
TimestampTo
{
dbuff
.
TimestampTo
=
ts
}
}
}
...
...
@@ -181,7 +181,7 @@ func (t *compactionTask) mergeDeltalogs(dBlobs map[UniqueID][]*Blob, timetravelT
dbuff
.
updateSize
(
dbuff
.
delData
.
RowCount
)
log
.
Debug
(
"mergeDeltalogs end"
,
zap
.
Int64
(
"PlanID"
,
t
.
getPlanID
()),
zap
.
Int
(
"number of pks to compact in insert logs"
,
len
(
pk2ts
)),
zap
.
Any
(
"elapse in ms"
,
nano2Milli
(
time
.
Since
(
mergeStart
))))
zap
.
Float64
(
"elapse in ms"
,
nano2Milli
(
time
.
Since
(
mergeStart
))))
return
pk2ts
,
dbuff
,
nil
}
...
...
@@ -310,7 +310,7 @@ func (t *compactionTask) merge(mergeItr iterator, delta map[primaryKey]Timestamp
log
.
Debug
(
"merge end"
,
zap
.
Int64
(
"planID"
,
t
.
getPlanID
()),
zap
.
Int64
(
"remaining insert numRows"
,
numRows
),
zap
.
Int64
(
"expired entities"
,
expired
),
zap
.
Any
(
"elapse in ms"
,
nano2Milli
(
time
.
Since
(
mergeStart
))))
zap
.
Float64
(
"elapse in ms"
,
nano2Milli
(
time
.
Since
(
mergeStart
))))
return
iDatas
,
numRows
,
nil
}
...
...
@@ -347,7 +347,7 @@ func (t *compactionTask) compact() error {
targetSegID
=
t
.
plan
.
GetSegmentBinlogs
()[
0
]
.
GetSegmentID
()
}
log
.
Debug
(
"compaction start"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"timeout in seconds"
,
t
.
plan
.
GetTimeoutInSeconds
()))
log
.
Debug
(
"compaction start"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Int32
(
"timeout in seconds"
,
t
.
plan
.
GetTimeoutInSeconds
()))
segIDs
:=
make
([]
UniqueID
,
0
,
len
(
t
.
plan
.
GetSegmentBinlogs
()))
for
_
,
s
:=
range
t
.
plan
.
GetSegmentBinlogs
()
{
segIDs
=
append
(
segIDs
,
s
.
GetSegmentID
())
...
...
@@ -370,7 +370,7 @@ func (t *compactionTask) compact() error {
<-
ti
.
Injected
()
injectEnd
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"inject elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"elapse"
,
nano2Milli
(
injectEnd
.
Sub
(
injectStart
))))
log
.
Debug
(
"inject elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Float64
(
"elapse"
,
nano2Milli
(
injectEnd
.
Sub
(
injectStart
))))
}()
var
(
...
...
@@ -463,7 +463,7 @@ func (t *compactionTask) compact() error {
err
=
g
.
Wait
()
downloadEnd
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"download elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"elapse"
,
nano2Milli
(
downloadEnd
.
Sub
(
downloadStart
))))
log
.
Debug
(
"download elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Float64
(
"elapse"
,
nano2Milli
(
downloadEnd
.
Sub
(
downloadStart
))))
}()
if
err
!=
nil
{
...
...
@@ -493,7 +493,7 @@ func (t *compactionTask) compact() error {
uploadEnd
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"upload elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"elapse"
,
nano2Milli
(
uploadEnd
.
Sub
(
uploadStart
))))
log
.
Debug
(
"upload elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Float64
(
"elapse"
,
nano2Milli
(
uploadEnd
.
Sub
(
uploadStart
))))
}()
for
_
,
fbl
:=
range
segPaths
.
deltaInfo
{
...
...
@@ -526,7 +526,7 @@ func (t *compactionTask) compact() error {
}
rpcEnd
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"rpc elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"elapse"
,
nano2Milli
(
rpcEnd
.
Sub
(
rpcStart
))))
log
.
Debug
(
"rpc elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Float64
(
"elapse"
,
nano2Milli
(
rpcEnd
.
Sub
(
rpcStart
))))
}()
// Compaction I: update pk range.
...
...
@@ -550,7 +550,7 @@ func (t *compactionTask) compact() error {
ti
.
injectDone
(
true
)
uninjectEnd
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"uninject elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"elapse"
,
nano2Milli
(
uninjectEnd
.
Sub
(
uninjectStart
))))
log
.
Debug
(
"uninject elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Float64
(
"elapse"
,
nano2Milli
(
uninjectEnd
.
Sub
(
uninjectStart
))))
}()
log
.
Info
(
"compaction done"
,
...
...
@@ -561,7 +561,7 @@ func (t *compactionTask) compact() error {
zap
.
Int
(
"num of delta paths"
,
len
(
segPaths
.
deltaInfo
)),
)
log
.
Info
(
"overall elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Any
(
"elapse"
,
nano2Milli
(
time
.
Since
(
compactStart
))))
log
.
Info
(
"overall elapse in ms"
,
zap
.
Int64
(
"planID"
,
t
.
plan
.
GetPlanID
()),
zap
.
Float64
(
"elapse"
,
nano2Milli
(
time
.
Since
(
compactStart
))))
metrics
.
DataNodeCompactionLatency
.
WithLabelValues
(
fmt
.
Sprint
(
Params
.
DataNodeCfg
.
GetNodeID
()))
.
Observe
(
float64
(
t
.
tr
.
ElapseSpan
()
.
Milliseconds
()))
return
nil
...
...
internal/datanode/data_node.go
浏览文件 @
74fd28df
...
...
@@ -457,9 +457,9 @@ func (node *DataNode) Start() error {
},
Count
:
1
,
})
if
err
!=
nil
{
log
.
Warn
(
"fail to alloc timestamp"
,
zap
.
Error
(
err
))
return
err
if
err
!=
nil
||
rep
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_Success
{
log
.
Warn
(
"fail to alloc timestamp"
,
zap
.
Any
(
"rep"
,
rep
),
zap
.
Error
(
err
))
return
err
ors
.
New
(
"DataNode fail to alloc timestamp"
)
}
connectEtcdFn
:=
func
()
error
{
...
...
@@ -480,10 +480,6 @@ func (node *DataNode) Start() error {
node
.
chunkManager
=
chunkManager
if
rep
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_Success
||
err
!=
nil
{
return
errors
.
New
(
"DataNode fail to start"
)
}
go
node
.
BackGroundGC
(
node
.
clearSignal
)
go
node
.
compactionExecutor
.
start
(
node
.
ctx
)
...
...
@@ -509,8 +505,7 @@ func (node *DataNode) GetStateCode() internalpb.StateCode {
}
func
(
node
*
DataNode
)
isHealthy
()
bool
{
code
:=
node
.
State
.
Load
()
.
(
internalpb
.
StateCode
)
return
code
==
internalpb
.
StateCode_Healthy
return
node
.
GetStateCode
()
==
internalpb
.
StateCode_Healthy
}
// WatchDmChannels is not in use
...
...
@@ -543,9 +538,9 @@ func (node *DataNode) GetComponentStates(ctx context.Context) (*internalpb.Compo
return
states
,
nil
}
// ReadyToFlush tells wether DataNode is ready for flushing
// ReadyToFlush tells w
h
ether DataNode is ready for flushing
func
(
node
*
DataNode
)
ReadyToFlush
()
error
{
if
node
.
State
.
Load
()
.
(
internalpb
.
StateCode
)
!=
internalpb
.
StateCode_Healthy
{
if
!
node
.
isHealthy
()
{
return
errors
.
New
(
"DataNode not in HEALTHY state"
)
}
return
nil
...
...
@@ -565,7 +560,7 @@ func (node *DataNode) FlushSegments(ctx context.Context, req *datapb.FlushSegmen
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
}
if
node
.
State
.
Load
()
.
(
internalpb
.
StateCode
)
!=
internalpb
.
StateCode_Healthy
{
if
!
node
.
isHealthy
()
{
errStatus
.
Reason
=
"dataNode not in HEALTHY state"
return
errStatus
,
nil
}
...
...
@@ -653,7 +648,6 @@ func (node *DataNode) ResendSegmentStats(ctx context.Context, req *datapb.Resend
return
&
datapb
.
ResendSegmentStatsResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
,
Reason
:
""
,
},
SegResent
:
segResent
,
},
nil
...
...
@@ -689,9 +683,7 @@ func (node *DataNode) GetTimeTickChannel(ctx context.Context) (*milvuspb.StringR
return
&
milvuspb
.
StringResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
,
Reason
:
""
,
},
Value
:
""
,
},
nil
}
...
...
@@ -700,9 +692,7 @@ func (node *DataNode) GetStatisticsChannel(ctx context.Context) (*milvuspb.Strin
return
&
milvuspb
.
StringResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
,
Reason
:
""
,
},
Value
:
""
,
},
nil
}
...
...
@@ -724,7 +714,6 @@ func (node *DataNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRe
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
Reason
:
msgDataNodeIsUnhealthy
(
Params
.
DataNodeCfg
.
GetNodeID
()),
},
Response
:
""
,
},
nil
}
...
...
@@ -740,7 +729,6 @@ func (node *DataNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRe
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
Reason
:
err
.
Error
(),
},
Response
:
""
,
},
nil
}
...
...
@@ -770,7 +758,6 @@ func (node *DataNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRe
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
Reason
:
metricsinfo
.
MsgUnimplementedMetric
,
},
Response
:
""
,
},
nil
}
...
...
@@ -818,8 +805,8 @@ func (node *DataNode) Import(ctx context.Context, req *datapb.ImportTaskRequest)
zap
.
Int64
(
"task ID"
,
req
.
GetImportTask
()
.
GetTaskId
()),
zap
.
Int64
(
"collection ID"
,
req
.
GetImportTask
()
.
GetCollectionId
()),
zap
.
Int64
(
"partition ID"
,
req
.
GetImportTask
()
.
GetPartitionId
()),
zap
.
Any
(
"channel names"
,
req
.
GetImportTask
()
.
GetChannelNames
()),
zap
.
Any
(
"working dataNodes"
,
req
.
WorkingNodes
))
zap
.
Strings
(
"channel names"
,
req
.
GetImportTask
()
.
GetChannelNames
()),
zap
.
Int64s
(
"working dataNodes"
,
req
.
WorkingNodes
))
defer
func
()
{
log
.
Info
(
"DataNode finish import request"
,
zap
.
Int64
(
"task ID"
,
req
.
GetImportTask
()
.
GetTaskId
()))
}()
...
...
@@ -956,6 +943,7 @@ func (node *DataNode) AddSegment(ctx context.Context, req *datapb.AddSegmentRequ
return
&
commonpb
.
Status
{
// TODO: Add specific error code.
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
Reason
:
err
.
Error
(),
},
nil
}
}
...
...
@@ -972,7 +960,7 @@ func importFlushReqFunc(node *DataNode, req *datapb.ImportTaskRequest, res *root
log
.
Error
(
"import task returns invalid shard number"
,
zap
.
Int
(
"shard num"
,
shardNum
),
zap
.
Int
(
"# of channels"
,
len
(
req
.
GetImportTask
()
.
GetChannelNames
())),
zap
.
Any
(
"channel names"
,
req
.
GetImportTask
()
.
GetChannelNames
()),
zap
.
Strings
(
"channel names"
,
req
.
GetImportTask
()
.
GetChannelNames
()),
)
return
fmt
.
Errorf
(
"syncSegmentID Failed: invalid shard number %d"
,
shardNum
)
}
...
...
internal/datanode/data_sync_service.go
浏览文件 @
74fd28df
...
...
@@ -131,14 +131,14 @@ func newParallelConfig() parallelConfig {
return
parallelConfig
{
Params
.
DataNodeCfg
.
FlowGraphMaxQueueLength
,
Params
.
DataNodeCfg
.
FlowGraphMaxParallelism
}
}
// start starts the flowgraph in datasyncservice
// start starts the flow
graph in datasyncservice
func
(
dsService
*
dataSyncService
)
start
()
{
if
dsService
.
fg
!=
nil
{
log
.
Info
(
"dataSyncService starting flowgraph"
,
zap
.
Int64
(
"collectionID"
,
dsService
.
collectionID
),
log
.
Info
(
"dataSyncService starting flow
graph"
,
zap
.
Int64
(
"collectionID"
,
dsService
.
collectionID
),
zap
.
String
(
"vChanName"
,
dsService
.
vchannelName
))
dsService
.
fg
.
Start
()
}
else
{
log
.
Warn
(
"dataSyncService starting flowgraph is nil"
,
zap
.
Int64
(
"collectionID"
,
dsService
.
collectionID
),
log
.
Warn
(
"dataSyncService starting flow
graph is nil"
,
zap
.
Int64
(
"collectionID"
,
dsService
.
collectionID
),
zap
.
String
(
"vChanName"
,
dsService
.
vchannelName
))
}
}
...
...
@@ -351,7 +351,6 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo) erro
// getSegmentInfos return the SegmentInfo details according to the given ids through RPC to datacoord
func
(
dsService
*
dataSyncService
)
getSegmentInfos
(
segmentIDs
[]
int64
)
([]
*
datapb
.
SegmentInfo
,
error
)
{
var
segmentInfos
[]
*
datapb
.
SegmentInfo
infoResp
,
err
:=
dsService
.
dataCoord
.
GetSegmentInfo
(
dsService
.
ctx
,
&
datapb
.
GetSegmentInfoRequest
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
commonpb
.
MsgType_SegmentInfo
,
...
...
@@ -371,6 +370,5 @@ func (dsService *dataSyncService) getSegmentInfos(segmentIDs []int64) ([]*datapb
log
.
Error
(
"Fail to get datapb.SegmentInfo by ids from datacoord"
,
zap
.
Error
(
err
))
return
nil
,
err
}
segmentInfos
=
infoResp
.
Infos
return
segmentInfos
,
nil
return
infoResp
.
Infos
,
nil
}
internal/datanode/meta_service.go
浏览文件 @
74fd28df
...
...
@@ -46,31 +46,13 @@ func newMetaService(rc types.RootCoord, collectionID UniqueID) *metaService {
}
}
// TODO: Replace with getCollectionInfo below.
// getCollectionSchema get collection schema with provided collection id at specified timestamp.
func
(
mService
*
metaService
)
getCollectionSchema
(
ctx
context
.
Context
,
collID
UniqueID
,
timestamp
Timestamp
)
(
*
schemapb
.
CollectionSchema
,
error
)
{
req
:=
&
milvuspb
.
DescribeCollectionRequest
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
commonpb
.
MsgType_DescribeCollection
,
MsgID
:
0
,
//GOOSE TODO
Timestamp
:
0
,
// GOOSE TODO
SourceID
:
Params
.
DataNodeCfg
.
GetNodeID
(),
},
DbName
:
"default"
,
// GOOSE TODO
CollectionID
:
collID
,
TimeStamp
:
timestamp
,
response
,
err
:=
mService
.
getCollectionInfo
(
ctx
,
collID
,
timestamp
)
if
response
!=
nil
{
return
response
.
GetSchema
(),
err
}
response
,
err
:=
mService
.
rootCoord
.
DescribeCollection
(
ctx
,
req
)
if
err
!=
nil
{
return
nil
,
fmt
.
Errorf
(
"grpc error when describe collection %v from rootcoord: %s"
,
collID
,
err
.
Error
())
}
if
response
.
GetStatus
()
.
GetErrorCode
()
!=
commonpb
.
ErrorCode_Success
{
return
nil
,
fmt
.
Errorf
(
"describe collection %v from rootcoord wrong: %s"
,
collID
,
response
.
GetStatus
()
.
GetReason
())
}
return
response
.
GetSchema
(),
nil
return
nil
,
err
}
// getCollectionInfo get collection info with provided collection id at specified timestamp.
...
...
internal/datanode/segment_replica.go
浏览文件 @
74fd28df
...
...
@@ -298,14 +298,7 @@ func (replica *SegmentReplica) listCompactedSegmentIDs() map[UniqueID][]UniqueID
compactedTo2From
:=
make
(
map
[
UniqueID
][]
UniqueID
)
for
segID
,
seg
:=
range
replica
.
compactedSegments
{
var
from
[]
UniqueID
from
,
ok
:=
compactedTo2From
[
seg
.
compactedTo
]
if
!
ok
{
from
=
[]
UniqueID
{}
}
from
=
append
(
from
,
segID
)
compactedTo2From
[
seg
.
compactedTo
]
=
from
compactedTo2From
[
seg
.
compactedTo
]
=
append
(
compactedTo2From
[
seg
.
compactedTo
],
segID
)
}
return
compactedTo2From
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录