Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
a03ab05c
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a03ab05c
编写于
12月 25, 2020
作者:
S
sunby
提交者:
yefu.chen
12月 25, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add DescribeIndex and DescribeIndexProgress RPC
Signed-off-by:
N
sunby
<
bingyi.sun@zilliz.com
>
上级
84a0cf70
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
199 addition
and
30 deletion
+199
-30
internal/master/grpc_service.go
internal/master/grpc_service.go
+68
-16
internal/master/index_task.go
internal/master/index_task.go
+87
-0
internal/master/runtime_stats.go
internal/master/runtime_stats.go
+36
-6
internal/master/runtime_stats_test.go
internal/master/runtime_stats_test.go
+2
-2
internal/master/stats_processor.go
internal/master/stats_processor.go
+4
-4
internal/master/stats_processor_test.go
internal/master/stats_processor_test.go
+2
-2
未找到文件。
internal/master/grpc_service.go
浏览文件 @
a03ab05c
...
...
@@ -2,6 +2,7 @@ package master
import
(
"context"
"fmt"
"time"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
...
...
@@ -444,6 +445,9 @@ func (s *Master) AssignSegmentID(ctx context.Context, request *internalpb.Assign
}
func
(
s
*
Master
)
CreateIndex
(
ctx
context
.
Context
,
req
*
internalpb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
{
ret
:=
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
}
task
:=
&
createIndexTask
{
baseTask
:
baseTask
{
sch
:
s
.
scheduler
,
...
...
@@ -458,30 +462,78 @@ func (s *Master) CreateIndex(ctx context.Context, req *internalpb.CreateIndexReq
err
:=
s
.
scheduler
.
Enqueue
(
task
)
if
err
!=
nil
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
Reason
:
"Enqueue failed: "
+
err
.
Error
(),
},
nil
ret
.
Reason
=
"Enqueue failed: "
+
err
.
Error
()
return
ret
,
nil
}
err
=
task
.
WaitToFinish
(
ctx
)
if
err
!=
nil
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
Reason
:
"Create Index error: "
+
err
.
Error
(),
},
nil
ret
.
Reason
=
"Create Index error: "
+
err
.
Error
()
return
ret
,
nil
}
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
Reason
:
""
,
},
nil
ret
.
ErrorCode
=
commonpb
.
ErrorCode_SUCCESS
return
ret
,
nil
}
func
(
s
*
Master
)
DescribeIndex
(
context
.
Context
,
*
internalpb
.
DescribeIndexRequest
)
(
*
servicepb
.
DescribeIndexResponse
,
error
)
{
return
nil
,
nil
func
(
s
*
Master
)
DescribeIndex
(
ctx
context
.
Context
,
req
*
internalpb
.
DescribeIndexRequest
)
(
*
servicepb
.
DescribeIndexResponse
,
error
)
{
resp
:=
&
servicepb
.
DescribeIndexResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
},
CollectionName
:
req
.
CollectionName
,
FieldName
:
req
.
FieldName
,
}
task
:=
&
describeIndexTask
{
baseTask
:
baseTask
{
sch
:
s
.
scheduler
,
mt
:
s
.
metaTable
,
cv
:
make
(
chan
error
),
},
req
:
req
,
resp
:
resp
,
}
if
err
:=
s
.
scheduler
.
Enqueue
(
task
);
err
!=
nil
{
resp
.
Status
.
Reason
=
fmt
.
Sprintf
(
"Enqueue failed: %s"
,
err
.
Error
())
return
resp
,
nil
}
if
err
:=
task
.
WaitToFinish
(
ctx
);
err
!=
nil
{
resp
.
Status
.
Reason
=
fmt
.
Sprintf
(
"Describe Index failed: %s"
,
err
.
Error
())
return
resp
,
nil
}
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_SUCCESS
return
resp
,
nil
}
func
(
s
*
Master
)
DescribeIndexProgress
(
context
.
Context
,
*
internalpb
.
DescribeIndexProgressRequest
)
(
*
servicepb
.
BoolResponse
,
error
)
{
return
nil
,
nil
func
(
s
*
Master
)
DescribeIndexProgress
(
ctx
context
.
Context
,
req
*
internalpb
.
DescribeIndexProgressRequest
)
(
*
servicepb
.
BoolResponse
,
error
)
{
resp
:=
&
servicepb
.
BoolResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
},
Value
:
false
,
}
task
:=
&
describeIndexProgressTask
{
baseTask
:
baseTask
{
sch
:
s
.
scheduler
,
mt
:
s
.
metaTable
,
cv
:
make
(
chan
error
),
},
req
:
req
,
resp
:
resp
,
}
if
err
:=
s
.
scheduler
.
Enqueue
(
task
);
err
!=
nil
{
resp
.
Status
.
Reason
=
"Enqueue failed :"
+
err
.
Error
()
return
resp
,
nil
}
if
err
:=
task
.
WaitToFinish
(
ctx
);
err
!=
nil
{
resp
.
Status
.
Reason
=
"Describe index progress failed:"
+
err
.
Error
()
return
resp
,
nil
}
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_SUCCESS
return
resp
,
nil
}
internal/master/index_task.go
浏览文件 @
a03ab05c
...
...
@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
)
type
createIndexTask
struct
{
...
...
@@ -93,3 +94,89 @@ func (task *createIndexTask) Execute() error {
// close unfilled segment
return
task
.
segManager
.
ForceClose
(
collMeta
.
ID
)
}
type
describeIndexTask
struct
{
baseTask
req
*
internalpb
.
DescribeIndexRequest
resp
*
servicepb
.
DescribeIndexResponse
}
func
(
task
*
describeIndexTask
)
Type
()
internalpb
.
MsgType
{
return
internalpb
.
MsgType_kDescribeIndex
}
func
(
task
*
describeIndexTask
)
Ts
()
(
Timestamp
,
error
)
{
return
task
.
req
.
Timestamp
,
nil
}
func
(
task
*
describeIndexTask
)
Execute
()
error
{
collMeta
,
err
:=
task
.
mt
.
GetCollectionByName
(
task
.
req
.
CollectionName
)
if
err
!=
nil
{
return
err
}
var
fieldID
int64
=
-
1
for
_
,
fieldSchema
:=
range
collMeta
.
Schema
.
Fields
{
if
fieldSchema
.
Name
==
task
.
req
.
FieldName
{
fieldID
=
fieldSchema
.
FieldID
break
}
}
if
fieldID
==
-
1
{
return
fmt
.
Errorf
(
"can not find field %s"
,
task
.
req
.
FieldName
)
}
indexParams
,
err
:=
task
.
mt
.
GetFieldIndexParams
(
collMeta
.
ID
,
fieldID
)
if
err
!=
nil
{
return
err
}
task
.
resp
.
ExtraParams
=
indexParams
return
nil
}
type
describeIndexProgressTask
struct
{
baseTask
req
*
internalpb
.
DescribeIndexProgressRequest
runtimeStats
*
RuntimeStats
resp
*
servicepb
.
BoolResponse
}
func
(
task
*
describeIndexProgressTask
)
Type
()
internalpb
.
MsgType
{
return
internalpb
.
MsgType_kDescribeIndexProgress
}
func
(
task
*
describeIndexProgressTask
)
Ts
()
(
Timestamp
,
error
)
{
return
task
.
req
.
Timestamp
,
nil
}
func
(
task
*
describeIndexProgressTask
)
Execute
()
error
{
// get field id, collection id
collMeta
,
err
:=
task
.
mt
.
GetCollectionByName
(
task
.
req
.
CollectionName
)
if
err
!=
nil
{
return
err
}
var
fieldID
int64
=
-
1
for
_
,
fieldSchema
:=
range
collMeta
.
Schema
.
Fields
{
if
fieldSchema
.
Name
==
task
.
req
.
FieldName
{
fieldID
=
fieldSchema
.
FieldID
break
}
}
if
fieldID
==
-
1
{
return
fmt
.
Errorf
(
"can not find field %s"
,
task
.
req
.
FieldName
)
}
// total segment nums
totalSegmentNums
:=
len
(
collMeta
.
SegmentIDs
)
// get completed segment nums from querynode's runtime stats
relatedSegments
:=
task
.
runtimeStats
.
GetTotalNumOfRelatedSegments
(
collMeta
.
ID
,
fieldID
,
task
.
req
.
ExtraParams
)
if
int64
(
totalSegmentNums
)
==
relatedSegments
{
task
.
resp
.
Value
=
true
}
else
{
task
.
resp
.
Value
=
false
}
return
nil
}
internal/master/runtime_stats.go
浏览文件 @
a03ab05c
package
master
import
(
"sync"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
type
RuntimeStats
struct
{
collStats
map
[
UniqueID
]
*
CollRuntimeStats
// collection id to array of field statistic
mu
sync
.
RWMutex
}
func
(
rs
*
RuntimeStats
)
UpdateFieldStat
(
collID
UniqueID
,
fieldID
UniqueID
,
stats
*
FieldRuntimeStats
)
error
{
func
(
rs
*
RuntimeStats
)
UpdateFieldStat
(
collID
UniqueID
,
fieldID
UniqueID
,
stats
*
FieldIndexRuntimeStats
)
error
{
rs
.
mu
.
Lock
()
defer
rs
.
mu
.
Unlock
()
peerID
:=
stats
.
peerID
_
,
ok
:=
rs
.
collStats
[
collID
]
if
!
ok
{
rs
.
collStats
[
collID
]
=
&
CollRuntimeStats
{
field
Stats
:
make
(
map
[
UniqueID
][]
*
Field
RuntimeStats
),
field
IndexStats
:
make
(
map
[
UniqueID
][]
*
FieldIndex
RuntimeStats
),
}
}
collRuntimeStats
:=
rs
.
collStats
[
collID
]
fieldStats
:=
collRuntimeStats
.
fieldStats
[
fieldID
]
fieldStats
:=
collRuntimeStats
.
field
Index
Stats
[
fieldID
]
for
i
,
v
:=
range
fieldStats
{
if
v
.
peerID
==
peerID
&&
typeutil
.
CompareIndexParams
(
v
.
indexParams
,
stats
.
indexParams
)
{
fieldStats
[
i
]
=
stats
...
...
@@ -27,15 +33,39 @@ func (rs *RuntimeStats) UpdateFieldStat(collID UniqueID, fieldID UniqueID, stats
}
}
collRuntimeStats
.
field
Stats
[
fieldID
]
=
append
(
collRuntimeStats
.
field
Stats
[
fieldID
],
stats
)
collRuntimeStats
.
field
IndexStats
[
fieldID
]
=
append
(
collRuntimeStats
.
fieldIndex
Stats
[
fieldID
],
stats
)
return
nil
}
func
(
rs
*
RuntimeStats
)
GetTotalNumOfRelatedSegments
(
collID
UniqueID
,
fieldID
UniqueID
,
indexParams
[]
*
commonpb
.
KeyValuePair
)
int64
{
rs
.
mu
.
RLock
()
defer
rs
.
mu
.
RUnlock
()
collStats
,
ok
:=
rs
.
collStats
[
collID
]
if
!
ok
{
return
0
}
fieldStats
,
ok
:=
collStats
.
fieldIndexStats
[
fieldID
]
if
!
ok
{
return
0
}
var
total
int64
=
0
for
_
,
stat
:=
range
fieldStats
{
if
typeutil
.
CompareIndexParams
(
stat
.
indexParams
,
indexParams
)
{
total
+=
stat
.
numOfRelatedSegments
}
}
return
total
}
type
CollRuntimeStats
struct
{
field
Stats
map
[
UniqueID
][]
*
Field
RuntimeStats
field
IndexStats
map
[
UniqueID
][]
*
FieldIndex
RuntimeStats
}
type
FieldRuntimeStats
struct
{
type
Field
Index
RuntimeStats
struct
{
peerID
int64
indexParams
[]
*
commonpb
.
KeyValuePair
numOfRelatedSegments
int64
...
...
internal/master/runtime_stats_test.go
浏览文件 @
a03ab05c
...
...
@@ -22,13 +22,13 @@ func TestRuntimeStats_UpdateFieldStats(t *testing.T) {
{
1
,
1
,
2
,
100
},
}
for
_
,
testcase
:=
range
cases
{
err
:=
runtimeStats
.
UpdateFieldStat
(
testcase
.
collID
,
testcase
.
fieldID
,
&
FieldRuntimeStats
{
err
:=
runtimeStats
.
UpdateFieldStat
(
testcase
.
collID
,
testcase
.
fieldID
,
&
Field
Index
RuntimeStats
{
peerID
:
testcase
.
peerID
,
indexParams
:
[]
*
commonpb
.
KeyValuePair
{},
numOfRelatedSegments
:
testcase
.
nums
,
})
assert
.
Nil
(
t
,
err
)
statsArray
:=
runtimeStats
.
collStats
[
testcase
.
collID
]
.
fieldStats
[
testcase
.
fieldID
]
statsArray
:=
runtimeStats
.
collStats
[
testcase
.
collID
]
.
field
Index
Stats
[
testcase
.
fieldID
]
assert
.
NotEmpty
(
t
,
statsArray
)
found
:=
0
...
...
internal/master/stats_processor.go
浏览文件 @
a03ab05c
...
...
@@ -8,7 +8,7 @@ import (
type
StatsProcessor
struct
{
metaTable
*
metaTable
run
T
imeStats
*
RuntimeStats
run
t
imeStats
*
RuntimeStats
segmentThreshold
float64
segmentThresholdFactor
float64
...
...
@@ -61,13 +61,13 @@ func (processor *StatsProcessor) processFieldStat(peerID int64, fieldStats *inte
fieldID
:=
fieldStats
.
FieldID
for
_
,
stat
:=
range
fieldStats
.
IndexStats
{
fieldStats
:=
&
FieldRuntimeStats
{
fieldStats
:=
&
Field
Index
RuntimeStats
{
peerID
:
peerID
,
indexParams
:
stat
.
IndexParams
,
numOfRelatedSegments
:
stat
.
NumRelatedSegments
,
}
if
err
:=
processor
.
run
T
imeStats
.
UpdateFieldStat
(
collID
,
fieldID
,
fieldStats
);
err
!=
nil
{
if
err
:=
processor
.
run
t
imeStats
.
UpdateFieldStat
(
collID
,
fieldID
,
fieldStats
);
err
!=
nil
{
return
err
}
}
...
...
@@ -77,7 +77,7 @@ func (processor *StatsProcessor) processFieldStat(peerID int64, fieldStats *inte
func
NewStatsProcessor
(
mt
*
metaTable
,
runTimeStats
*
RuntimeStats
,
globalTSOAllocator
func
()
(
Timestamp
,
error
))
*
StatsProcessor
{
return
&
StatsProcessor
{
metaTable
:
mt
,
run
T
imeStats
:
runTimeStats
,
run
t
imeStats
:
runTimeStats
,
segmentThreshold
:
Params
.
SegmentSize
*
1024
*
1024
,
segmentThresholdFactor
:
Params
.
SegmentSizeFactor
,
globalTSOAllocator
:
globalTSOAllocator
,
...
...
internal/master/stats_processor_test.go
浏览文件 @
a03ab05c
...
...
@@ -110,7 +110,7 @@ func TestStatsProcess(t *testing.T) {
assert
.
Equal
(
t
,
int64
(
2500000
),
segMeta
.
MemSize
)
assert
.
Equal
(
t
,
int64
(
25000
),
segMeta
.
NumRows
)
assert
.
EqualValues
(
t
,
100
,
runtimeStats
.
collStats
[
1
]
.
fieldStats
[
100
][
0
]
.
numOfRelatedSegments
)
assert
.
EqualValues
(
t
,
200
,
runtimeStats
.
collStats
[
2
]
.
fieldStats
[
100
][
0
]
.
numOfRelatedSegments
)
assert
.
EqualValues
(
t
,
100
,
runtimeStats
.
collStats
[
1
]
.
field
Index
Stats
[
100
][
0
]
.
numOfRelatedSegments
)
assert
.
EqualValues
(
t
,
200
,
runtimeStats
.
collStats
[
2
]
.
field
Index
Stats
[
100
][
0
]
.
numOfRelatedSegments
)
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录