Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
add31188
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
add31188
编写于
12月 24, 2020
作者:
S
sunby
提交者:
yefu.chen
12月 24, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add CreateIndex RPC
Signed-off-by:
N
sunby
<
bingyi.sun@zilliz.com
>
上级
14332d49
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
187 addition
and
4 deletion
+187
-4
internal/master/grpc_service.go
internal/master/grpc_service.go
+33
-2
internal/master/index_task.go
internal/master/index_task.go
+95
-0
internal/master/meta_table.go
internal/master/meta_table.go
+44
-1
internal/master/meta_table_test.go
internal/master/meta_table_test.go
+1
-1
internal/master/segment_manager.go
internal/master/segment_manager.go
+14
-0
未找到文件。
internal/master/grpc_service.go
浏览文件 @
add31188
...
...
@@ -443,8 +443,39 @@ func (s *Master) AssignSegmentID(ctx context.Context, request *internalpb.Assign
},
nil
}
func
(
s
*
Master
)
CreateIndex
(
context
.
Context
,
*
internalpb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
{
return
nil
,
nil
func
(
s
*
Master
)
CreateIndex
(
ctx
context
.
Context
,
req
*
internalpb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
{
task
:=
&
createIndexTask
{
baseTask
:
baseTask
{
sch
:
s
.
scheduler
,
mt
:
s
.
metaTable
,
cv
:
make
(
chan
error
),
},
req
:
req
,
indexBuildScheduler
:
s
.
indexBuildSch
,
indexLoadScheduler
:
s
.
indexLoadSch
,
segManager
:
s
.
segmentManager
,
}
err
:=
s
.
scheduler
.
Enqueue
(
task
)
if
err
!=
nil
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
Reason
:
"Enqueue failed: "
+
err
.
Error
(),
},
nil
}
err
=
task
.
WaitToFinish
(
ctx
)
if
err
!=
nil
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
Reason
:
"Create Index error: "
+
err
.
Error
(),
},
nil
}
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
Reason
:
""
,
},
nil
}
func
(
s
*
Master
)
DescribeIndex
(
context
.
Context
,
*
internalpb
.
DescribeIndexRequest
)
(
*
servicepb
.
DescribeIndexResponse
,
error
)
{
...
...
internal/master/index_task.go
0 → 100644
浏览文件 @
add31188
package
master
import
(
"fmt"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
)
type
createIndexTask
struct
{
baseTask
req
*
internalpb
.
CreateIndexRequest
indexBuildScheduler
*
IndexBuildScheduler
indexLoadScheduler
*
IndexLoadScheduler
segManager
*
SegmentManager
}
func
(
task
*
createIndexTask
)
Type
()
internalpb
.
MsgType
{
return
internalpb
.
MsgType_kCreateIndex
}
func
(
task
*
createIndexTask
)
Ts
()
(
Timestamp
,
error
)
{
return
task
.
req
.
Timestamp
,
nil
}
func
(
task
*
createIndexTask
)
Execute
()
error
{
// modify schema
if
err
:=
task
.
mt
.
UpdateFieldIndexParams
(
task
.
req
.
CollectionName
,
task
.
req
.
FieldName
,
task
.
req
.
ExtraParams
);
err
!=
nil
{
return
err
}
// check if closed segment has the same index build history
collMeta
,
err
:=
task
.
mt
.
GetCollectionByName
(
task
.
req
.
CollectionName
)
if
err
!=
nil
{
return
err
}
var
fieldID
int64
=
-
1
for
_
,
fieldSchema
:=
range
collMeta
.
Schema
.
Fields
{
if
fieldSchema
.
Name
==
task
.
req
.
FieldName
{
fieldID
=
fieldSchema
.
FieldID
break
}
}
if
fieldID
==
-
1
{
return
fmt
.
Errorf
(
"can not find field name %s"
,
task
.
req
.
FieldName
)
}
for
_
,
segID
:=
range
collMeta
.
SegmentIDs
{
segMeta
,
err
:=
task
.
mt
.
GetSegmentByID
(
segID
)
if
err
!=
nil
{
return
err
}
if
segMeta
.
CloseTime
==
0
{
continue
}
hasIndexMeta
,
err
:=
task
.
mt
.
HasFieldIndexMeta
(
segID
,
fieldID
,
task
.
req
.
ExtraParams
)
if
err
!=
nil
{
return
err
}
if
hasIndexMeta
{
// load index
indexMeta
,
err
:=
task
.
mt
.
GetFieldIndexMeta
(
segID
,
fieldID
,
task
.
req
.
ExtraParams
)
if
err
!=
nil
{
return
err
}
err
=
task
.
indexLoadScheduler
.
Enqueue
(
&
IndexLoadInfo
{
segmentID
:
segID
,
fieldID
:
fieldID
,
fieldName
:
task
.
req
.
FieldName
,
indexFilePaths
:
indexMeta
.
IndexFilePaths
,
})
if
err
!=
nil
{
return
err
}
}
else
{
// create index
for
_
,
kv
:=
range
segMeta
.
BinlogFilePaths
{
if
kv
.
FieldID
!=
fieldID
{
continue
}
err
:=
task
.
indexBuildScheduler
.
Enqueue
(
&
IndexBuildInfo
{
segmentID
:
segID
,
fieldID
:
fieldID
,
binlogFilePath
:
kv
.
BinlogFiles
,
})
if
err
!=
nil
{
return
err
}
break
}
}
}
// close unfilled segment
return
task
.
segManager
.
ForceClose
(
collMeta
.
ID
)
}
internal/master/meta_table.go
浏览文件 @
add31188
...
...
@@ -524,7 +524,7 @@ func (mt *metaTable) saveFieldIndexMetaToEtcd(meta *pb.FieldIndexMeta) error {
return
mt
.
client
.
Save
(
key
,
marshaledMeta
)
}
func
(
mt
*
metaTable
)
DeleteFieldIndexMeta
(
segID
UniqueID
,
fieldID
UniqueID
,
index
Type
string
,
index
Params
[]
*
commonpb
.
KeyValuePair
)
error
{
func
(
mt
*
metaTable
)
DeleteFieldIndexMeta
(
segID
UniqueID
,
fieldID
UniqueID
,
indexParams
[]
*
commonpb
.
KeyValuePair
)
error
{
mt
.
indexLock
.
Lock
()
defer
mt
.
indexLock
.
Unlock
()
...
...
@@ -568,6 +568,22 @@ func (mt *metaTable) HasFieldIndexMeta(segID UniqueID, fieldID UniqueID, indexPa
return
false
,
nil
}
func
(
mt
*
metaTable
)
GetFieldIndexMeta
(
segID
UniqueID
,
fieldID
UniqueID
,
indexParams
[]
*
commonpb
.
KeyValuePair
)
(
*
pb
.
FieldIndexMeta
,
error
)
{
mt
.
indexLock
.
RLock
()
defer
mt
.
indexLock
.
RUnlock
()
if
_
,
ok
:=
mt
.
segID2IndexMetas
[
segID
];
!
ok
{
return
nil
,
fmt
.
Errorf
(
"can not find segment %d"
,
segID
)
}
for
_
,
v
:=
range
mt
.
segID2IndexMetas
[
segID
]
{
if
v
.
FieldID
==
fieldID
&&
typeutil
.
CompareIndexParams
(
v
.
IndexParams
,
indexParams
)
{
return
&
v
,
nil
}
}
return
nil
,
fmt
.
Errorf
(
"can not find field %d"
,
fieldID
)
}
func
(
mt
*
metaTable
)
UpdateFieldIndexMeta
(
meta
*
pb
.
FieldIndexMeta
)
error
{
mt
.
indexLock
.
Lock
()
defer
mt
.
indexLock
.
Unlock
()
...
...
@@ -635,3 +651,30 @@ func (mt *metaTable) GetFieldIndexParams(collID UniqueID, fieldID UniqueID) ([]*
}
return
nil
,
fmt
.
Errorf
(
"can not find field %d in collection %d"
,
fieldID
,
collID
)
}
func
(
mt
*
metaTable
)
UpdateFieldIndexParams
(
collName
string
,
fieldName
string
,
indexParams
[]
*
commonpb
.
KeyValuePair
)
error
{
mt
.
ddLock
.
Lock
()
defer
mt
.
ddLock
.
Unlock
()
vid
,
ok
:=
mt
.
collName2ID
[
collName
]
if
!
ok
{
return
errors
.
Errorf
(
"can't find collection: "
+
collName
)
}
meta
,
ok
:=
mt
.
collID2Meta
[
vid
]
if
!
ok
{
return
errors
.
Errorf
(
"can't find collection: "
+
collName
)
}
for
_
,
fieldSchema
:=
range
meta
.
Schema
.
Fields
{
if
fieldSchema
.
Name
==
fieldName
{
fieldSchema
.
IndexParams
=
indexParams
if
err
:=
mt
.
saveCollectionMeta
(
&
meta
);
err
!=
nil
{
_
=
mt
.
reloadFromKV
()
return
err
}
return
nil
}
}
return
fmt
.
Errorf
(
"can not find field with id %s"
,
fieldName
)
}
internal/master/meta_table_test.go
浏览文件 @
add31188
...
...
@@ -497,7 +497,7 @@ func TestMetaTable_IndexMeta(t *testing.T) {
})
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
indexbuilderpb
.
IndexStatus_FINISHED
,
meta
.
segID2IndexMetas
[
1
][
0
]
.
Status
)
err
=
meta
.
DeleteFieldIndexMeta
(
1
,
100
,
"type1"
,
[]
*
commonpb
.
KeyValuePair
{{
Key
:
"k1"
,
Value
:
"v1"
}})
err
=
meta
.
DeleteFieldIndexMeta
(
1
,
100
,
[]
*
commonpb
.
KeyValuePair
{{
Key
:
"k1"
,
Value
:
"v1"
}})
assert
.
Nil
(
t
,
err
)
res
,
err
=
meta
.
HasFieldIndexMeta
(
1
,
100
,
[]
*
commonpb
.
KeyValuePair
{{
Key
:
"k1"
,
Value
:
"v1"
}})
assert
.
Nil
(
t
,
err
)
...
...
internal/master/segment_manager.go
浏览文件 @
add31188
...
...
@@ -358,6 +358,20 @@ func (manager *SegmentManager) initChannelRanges() error {
}
return
nil
}
// ForceClose set segments of collection with collID closable, segment will be closed after the assignments of it has expired
func
(
manager
*
SegmentManager
)
ForceClose
(
collID
UniqueID
)
error
{
status
,
ok
:=
manager
.
collStatus
[
collID
]
if
!
ok
{
return
nil
}
for
_
,
segStatus
:=
range
status
.
segments
{
segStatus
.
closable
=
true
}
return
nil
}
func
NewSegmentManager
(
ctx
context
.
Context
,
meta
*
metaTable
,
globalIDAllocator
func
()
(
UniqueID
,
error
),
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录