Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
8f8fa0aa
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
8f8fa0aa
编写于
8月 04, 2020
作者:
G
groot
提交者:
GitHub
8月 04, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
use index type replace name (#3114)
* use index type replace name Signed-off-by:
N
yhmo
<
yihua.mo@zilliz.com
>
上级
6ecf7a27
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
31 addition
and
20 deletion
+31
-20
core/src/db/SnapshotUtils.cpp
core/src/db/SnapshotUtils.cpp
+10
-11
core/src/db/Types.h
core/src/db/Types.h
+1
-0
core/src/db/Utils.cpp
core/src/db/Utils.cpp
+1
-1
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+2
-1
core/src/server/delivery/request/CreateIndexReq.cpp
core/src/server/delivery/request/CreateIndexReq.cpp
+9
-3
core/unittest/db/test_db.cpp
core/unittest/db/test_db.cpp
+8
-4
未找到文件。
core/src/db/SnapshotUtils.cpp
浏览文件 @
8f8fa0aa
...
@@ -50,26 +50,23 @@ SetSnapshotIndex(const std::string& collection_name, const std::string& field_na
...
@@ -50,26 +50,23 @@ SetSnapshotIndex(const std::string& collection_name, const std::string& field_na
}
}
snapshot
::
OperationContext
ss_context
;
snapshot
::
OperationContext
ss_context
;
auto
index_element
=
std
::
make_shared
<
snapshot
::
FieldElement
>
(
ss
->
GetCollectionId
(),
field
->
GetID
(),
index_info
.
index_name_
,
milvus
::
engine
::
FieldElementType
::
FET_INDEX
,
index_info
.
index_type_
);
ss_context
.
new_field_elements
.
push_back
(
index_element
);
if
(
IsVectorField
(
field
))
{
if
(
IsVectorField
(
field
))
{
auto
new_element
=
std
::
make_shared
<
snapshot
::
FieldElement
>
(
ss
->
GetCollectionId
(),
field
->
GetID
(),
index_info
.
index_name_
,
milvus
::
engine
::
FieldElementType
::
FET_INDEX
);
milvus
::
json
json
;
milvus
::
json
json
;
json
[
engine
::
PARAM_INDEX_METRIC_TYPE
]
=
index_info
.
metric_name_
;
json
[
engine
::
PARAM_INDEX_METRIC_TYPE
]
=
index_info
.
metric_name_
;
json
[
engine
::
PARAM_INDEX_EXTRA_PARAMS
]
=
index_info
.
extra_params_
;
json
[
engine
::
PARAM_INDEX_EXTRA_PARAMS
]
=
index_info
.
extra_params_
;
new_element
->
SetParams
(
json
);
index_element
->
SetParams
(
json
);
ss_context
.
new_field_elements
.
push_back
(
new_element
);
if
(
index_info
.
index_name_
==
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFSQ8NR
||
if
(
index_info
.
index_name_
==
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFSQ8NR
||
index_info
.
index_name_
==
knowhere
::
IndexEnum
::
INDEX_HNSW_SQ8NM
)
{
index_info
.
index_name_
==
knowhere
::
IndexEnum
::
INDEX_HNSW_SQ8NM
)
{
auto
new
_element
=
std
::
make_shared
<
snapshot
::
FieldElement
>
(
auto
compress
_element
=
std
::
make_shared
<
snapshot
::
FieldElement
>
(
ss
->
GetCollectionId
(),
field
->
GetID
(),
DEFAULT_INDEX_COMPRESS_NAME
,
ss
->
GetCollectionId
(),
field
->
GetID
(),
DEFAULT_INDEX_COMPRESS_NAME
,
milvus
::
engine
::
FieldElementType
::
FET_COMPRESS_SQ8
);
milvus
::
engine
::
FieldElementType
::
FET_COMPRESS_SQ8
);
ss_context
.
new_field_elements
.
push_back
(
new
_element
);
ss_context
.
new_field_elements
.
push_back
(
compress
_element
);
}
}
}
else
{
auto
new_element
=
std
::
make_shared
<
snapshot
::
FieldElement
>
(
ss
->
GetCollectionId
(),
field
->
GetID
(),
index_info
.
index_name_
,
milvus
::
engine
::
FieldElementType
::
FET_INDEX
);
ss_context
.
new_field_elements
.
push_back
(
new_element
);
}
}
auto
op
=
std
::
make_shared
<
snapshot
::
AddFieldElementOperation
>
(
ss_context
,
ss
);
auto
op
=
std
::
make_shared
<
snapshot
::
AddFieldElementOperation
>
(
ss_context
,
ss
);
...
@@ -97,6 +94,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
...
@@ -97,6 +94,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
for
(
auto
&
field_element
:
field_elements
)
{
for
(
auto
&
field_element
:
field_elements
)
{
if
(
field_element
->
GetFtype
()
==
(
int64_t
)
milvus
::
engine
::
FieldElementType
::
FET_INDEX
)
{
if
(
field_element
->
GetFtype
()
==
(
int64_t
)
milvus
::
engine
::
FieldElementType
::
FET_INDEX
)
{
index_info
.
index_name_
=
field_element
->
GetName
();
index_info
.
index_name_
=
field_element
->
GetName
();
index_info
.
index_type_
=
field_element
->
GetTypeName
();
auto
json
=
field_element
->
GetParams
();
auto
json
=
field_element
->
GetParams
();
if
(
json
.
find
(
engine
::
PARAM_INDEX_METRIC_TYPE
)
!=
json
.
end
())
{
if
(
json
.
find
(
engine
::
PARAM_INDEX_METRIC_TYPE
)
!=
json
.
end
())
{
index_info
.
metric_name_
=
json
[
engine
::
PARAM_INDEX_METRIC_TYPE
];
index_info
.
metric_name_
=
json
[
engine
::
PARAM_INDEX_METRIC_TYPE
];
...
@@ -110,7 +108,8 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
...
@@ -110,7 +108,8 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
}
else
{
}
else
{
for
(
auto
&
field_element
:
field_elements
)
{
for
(
auto
&
field_element
:
field_elements
)
{
if
(
field_element
->
GetFtype
()
==
(
int64_t
)
milvus
::
engine
::
FieldElementType
::
FET_INDEX
)
{
if
(
field_element
->
GetFtype
()
==
(
int64_t
)
milvus
::
engine
::
FieldElementType
::
FET_INDEX
)
{
index_info
.
index_name_
=
DEFAULT_STRUCTURED_INDEX_NAME
;
index_info
.
index_name_
=
field_element
->
GetName
();
index_info
.
index_type_
=
field_element
->
GetTypeName
();
}
}
}
}
}
}
...
...
core/src/db/Types.h
浏览文件 @
8f8fa0aa
...
@@ -78,6 +78,7 @@ using DataChunkPtr = std::shared_ptr<DataChunk>;
...
@@ -78,6 +78,7 @@ using DataChunkPtr = std::shared_ptr<DataChunk>;
struct
CollectionIndex
{
struct
CollectionIndex
{
std
::
string
index_name_
;
std
::
string
index_name_
;
std
::
string
index_type_
;
std
::
string
metric_name_
;
std
::
string
metric_name_
;
milvus
::
json
extra_params_
=
{{
"nlist"
,
2048
}};
milvus
::
json
extra_params_
=
{{
"nlist"
,
2048
}};
};
};
...
...
core/src/db/Utils.cpp
浏览文件 @
8f8fa0aa
...
@@ -49,7 +49,7 @@ GetMicroSecTimeStamp() {
...
@@ -49,7 +49,7 @@ GetMicroSecTimeStamp() {
bool
bool
IsSameIndex
(
const
CollectionIndex
&
index1
,
const
CollectionIndex
&
index2
)
{
IsSameIndex
(
const
CollectionIndex
&
index1
,
const
CollectionIndex
&
index2
)
{
return
index1
.
index_
name_
==
index2
.
index_nam
e_
&&
index1
.
extra_params_
==
index2
.
extra_params_
&&
return
index1
.
index_
type_
==
index2
.
index_typ
e_
&&
index1
.
extra_params_
==
index2
.
extra_params_
&&
index1
.
metric_name_
==
index2
.
metric_name_
;
index1
.
metric_name_
==
index2
.
metric_name_
;
}
}
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
8f8fa0aa
...
@@ -647,6 +647,7 @@ ExecutionEngineImpl::CreateSnapshotIndexFile(AddSegmentFileOperation& operation,
...
@@ -647,6 +647,7 @@ ExecutionEngineImpl::CreateSnapshotIndexFile(AddSegmentFileOperation& operation,
auto
&
index_element
=
element_visitor
->
GetElement
();
auto
&
index_element
=
element_visitor
->
GetElement
();
index_info
.
index_name_
=
index_element
->
GetName
();
index_info
.
index_name_
=
index_element
->
GetName
();
index_info
.
index_type_
=
index_element
->
GetTypeName
();
auto
params
=
index_element
->
GetParams
();
auto
params
=
index_element
->
GetParams
();
if
(
params
.
find
(
engine
::
PARAM_INDEX_METRIC_TYPE
)
!=
params
.
end
())
{
if
(
params
.
find
(
engine
::
PARAM_INDEX_METRIC_TYPE
)
!=
params
.
end
())
{
index_info
.
metric_name_
=
params
[
engine
::
PARAM_INDEX_METRIC_TYPE
];
index_info
.
metric_name_
=
params
[
engine
::
PARAM_INDEX_METRIC_TYPE
];
...
@@ -728,7 +729,7 @@ ExecutionEngineImpl::BuildKnowhereIndex(const std::string& field_name, const Col
...
@@ -728,7 +729,7 @@ ExecutionEngineImpl::BuildKnowhereIndex(const std::string& field_name, const Col
}
}
// build index by knowhere
// build index by knowhere
new_index
=
CreateVecIndex
(
index_info
.
index_
nam
e_
);
new_index
=
CreateVecIndex
(
index_info
.
index_
typ
e_
);
if
(
!
new_index
)
{
if
(
!
new_index
)
{
throw
Exception
(
DB_ERROR
,
"Unsupported index type"
);
throw
Exception
(
DB_ERROR
,
"Unsupported index type"
);
}
}
...
...
core/src/server/delivery/request/CreateIndexReq.cpp
浏览文件 @
8f8fa0aa
...
@@ -84,7 +84,7 @@ CreateIndexReq::OnExecute() {
...
@@ -84,7 +84,7 @@ CreateIndexReq::OnExecute() {
int64_t
dimension
=
params
[
engine
::
PARAM_DIMENSION
].
get
<
int64_t
>
();
int64_t
dimension
=
params
[
engine
::
PARAM_DIMENSION
].
get
<
int64_t
>
();
// validate index type
// validate index type
std
::
string
index_type
=
0
;
std
::
string
index_type
;
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_TYPE
))
{
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_TYPE
))
{
index_type
=
json_params_
[
engine
::
PARAM_INDEX_TYPE
].
get
<
std
::
string
>
();
index_type
=
json_params_
[
engine
::
PARAM_INDEX_TYPE
].
get
<
std
::
string
>
();
}
}
...
@@ -94,7 +94,7 @@ CreateIndexReq::OnExecute() {
...
@@ -94,7 +94,7 @@ CreateIndexReq::OnExecute() {
}
}
// validate metric type
// validate metric type
std
::
string
metric_type
=
0
;
std
::
string
metric_type
;
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_METRIC_TYPE
))
{
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_METRIC_TYPE
))
{
metric_type
=
json_params_
[
engine
::
PARAM_INDEX_METRIC_TYPE
].
get
<
std
::
string
>
();
metric_type
=
json_params_
[
engine
::
PARAM_INDEX_METRIC_TYPE
].
get
<
std
::
string
>
();
}
}
...
@@ -111,13 +111,19 @@ CreateIndexReq::OnExecute() {
...
@@ -111,13 +111,19 @@ CreateIndexReq::OnExecute() {
rc
.
RecordSection
(
"check validation"
);
rc
.
RecordSection
(
"check validation"
);
index
.
index_name_
=
index_type
;
index
.
index_name_
=
index_name_
;
index
.
index_type_
=
index_type
;
index
.
metric_name_
=
metric_type
;
index
.
metric_name_
=
metric_type
;
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_EXTRA_PARAMS
))
{
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_EXTRA_PARAMS
))
{
index
.
extra_params_
=
json_params_
[
engine
::
PARAM_INDEX_EXTRA_PARAMS
];
index
.
extra_params_
=
json_params_
[
engine
::
PARAM_INDEX_EXTRA_PARAMS
];
}
}
}
else
{
}
else
{
index
.
index_name_
=
index_name_
;
index
.
index_name_
=
index_name_
;
std
::
string
index_type
;
if
(
json_params_
.
contains
(
engine
::
PARAM_INDEX_TYPE
))
{
index_type
=
json_params_
[
engine
::
PARAM_INDEX_TYPE
].
get
<
std
::
string
>
();
}
index
.
index_type_
=
index_type
;
}
}
STATUS_CHECK
(
DBWrapper
::
DB
()
->
CreateIndex
(
context_
,
collection_name_
,
field_name_
,
index
));
STATUS_CHECK
(
DBWrapper
::
DB
()
->
CreateIndex
(
context_
,
collection_name_
,
field_name_
,
index
));
...
...
core/unittest/db/test_db.cpp
浏览文件 @
8f8fa0aa
...
@@ -502,7 +502,8 @@ TEST_F(DBTest, IndexTest) {
...
@@ -502,7 +502,8 @@ TEST_F(DBTest, IndexTest) {
{
{
milvus
::
engine
::
CollectionIndex
index
;
milvus
::
engine
::
CollectionIndex
index
;
index
.
index_name_
=
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
;
index
.
index_name_
=
"my_index1"
;
index
.
index_type_
=
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
;
index
.
metric_name_
=
milvus
::
knowhere
::
Metric
::
L2
;
index
.
metric_name_
=
milvus
::
knowhere
::
Metric
::
L2
;
index
.
extra_params_
[
"nlist"
]
=
2048
;
index
.
extra_params_
[
"nlist"
]
=
2048
;
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
VECTOR_FIELD_NAME
,
index
);
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
VECTOR_FIELD_NAME
,
index
);
...
@@ -512,13 +513,15 @@ TEST_F(DBTest, IndexTest) {
...
@@ -512,13 +513,15 @@ TEST_F(DBTest, IndexTest) {
status
=
db_
->
DescribeIndex
(
collection_name
,
VECTOR_FIELD_NAME
,
index_get
);
status
=
db_
->
DescribeIndex
(
collection_name
,
VECTOR_FIELD_NAME
,
index_get
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_TRUE
(
status
.
ok
());
ASSERT_EQ
(
index
.
index_name_
,
index_get
.
index_name_
);
ASSERT_EQ
(
index
.
index_name_
,
index_get
.
index_name_
);
ASSERT_EQ
(
index
.
index_type_
,
index_get
.
index_type_
);
ASSERT_EQ
(
index
.
metric_name_
,
index_get
.
metric_name_
);
ASSERT_EQ
(
index
.
metric_name_
,
index_get
.
metric_name_
);
ASSERT_EQ
(
index
.
extra_params_
,
index_get
.
extra_params_
);
ASSERT_EQ
(
index
.
extra_params_
,
index_get
.
extra_params_
);
}
}
{
{
milvus
::
engine
::
CollectionIndex
index
;
milvus
::
engine
::
CollectionIndex
index
;
index
.
index_name_
=
"SORTED"
;
index
.
index_name_
=
"my_index2"
;
index
.
index_type_
=
milvus
::
engine
::
DEFAULT_STRUCTURED_INDEX_NAME
;
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_0"
,
index
);
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_0"
,
index
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_TRUE
(
status
.
ok
());
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_1"
,
index
);
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_1"
,
index
);
...
@@ -530,6 +533,7 @@ TEST_F(DBTest, IndexTest) {
...
@@ -530,6 +533,7 @@ TEST_F(DBTest, IndexTest) {
status
=
db_
->
DescribeIndex
(
collection_name
,
"field_0"
,
index_get
);
status
=
db_
->
DescribeIndex
(
collection_name
,
"field_0"
,
index_get
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_TRUE
(
status
.
ok
());
ASSERT_EQ
(
index
.
index_name_
,
index_get
.
index_name_
);
ASSERT_EQ
(
index
.
index_name_
,
index_get
.
index_name_
);
ASSERT_EQ
(
index
.
index_type_
,
index_get
.
index_type_
);
}
}
{
{
...
@@ -577,7 +581,7 @@ TEST_F(DBTest, StatsTest) {
...
@@ -577,7 +581,7 @@ TEST_F(DBTest, StatsTest) {
{
{
milvus
::
engine
::
CollectionIndex
index
;
milvus
::
engine
::
CollectionIndex
index
;
index
.
index_
nam
e_
=
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
;
index
.
index_
typ
e_
=
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
;
index
.
metric_name_
=
milvus
::
knowhere
::
Metric
::
L2
;
index
.
metric_name_
=
milvus
::
knowhere
::
Metric
::
L2
;
index
.
extra_params_
[
"nlist"
]
=
2048
;
index
.
extra_params_
[
"nlist"
]
=
2048
;
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
VECTOR_FIELD_NAME
,
index
);
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
VECTOR_FIELD_NAME
,
index
);
...
@@ -586,7 +590,7 @@ TEST_F(DBTest, StatsTest) {
...
@@ -586,7 +590,7 @@ TEST_F(DBTest, StatsTest) {
{
{
milvus
::
engine
::
CollectionIndex
index
;
milvus
::
engine
::
CollectionIndex
index
;
index
.
index_
name_
=
"SORTED"
;
index
.
index_
type_
=
milvus
::
engine
::
DEFAULT_STRUCTURED_INDEX_NAME
;
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_0"
,
index
);
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_0"
,
index
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_TRUE
(
status
.
ok
());
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_1"
,
index
);
status
=
db_
->
CreateIndex
(
dummy_context_
,
collection_name
,
"field_1"
,
index
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录