Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
2e198e27
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2e198e27
编写于
12月 30, 2020
作者:
D
dragondriver
提交者:
yefu.chen
12月 30, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add support for idmap, nsg, sq8
Signed-off-by:
N
dragondriver
<
jiquan.long@zilliz.com
>
上级
09620efa
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
137 addition
and
29 deletion
+137
-29
internal/core/src/indexbuilder/IndexWrapper.cpp
internal/core/src/indexbuilder/IndexWrapper.cpp
+55
-12
internal/core/src/indexbuilder/IndexWrapper.h
internal/core/src/indexbuilder/IndexWrapper.h
+3
-0
internal/core/src/indexbuilder/utils.h
internal/core/src/indexbuilder/utils.h
+35
-4
internal/core/unittest/test_index_wrapper.cpp
internal/core/unittest/test_index_wrapper.cpp
+44
-13
未找到文件。
internal/core/src/indexbuilder/IndexWrapper.cpp
浏览文件 @
2e198e27
...
...
@@ -105,6 +105,32 @@ IndexWrapper::parse() {
config_
[
milvus
::
knowhere
::
IndexParams
::
m
]
=
std
::
stoi
(
m
);
}
/************************** NSG Parameter **************************/
if
(
!
config_
.
contains
(
milvus
::
knowhere
::
IndexParams
::
knng
))
{
}
else
{
auto
knng
=
config_
[
milvus
::
knowhere
::
IndexParams
::
knng
].
get
<
std
::
string
>
();
config_
[
milvus
::
knowhere
::
IndexParams
::
knng
]
=
std
::
stoi
(
knng
);
}
if
(
!
config_
.
contains
(
milvus
::
knowhere
::
IndexParams
::
search_length
))
{
}
else
{
auto
search_length
=
config_
[
milvus
::
knowhere
::
IndexParams
::
search_length
].
get
<
std
::
string
>
();
config_
[
milvus
::
knowhere
::
IndexParams
::
search_length
]
=
std
::
stoi
(
search_length
);
}
if
(
!
config_
.
contains
(
milvus
::
knowhere
::
IndexParams
::
out_degree
))
{
}
else
{
auto
out_degree
=
config_
[
milvus
::
knowhere
::
IndexParams
::
out_degree
].
get
<
std
::
string
>
();
config_
[
milvus
::
knowhere
::
IndexParams
::
out_degree
]
=
std
::
stoi
(
out_degree
);
}
if
(
!
config_
.
contains
(
milvus
::
knowhere
::
IndexParams
::
candidate
))
{
}
else
{
auto
candidate
=
config_
[
milvus
::
knowhere
::
IndexParams
::
candidate
].
get
<
std
::
string
>
();
config_
[
milvus
::
knowhere
::
IndexParams
::
candidate
]
=
std
::
stoi
(
candidate
);
}
/************************** Serialize *******************************/
if
(
!
config_
.
contains
(
milvus
::
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
))
{
config_
[
milvus
::
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
]
=
4
;
}
else
{
...
...
@@ -132,12 +158,37 @@ IndexWrapper::dim() {
void
IndexWrapper
::
BuildWithoutIds
(
const
knowhere
::
DatasetPtr
&
dataset
)
{
auto
index_type
=
get_index_type
();
// if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
// PanicInfo(std::string(index_type) + " doesn't support build without ids yet!");
if
(
is_in_need_id_list
(
index_type
))
{
PanicInfo
(
std
::
string
(
index_type
)
+
" doesn't support build without ids yet!"
);
}
// if (is_in_need_build_all_list(index_type)) {
// index_->BuildAll(dataset, config_);
// } else {
// index_->Train(dataset, config_);
// index_->AddWithoutIds(dataset, config_);
// }
index_
->
Train
(
dataset
,
config_
);
index_
->
AddWithoutIds
(
dataset
,
config_
);
index_
->
BuildAll
(
dataset
,
config_
);
if
(
is_in_nm_list
(
index_type
))
{
StoreRawData
(
dataset
);
}
}
void
IndexWrapper
::
BuildWithIds
(
const
knowhere
::
DatasetPtr
&
dataset
)
{
Assert
(
dataset
->
data
().
find
(
milvus
::
knowhere
::
meta
::
IDS
)
!=
dataset
->
data
().
end
());
// index_->Train(dataset, config_);
// index_->Add(dataset, config_);
index_
->
BuildAll
(
dataset
,
config_
);
if
(
is_in_nm_list
(
get_index_type
()))
{
StoreRawData
(
dataset
);
}
}
void
IndexWrapper
::
StoreRawData
(
const
knowhere
::
DatasetPtr
&
dataset
)
{
auto
index_type
=
get_index_type
();
if
(
is_in_nm_list
(
index_type
))
{
auto
tensor
=
dataset
->
Get
<
const
void
*>
(
milvus
::
knowhere
::
meta
::
TENSOR
);
auto
row_num
=
dataset
->
Get
<
int64_t
>
(
milvus
::
knowhere
::
meta
::
ROWS
);
...
...
@@ -153,14 +204,6 @@ IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
}
}
void
IndexWrapper
::
BuildWithIds
(
const
knowhere
::
DatasetPtr
&
dataset
)
{
Assert
(
dataset
->
data
().
find
(
milvus
::
knowhere
::
meta
::
IDS
)
!=
dataset
->
data
().
end
());
// index_->Train(dataset, config_);
// index_->Add(dataset, config_);
index_
->
BuildAll
(
dataset
,
config_
);
}
/*
* brief Return serialized binary set
*/
...
...
internal/core/src/indexbuilder/IndexWrapper.h
浏览文件 @
2e198e27
...
...
@@ -49,6 +49,9 @@ class IndexWrapper {
std
::
optional
<
T
>
get_config_by_name
(
std
::
string
name
);
void
StoreRawData
(
const
knowhere
::
DatasetPtr
&
dataset
);
public:
void
BuildWithIds
(
const
knowhere
::
DatasetPtr
&
dataset
);
...
...
internal/core/src/indexbuilder/utils.h
浏览文件 @
2e198e27
...
...
@@ -36,16 +36,47 @@ BIN_List() {
return
ret
;
}
std
::
vector
<
std
::
string
>
Need_ID_List
()
{
static
std
::
vector
<
std
::
string
>
ret
{
// milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
// milvus::knowhere::IndexEnum::INDEX_NSG
};
return
ret
;
}
std
::
vector
<
std
::
string
>
Need_BuildAll_list
()
{
static
std
::
vector
<
std
::
string
>
ret
{
milvus
::
knowhere
::
IndexEnum
::
INDEX_NSG
};
return
ret
;
}
template
<
typename
T
>
bool
is_in_list
(
const
T
&
t
,
std
::
function
<
std
::
vector
<
T
>
()
>
list_func
)
{
auto
l
=
list_func
();
return
std
::
find
(
l
.
begin
(),
l
.
end
(),
t
)
!=
l
.
end
();
}
bool
is_in_bin_list
(
const
milvus
::
knowhere
::
IndexType
&
index_type
)
{
auto
bin_list
=
BIN_List
();
return
std
::
find
(
bin_list
.
begin
(),
bin_list
.
end
(),
index_type
)
!=
bin_list
.
end
();
return
is_in_list
<
std
::
string
>
(
index_type
,
BIN_List
);
}
bool
is_in_nm_list
(
const
milvus
::
knowhere
::
IndexType
&
index_type
)
{
auto
nm_list
=
NM_List
();
return
std
::
find
(
nm_list
.
begin
(),
nm_list
.
end
(),
index_type
)
!=
nm_list
.
end
();
return
is_in_list
<
std
::
string
>
(
index_type
,
NM_List
);
}
bool
is_in_need_build_all_list
(
const
milvus
::
knowhere
::
IndexType
&
index_type
)
{
return
is_in_list
<
std
::
string
>
(
index_type
,
Need_BuildAll_list
);
}
bool
is_in_need_id_list
(
const
milvus
::
knowhere
::
IndexType
&
index_type
)
{
return
is_in_list
<
std
::
string
>
(
index_type
,
Need_ID_List
);
}
}
// namespace indexbuilder
...
...
internal/core/unittest/test_index_wrapper.cpp
浏览文件 @
2e198e27
...
...
@@ -38,7 +38,14 @@ int DEVICEID = 0;
namespace
{
auto
generate_conf
(
const
milvus
::
knowhere
::
IndexType
&
index_type
,
const
milvus
::
knowhere
::
MetricType
&
metric_type
)
{
if
(
index_type
==
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
)
{
if
(
index_type
==
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IDMAP
)
{
return
milvus
::
knowhere
::
Config
{
{
milvus
::
knowhere
::
meta
::
DIM
,
DIM
},
// {milvus::knowhere::meta::TOPK, K},
{
milvus
::
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
milvus
::
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
};
}
else
if
(
index_type
==
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
)
{
return
milvus
::
knowhere
::
Config
{
{
milvus
::
knowhere
::
meta
::
DIM
,
DIM
},
// {milvus::knowhere::meta::TOPK, K},
...
...
@@ -55,7 +62,20 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
// {milvus::knowhere::meta::TOPK, K},
{
milvus
::
knowhere
::
IndexParams
::
nlist
,
100
},
// {milvus::knowhere::IndexParams::nprobe, 4},
{
milvus
::
knowhere
::
Metric
::
TYPE
,
milvus
::
knowhere
::
Metric
::
L2
},
{
milvus
::
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
milvus
::
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
#ifdef MILVUS_GPU_VERSION
{
milvus
::
knowhere
::
meta
::
DEVICEID
,
DEVICEID
},
#endif
};
}
else
if
(
index_type
==
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFSQ8
)
{
return
milvus
::
knowhere
::
Config
{
{
milvus
::
knowhere
::
meta
::
DIM
,
DIM
},
// {milvus::knowhere::meta::TOPK, K},
{
milvus
::
knowhere
::
IndexParams
::
nlist
,
100
},
// {milvus::knowhere::IndexParams::nprobe, 4},
{
milvus
::
knowhere
::
IndexParams
::
nbits
,
8
},
{
milvus
::
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
milvus
::
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
#ifdef MILVUS_GPU_VERSION
{
milvus
::
knowhere
::
meta
::
DEVICEID
,
DEVICEID
},
...
...
@@ -78,6 +98,15 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
// {milvus::knowhere::meta::TOPK, K},
{
milvus
::
knowhere
::
Metric
::
TYPE
,
metric_type
},
};
}
else
if
(
index_type
==
milvus
::
knowhere
::
IndexEnum
::
INDEX_NSG
)
{
return
milvus
::
knowhere
::
Config
{{
milvus
::
knowhere
::
meta
::
DIM
,
DIM
},
{
milvus
::
knowhere
::
IndexParams
::
nlist
,
163
},
{
milvus
::
knowhere
::
IndexParams
::
nprobe
,
8
},
{
milvus
::
knowhere
::
IndexParams
::
knng
,
20
},
{
milvus
::
knowhere
::
IndexParams
::
search_length
,
40
},
{
milvus
::
knowhere
::
IndexParams
::
out_degree
,
30
},
{
milvus
::
knowhere
::
IndexParams
::
candidate
,
100
},
{
milvus
::
knowhere
::
Metric
::
TYPE
,
metric_type
}};
}
return
milvus
::
knowhere
::
Config
();
}
...
...
@@ -142,11 +171,6 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
if
(
!
is_binary
)
{
xb_data
=
dataset
.
get_col
<
float
>
(
0
);
xb_dataset
=
milvus
::
knowhere
::
GenDataset
(
NB
,
DIM
,
xb_data
.
data
());
}
else
if
(
index_type
==
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IVFFLAT
)
{
xb_bin_data
=
dataset
.
get_col
<
uint8_t
>
(
0
);
ids
.
resize
(
NB
);
std
::
iota
(
ids
.
begin
(),
ids
.
end
(),
0
);
xb_dataset
=
milvus
::
knowhere
::
GenDataset
(
NB
,
DIM
,
xb_bin_data
.
data
());
}
else
{
xb_bin_data
=
dataset
.
get_col
<
uint8_t
>
(
0
);
xb_dataset
=
milvus
::
knowhere
::
GenDataset
(
NB
,
DIM
,
xb_bin_data
.
data
());
...
...
@@ -335,11 +359,14 @@ TEST(BinIdMapWrapper, Build) {
INSTANTIATE_TEST_CASE_P
(
IndexTypeParameters
,
IndexWrapperTest
,
::
testing
::
Values
(
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
,
milvus
::
knowhere
::
Metric
::
L2
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
,
milvus
::
knowhere
::
Metric
::
L2
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IVFFLAT
,
milvus
::
knowhere
::
Metric
::
JACCARD
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IDMAP
,
milvus
::
knowhere
::
Metric
::
JACCARD
)));
::
testing
::
Values
(
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IDMAP
,
milvus
::
knowhere
::
Metric
::
L2
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
,
milvus
::
knowhere
::
Metric
::
L2
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
,
milvus
::
knowhere
::
Metric
::
L2
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFSQ8
,
milvus
::
knowhere
::
Metric
::
L2
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IVFFLAT
,
milvus
::
knowhere
::
Metric
::
JACCARD
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IDMAP
,
milvus
::
knowhere
::
Metric
::
JACCARD
),
std
::
pair
(
milvus
::
knowhere
::
IndexEnum
::
INDEX_NSG
,
milvus
::
knowhere
::
Metric
::
L2
)));
TEST_P
(
IndexWrapperTest
,
Constructor
)
{
auto
index
=
...
...
@@ -357,7 +384,11 @@ TEST_P(IndexWrapperTest, BuildWithoutIds) {
auto
index
=
std
::
make_unique
<
milvus
::
indexbuilder
::
IndexWrapper
>
(
type_params_str
.
c_str
(),
index_params_str
.
c_str
());
ASSERT_NO_THROW
(
index
->
BuildWithoutIds
(
xb_dataset
));
if
(
milvus
::
indexbuilder
::
is_in_need_id_list
(
index_type
))
{
ASSERT_ANY_THROW
(
index
->
BuildWithoutIds
(
xb_dataset
));
}
else
{
ASSERT_NO_THROW
(
index
->
BuildWithoutIds
(
xb_dataset
));
}
}
TEST_P
(
IndexWrapperTest
,
Codec
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录