Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
ba7e3c3d
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ba7e3c3d
编写于
7月 22, 2019
作者:
X
xj.lin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-265 IVFSQ
Former-commit-id: 6b0638af562ed60bc2252e4430c1018e29ae857c
上级
168ca0a5
变更
6
显示空白变更内容
内联
并排
Showing
6 changed file
with
49 addition
and
23 deletion
+49
-23
cpp/src/db/ExecutionEngineImpl.cpp
cpp/src/db/ExecutionEngineImpl.cpp
+7
-2
cpp/src/wrapper/knowhere/vec_impl.cpp
cpp/src/wrapper/knowhere/vec_impl.cpp
+4
-10
cpp/src/wrapper/knowhere/vec_impl.h
cpp/src/wrapper/knowhere/vec_impl.h
+3
-2
cpp/src/wrapper/knowhere/vec_index.cpp
cpp/src/wrapper/knowhere/vec_index.cpp
+19
-1
cpp/src/wrapper/knowhere/vec_index.h
cpp/src/wrapper/knowhere/vec_index.h
+3
-0
cpp/unittest/index_wrapper/knowhere_test.cpp
cpp/unittest/index_wrapper/knowhere_test.cpp
+13
-8
未找到文件。
cpp/src/db/ExecutionEngineImpl.cpp
浏览文件 @
ba7e3c3d
...
...
@@ -69,7 +69,7 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
}
Status
ExecutionEngineImpl
::
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
{
auto
ec
=
index_
->
Add
(
n
,
xdata
,
xids
,
Config
::
object
{{
"dim"
,
dim
}}
);
auto
ec
=
index_
->
Add
(
n
,
xdata
,
xids
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
return
Status
::
Error
(
"Add error"
);
}
...
...
@@ -171,10 +171,15 @@ ExecutionEngineImpl::BuildIndex(const std::string &location) {
throw
Exception
(
"Create Empty VecIndex"
);
}
Config
build_cfg
;
build_cfg
[
"dim"
]
=
Dimension
();
build_cfg
[
"gpu_id"
]
=
gpu_num
;
AutoGenParams
(
to_index
->
GetType
(),
Count
(),
build_cfg
);
auto
ec
=
to_index
->
BuildAll
(
Count
(),
from_index
->
GetRawVectors
(),
from_index
->
GetRawIds
(),
Config
::
object
{{
"dim"
,
Dimension
()},
{
"gpu_id"
,
gpu_num
}}
);
build_cfg
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
throw
Exception
(
"Build index error"
);
}
return
std
::
make_shared
<
ExecutionEngineImpl
>
(
to_index
,
location
,
build_type
);
...
...
cpp/src/wrapper/knowhere/vec_impl.cpp
浏览文件 @
ba7e3c3d
...
...
@@ -32,9 +32,7 @@ server::KnowhereError VecIndexImpl::BuildAll(const long &nb,
auto
preprocessor
=
index_
->
BuildPreprocessor
(
dataset
,
cfg
);
index_
->
set_preprocessor
(
preprocessor
);
auto
nlist
=
int
(
nb
/
1000000.0
*
16384
);
auto
cfg_t
=
Config
::
object
{{
"nlist"
,
nlist
},
{
"dim"
,
dim
}};
auto
model
=
index_
->
Train
(
dataset
,
cfg_t
);
auto
model
=
index_
->
Train
(
dataset
,
cfg
);
index_
->
set_index_model
(
model
);
index_
->
Add
(
dataset
,
cfg
);
}
catch
(
KnowhereException
&
e
)
{
...
...
@@ -52,8 +50,7 @@ server::KnowhereError VecIndexImpl::BuildAll(const long &nb,
server
::
KnowhereError
VecIndexImpl
::
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
{
try
{
auto
d
=
cfg
.
get_with_default
(
"dim"
,
dim
);
auto
dataset
=
GenDatasetWithIds
(
nb
,
d
,
xb
,
ids
);
auto
dataset
=
GenDatasetWithIds
(
nb
,
dim
,
xb
,
ids
);
index_
->
Add
(
dataset
,
cfg
);
}
catch
(
KnowhereException
&
e
)
{
...
...
@@ -72,8 +69,7 @@ server::KnowhereError VecIndexImpl::Add(const long &nb, const float *xb, const l
server
::
KnowhereError
VecIndexImpl
::
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
{
try
{
auto
k
=
cfg
[
"k"
].
as
<
int
>
();
auto
d
=
cfg
.
get_with_default
(
"dim"
,
dim
);
auto
dataset
=
GenDataset
(
nq
,
d
,
xq
);
auto
dataset
=
GenDataset
(
nq
,
dim
,
xq
);
Config
search_cfg
;
auto
res
=
index_
->
Search
(
dataset
,
cfg
);
...
...
@@ -203,9 +199,7 @@ server::KnowhereError IVFMixIndex::BuildAll(const long &nb,
auto
preprocessor
=
index_
->
BuildPreprocessor
(
dataset
,
cfg
);
index_
->
set_preprocessor
(
preprocessor
);
auto
nlist
=
int
(
nb
/
1000000.0
*
16384
);
auto
cfg_t
=
Config
::
object
{{
"nlist"
,
nlist
},
{
"dim"
,
dim
}};
auto
model
=
index_
->
Train
(
dataset
,
cfg_t
);
auto
model
=
index_
->
Train
(
dataset
,
cfg
);
index_
->
set_index_model
(
model
);
index_
->
Add
(
dataset
,
cfg
);
...
...
cpp/src/wrapper/knowhere/vec_impl.h
浏览文件 @
ba7e3c3d
...
...
@@ -41,8 +41,9 @@ class VecIndexImpl : public VecIndex {
class
IVFMixIndex
:
public
VecIndexImpl
{
public:
explicit
IVFMixIndex
(
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
)
:
VecIndexImpl
(
std
::
move
(
index
),
IndexType
::
FAISS_IVFFLAT_MIX
)
{};
explicit
IVFMixIndex
(
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
,
const
IndexType
&
type
)
:
VecIndexImpl
(
std
::
move
(
index
),
type
)
{};
server
::
KnowhereError
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
...
...
cpp/src/wrapper/knowhere/vec_index.cpp
浏览文件 @
ba7e3c3d
...
...
@@ -85,7 +85,7 @@ VecIndexPtr GetVecIndexFactory(const IndexType &type) {
}
case
IndexType
::
FAISS_IVFFLAT_MIX
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVF
>
(
0
);
return
std
::
make_shared
<
IVFMixIndex
>
(
index
);
return
std
::
make_shared
<
IVFMixIndex
>
(
index
,
IndexType
::
FAISS_IVFFLAT_MIX
);
}
case
IndexType
::
FAISS_IVFPQ_CPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
IVFPQ
>
();
...
...
@@ -98,6 +98,10 @@ VecIndexPtr GetVecIndexFactory(const IndexType &type) {
case
IndexType
::
SPTAG_KDT_RNT_CPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
CPUKDTRNG
>
();
break
;
}
case
IndexType
::
FAISS_IVFSQ8_MIX
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVFSQ
>
(
0
);
return
std
::
make_shared
<
IVFMixIndex
>
(
index
,
IndexType
::
FAISS_IVFSQ8_MIX
);
}
//case IndexType::NSG: { // TODO(linxj): bug.
// index = std::make_shared<zilliz::knowhere::NSG>();
...
...
@@ -183,6 +187,20 @@ server::KnowhereError write_index(VecIndexPtr index, const std::string &location
return
server
::
KNOWHERE_SUCCESS
;
}
// TODO(linxj): redo here.
void
AutoGenParams
(
const
IndexType
&
type
,
const
long
&
size
,
zilliz
::
knowhere
::
Config
&
cfg
)
{
if
(
!
cfg
.
contains
(
"nlist"
))
{
cfg
[
"nlist"
]
=
int
(
size
/
1000000.0
*
16384
);
}
if
(
!
cfg
.
contains
(
"gpu_id"
))
{
cfg
[
"gpu_id"
]
=
int
(
0
);
}
switch
(
type
)
{
case
IndexType
::
FAISS_IVFSQ8_MIX
:
{
if
(
!
cfg
.
contains
(
"nbits"
))
{
cfg
[
"nbits"
]
=
int
(
8
);
}
break
;
}
}
}
}
}
}
cpp/src/wrapper/knowhere/vec_index.h
浏览文件 @
ba7e3c3d
...
...
@@ -31,6 +31,7 @@ enum class IndexType {
FAISS_IVFPQ_CPU
,
FAISS_IVFPQ_GPU
,
SPTAG_KDT_RNT_CPU
,
FAISS_IVFSQ8_MIX
,
//NSG,
};
...
...
@@ -75,6 +76,8 @@ extern VecIndexPtr GetVecIndexFactory(const IndexType &type);
extern
VecIndexPtr
LoadVecIndex
(
const
IndexType
&
index_type
,
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
);
extern
void
AutoGenParams
(
const
IndexType
&
type
,
const
long
&
size
,
Config
&
cfg
);
}
}
}
cpp/unittest/index_wrapper/knowhere_test.cpp
浏览文件 @
ba7e3c3d
...
...
@@ -41,7 +41,7 @@ class KnowhereWrapperTest
for
(
auto
i
=
0
;
i
<
nq
;
i
++
)
{
EXPECT_EQ
(
ids
[
i
*
k
],
gt_ids
[
i
*
k
]);
EXPECT_EQ
(
dis
[
i
*
k
],
gt_dis
[
i
*
k
]);
//
EXPECT_EQ(dis[i * k], gt_dis[i * k]);
}
int
match
=
0
;
...
...
@@ -84,11 +84,11 @@ class KnowhereWrapperTest
INSTANTIATE_TEST_CASE_P
(
WrapperParam
,
KnowhereWrapperTest
,
Values
(
//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
std
::
make_tuple
(
IndexType
::
FAISS_IVFFLAT_CPU
,
"Default"
,
64
,
100000
,
10
,
10
,
Config
::
object
{{
"nlist"
,
100
},
{
"dim"
,
64
}},
Config
::
object
{{
"dim"
,
64
},
{
"k"
,
10
},
{
"nprobe"
,
10
}}
),
//
std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default",
//
64, 100000, 10, 10,
//
Config::object{{"nlist", 100}, {"dim", 64}},
//
Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 10}}
//
),
//std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default",
// 64, 10000, 10, 10,
// Config::object{{"nlist", 100}, {"dim", 64}},
...
...
@@ -96,13 +96,18 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
//),
std
::
make_tuple
(
IndexType
::
FAISS_IVFFLAT_MIX
,
"Default"
,
64
,
100000
,
10
,
10
,
Config
::
object
{{
"nlist"
,
100
},
{
"dim"
,
64
}},
Config
::
object
{{
"dim"
,
64
},
{
"k"
,
10
},
{
"nprobe"
,
10
}}
Config
::
object
{{
"nlist"
,
100
0
},
{
"dim"
,
64
}},
Config
::
object
{{
"dim"
,
64
},
{
"k"
,
10
},
{
"nprobe"
,
5
}}
),
std
::
make_tuple
(
IndexType
::
FAISS_IDMAP
,
"Default"
,
64
,
100000
,
10
,
10
,
Config
::
object
{{
"dim"
,
64
}},
Config
::
object
{{
"dim"
,
64
},
{
"k"
,
10
}}
),
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_MIX
,
"Default"
,
64
,
100000
,
10
,
10
,
Config
::
object
{{
"dim"
,
64
},
{
"nlist"
,
1000
},
{
"nbits"
,
8
}},
Config
::
object
{{
"dim"
,
64
},
{
"k"
,
10
},
{
"nprobe"
,
5
}}
)
//std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default",
// 64, 10000, 10, 10,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录