Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
c6501ec0
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c6501ec0
编写于
9月 27, 2019
作者:
X
xiaojun.lin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-596 enable knowhere and fix compile error
Former-commit-id: 75c435ef22ceec6f07e92e42f99b369657e3911a
上级
81c37afc
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
56 addition
and
59 deletion
+56
-59
cpp/src/core/CMakeLists.txt
cpp/src/core/CMakeLists.txt
+4
-2
cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
...src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
+4
-2
cpp/src/core/test/CMakeLists.txt
cpp/src/core/test/CMakeLists.txt
+3
-11
cpp/src/core/test/test_nsg/CMakeLists.txt
cpp/src/core/test/test_nsg/CMakeLists.txt
+6
-20
cpp/src/core/test/test_nsg/test_nsg.cpp
cpp/src/core/test/test_nsg/test_nsg.cpp
+30
-22
cpp/src/wrapper/ConfAdapter.cpp
cpp/src/wrapper/ConfAdapter.cpp
+4
-0
cpp/src/wrapper/ConfAdapter.h
cpp/src/wrapper/ConfAdapter.h
+3
-0
cpp/src/wrapper/VecIndex.cpp
cpp/src/wrapper/VecIndex.cpp
+1
-1
cpp/unittest/wrapper/wrapper_test.cpp
cpp/unittest/wrapper/wrapper_test.cpp
+1
-1
未找到文件。
cpp/src/core/CMakeLists.txt
浏览文件 @
c6501ec0
...
@@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE)
...
@@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE)
endif
(
NOT CMAKE_BUILD_TYPE
)
endif
(
NOT CMAKE_BUILD_TYPE
)
if
(
CMAKE_BUILD_TYPE STREQUAL
"Release"
)
if
(
CMAKE_BUILD_TYPE STREQUAL
"Release"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O3 -fPIC -fopenmp"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O3 -fPIC -DELPP_THREAD_SAFE -fopenmp"
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-O3"
)
else
()
else
()
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O0 -g -fPIC -fopenmp"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp"
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-O0 -g"
)
endif
()
endif
()
MESSAGE
(
STATUS
"CMAKE_CXX_FLAGS"
${
CMAKE_CXX_FLAGS
}
)
MESSAGE
(
STATUS
"CMAKE_CXX_FLAGS"
${
CMAKE_CXX_FLAGS
}
)
...
...
cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
浏览文件 @
c6501ec0
...
@@ -24,6 +24,7 @@
...
@@ -24,6 +24,7 @@
#include "NSG.h"
#include "NSG.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Log.h"
#include "knowhere/common/Timer.h"
#include "knowhere/common/Timer.h"
#include "NSGHelper.h"
#include "NSGHelper.h"
...
@@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con
...
@@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con
for
(
int
i
=
0
;
i
<
ntotal
;
++
i
)
{
for
(
int
i
=
0
;
i
<
ntotal
;
++
i
)
{
total_degree
+=
nsg
[
i
].
size
();
total_degree
+=
nsg
[
i
].
size
();
}
}
std
::
cout
<<
"graph physical size: "
<<
total_degree
*
sizeof
(
node_t
)
/
1024
/
1024
;
std
::
cout
<<
"average degree: "
<<
total_degree
/
ntotal
;
KNOWHERE_LOG_DEBUG
<<
"Graph physical size: "
<<
total_degree
*
sizeof
(
node_t
)
/
1024
/
1024
<<
"m"
;
KNOWHERE_LOG_DEBUG
<<
"Average degree: "
<<
total_degree
/
ntotal
;
/////
/////
is_trained
=
true
;
is_trained
=
true
;
...
...
cpp/src/core/test/CMakeLists.txt
浏览文件 @
c6501ec0
...
@@ -29,7 +29,7 @@ set(util_srcs
...
@@ -29,7 +29,7 @@ set(util_srcs
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/adapter/ArrowAdapter.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/adapter/ArrowAdapter.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/common/Exception.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/common/Exception.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/common/Timer.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/common/Timer.cpp
utils.cpp
${
CORE_SOURCE_DIR
}
/test/
utils.cpp
)
)
#<IVF-TEST>
#<IVF-TEST>
...
@@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
...
@@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<IDMAP-TEST>
#<IDMAP-TEST>
set
(
idmap_srcs
set
(
idmap_srcs
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexIVF.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp
)
)
if
(
NOT TARGET test_idmap
)
if
(
NOT TARGET test_idmap
)
add_executable
(
test_idmap test_idmap.cpp
${
idmap_srcs
}
${
util_srcs
}
)
add_executable
(
test_idmap test_idmap.cpp
${
idmap_srcs
}
${
ivf_srcs
}
${
util_srcs
}
)
endif
()
endif
()
target_link_libraries
(
test_idmap
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
target_link_libraries
(
test_idmap
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
...
@@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest)
...
@@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest)
install
(
TARGETS test_kdt DESTINATION unittest
)
install
(
TARGETS test_kdt DESTINATION unittest
)
#add_subdirectory(faiss_ori)
#add_subdirectory(faiss_ori)
#
add_subdirectory(test_nsg)
add_subdirectory
(
test_nsg
)
cpp/src/core/test/test_nsg/CMakeLists.txt
浏览文件 @
c6501ec0
##############################
##############################
include_directories
(
/usr/local/include/gperftools
)
#
include_directories(/usr/local/include/gperftools)
link_directories
(
/usr/local/lib
)
#
link_directories(/usr/local/lib)
add_definitions
(
-std=c++11 -O3 -lboost -march=native -Wall -DINFO
)
add_definitions
(
-std=c++11 -O3 -lboost -march=native -Wall -DINFO
)
...
@@ -13,29 +13,15 @@ else ()
...
@@ -13,29 +13,15 @@ else ()
endif
()
endif
()
message
(
${
OpenMP_CXX_FLAGS
}
)
message
(
${
OpenMP_CXX_FLAGS
}
)
include_directories
(
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/nsg
)
include_directories
(
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/nsg
)
aux_source_directory
(
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/nsg nsg_src
)
aux_source_directory
(
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/nsg nsg_src
)
set
(
interface_src
set
(
interface_src
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/ivf.cpp
${
CORE_SOURCE_DIR
}
/knowhere/knowhere/index/vector_index/IndexNSG.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/gpu_ivf.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/cloner.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/idmap.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/index/vector_index/nsg_index.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/adapter/structure.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/common/exception.cpp
${
CORE_SOURCE_DIR
}
/src/knowhere/common/timer.cpp
../utils.cpp
)
)
if
(
NOT TARGET test_nsg
)
if
(
NOT TARGET test_nsg
)
add_executable
(
test_nsg
add_executable
(
test_nsg test_nsg.cpp
${
interface_src
}
${
nsg_src
}
${
util_srcs
}
${
ivf_srcs
}
)
test_nsg.cpp
${
interface_src
}
${
nsg_src
}
${
util_srcs
}
)
endif
()
endif
()
target_link_libraries
(
test_nsg
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
target_link_libraries
(
test_nsg
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
...
...
cpp/src/core/test/test_nsg/test_nsg.cpp
浏览文件 @
c6501ec0
...
@@ -19,10 +19,11 @@
...
@@ -19,10 +19,11 @@
#include <gtest/gtest.h>
#include <gtest/gtest.h>
#include <memory>
#include <memory>
#include "knowhere/common/exception.h"
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/FaissBaseIndex.h"
#include "knowhere/index/vector_index/nsg_index.h"
#include "knowhere/index/vector_index/IndexNSG.h"
#include "knowhere/index/vector_index/nsg/nsg_io.h"
#include "knowhere/index/vector_index/nsg/NSGIO.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "../utils.h"
#include "../utils.h"
...
@@ -32,16 +33,31 @@ using ::testing::TestWithParam;
...
@@ -32,16 +33,31 @@ using ::testing::TestWithParam;
using
::
testing
::
Values
;
using
::
testing
::
Values
;
using
::
testing
::
Combine
;
using
::
testing
::
Combine
;
constexpr
int64_t
DEVICE_ID
=
0
;
constexpr
int64_t
DEVICE_ID
=
1
;
class
NSGInterfaceTest
:
public
DataGen
,
public
TestWithParam
<::
std
::
tuple
<
Config
,
Config
>>
{
class
NSGInterfaceTest
:
public
DataGen
,
public
::
testing
::
Test
{
protected:
protected:
void
SetUp
()
override
{
void
SetUp
()
override
{
//Init_with_default();
//Init_with_default();
FaissGpuResourceMgr
::
GetInstance
().
InitDevice
(
DEVICE_ID
,
1024
*
1024
*
200
,
1024
*
1024
*
600
,
2
);
FaissGpuResourceMgr
::
GetInstance
().
InitDevice
(
DEVICE_ID
,
1024
*
1024
*
200
,
1024
*
1024
*
600
,
2
);
Generate
(
256
,
10000
,
1
);
Generate
(
256
,
10000
00
,
1
);
index_
=
std
::
make_shared
<
NSG
>
();
index_
=
std
::
make_shared
<
NSG
>
();
std
::
tie
(
train_cfg
,
search_cfg
)
=
GetParam
();
auto
tmp_conf
=
std
::
make_shared
<
NSGCfg
>
();
tmp_conf
->
gpu_id
=
DEVICE_ID
;
tmp_conf
->
knng
=
100
;
tmp_conf
->
nprobe
=
32
;
tmp_conf
->
nlist
=
16384
;
tmp_conf
->
search_length
=
60
;
tmp_conf
->
out_degree
=
70
;
tmp_conf
->
candidate_pool_size
=
500
;
tmp_conf
->
metric_type
=
METRICTYPE
::
L2
;
train_conf
=
tmp_conf
;
auto
tmp2_conf
=
std
::
make_shared
<
NSGCfg
>
();
tmp2_conf
->
k
=
k
;
tmp2_conf
->
search_length
=
30
;
search_conf
=
tmp2_conf
;
}
}
void
TearDown
()
override
{
void
TearDown
()
override
{
...
@@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Confi
...
@@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Confi
protected:
protected:
std
::
shared_ptr
<
NSG
>
index_
;
std
::
shared_ptr
<
NSG
>
index_
;
Config
train_c
fg
;
Config
train_c
onf
;
Config
search_c
fg
;
Config
search_c
onf
;
};
};
INSTANTIATE_TEST_CASE_P
(
NSGparameters
,
NSGInterfaceTest
,
Values
(
std
::
make_tuple
(
// search length > out_degree
Config
::
object
{{
"nlist"
,
128
},
{
"nprobe"
,
50
},
{
"knng"
,
100
},
{
"metric_type"
,
"L2"
},
{
"search_length"
,
60
},
{
"out_degree"
,
70
},
{
"candidate_pool_size"
,
500
}},
Config
::
object
{{
"k"
,
20
},
{
"search_length"
,
30
}}))
);
void
AssertAnns
(
const
DatasetPtr
&
result
,
void
AssertAnns
(
const
DatasetPtr
&
result
,
const
int
&
nq
,
const
int
&
nq
,
const
int
&
k
)
{
const
int
&
k
)
{
...
@@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result,
...
@@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result,
}
}
}
}
TEST_
P
(
NSGInterfaceTest
,
basic_test
)
{
TEST_
F
(
NSGInterfaceTest
,
basic_test
)
{
assert
(
!
xb
.
empty
());
assert
(
!
xb
.
empty
());
auto
model
=
index_
->
Train
(
base_dataset
,
train_c
fg
);
auto
model
=
index_
->
Train
(
base_dataset
,
train_c
onf
);
auto
result
=
index_
->
Search
(
query_dataset
,
search_c
fg
);
auto
result
=
index_
->
Search
(
query_dataset
,
search_c
onf
);
AssertAnns
(
result
,
nq
,
k
);
AssertAnns
(
result
,
nq
,
k
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
();
auto
new_index
=
std
::
make_shared
<
NSG
>
();
auto
new_index
=
std
::
make_shared
<
NSG
>
();
new_index
->
Load
(
binaryset
);
new_index
->
Load
(
binaryset
);
auto
new_result
=
new_index
->
Search
(
query_dataset
,
Config
::
object
{{
"k"
,
k
}}
);
auto
new_result
=
new_index
->
Search
(
query_dataset
,
search_conf
);
AssertAnns
(
result
,
nq
,
k
);
AssertAnns
(
result
,
nq
,
k
);
ASSERT_EQ
(
index_
->
Count
(),
nb
);
ASSERT_EQ
(
index_
->
Count
(),
nb
);
...
...
cpp/src/wrapper/ConfAdapter.cpp
浏览文件 @
c6501ec0
...
@@ -145,6 +145,10 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) {
...
@@ -145,6 +145,10 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) {
conf
->
out_degree
=
50
+
5
*
scale_factor
;
conf
->
out_degree
=
50
+
5
*
scale_factor
;
conf
->
candidate_pool_size
=
200
+
100
*
scale_factor
;
conf
->
candidate_pool_size
=
200
+
100
*
scale_factor
;
MatchBase
(
conf
);
MatchBase
(
conf
);
// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist
// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d
// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng;
return
conf
;
return
conf
;
}
}
...
...
cpp/src/wrapper/ConfAdapter.h
浏览文件 @
c6501ec0
...
@@ -47,6 +47,9 @@ class ConfAdapter {
...
@@ -47,6 +47,9 @@ class ConfAdapter {
virtual
knowhere
::
Config
virtual
knowhere
::
Config
MatchSearch
(
const
TempMetaConf
&
metaconf
,
const
IndexType
&
type
);
MatchSearch
(
const
TempMetaConf
&
metaconf
,
const
IndexType
&
type
);
// virtual void
// Dump(){}
protected:
protected:
static
void
MatchBase
(
knowhere
::
Config
conf
);
static
void
MatchBase
(
knowhere
::
Config
conf
);
};
};
...
...
cpp/src/wrapper/VecIndex.cpp
浏览文件 @
c6501ec0
...
@@ -137,7 +137,7 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) {
...
@@ -137,7 +137,7 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) {
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVFSQ
>
(
gpu_device
);
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVFSQ
>
(
gpu_device
);
break
;
break
;
}
}
case
IndexType
::
NSG_MIX
:
{
// TODO(linxj): bug.
case
IndexType
::
NSG_MIX
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
NSG
>
(
gpu_device
);
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
NSG
>
(
gpu_device
);
break
;
break
;
}
}
...
...
cpp/unittest/wrapper/wrapper_test.cpp
浏览文件 @
c6501ec0
...
@@ -173,7 +173,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
...
@@ -173,7 +173,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_CPU
,
"Default"
,
DIM
,
NB
,
10
,
10
),
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_CPU
,
"Default"
,
DIM
,
NB
,
10
,
10
),
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_GPU
,
"Default"
,
DIM
,
NB
,
10
,
10
),
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_GPU
,
"Default"
,
DIM
,
NB
,
10
,
10
),
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_MIX
,
"Default"
,
DIM
,
NB
,
10
,
10
),
std
::
make_tuple
(
IndexType
::
FAISS_IVFSQ8_MIX
,
"Default"
,
DIM
,
NB
,
10
,
10
),
//
std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
std
::
make_tuple
(
IndexType
::
NSG_MIX
,
"Default"
,
128
,
250000
,
10
,
10
),
// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
std
::
make_tuple
(
IndexType
::
FAISS_IDMAP
,
"Default"
,
64
,
100000
,
10
,
10
)
std
::
make_tuple
(
IndexType
::
FAISS_IDMAP
,
"Default"
,
64
,
100000
,
10
,
10
)
)
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录