Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
d5eb5675
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d5eb5675
编写于
11月 18, 2019
作者:
X
xiaojun.lin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix by reduce knng size
上级
fa4f0a07
变更
9
显示空白变更内容
内联
并排
Showing
9 changed file
with
188 addition
and
29 deletion
+188
-29
core/src/index/CMakeLists.txt
core/src/index/CMakeLists.txt
+3
-3
core/src/index/knowhere/knowhere/common/Config.h
core/src/index/knowhere/knowhere/common/Config.h
+14
-0
core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp
...re/knowhere/index/vector_index/helpers/IndexParameter.cpp
+22
-0
core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h
...here/knowhere/index/vector_index/helpers/IndexParameter.h
+9
-0
core/src/wrapper/ConfAdapter.cpp
core/src/wrapper/ConfAdapter.cpp
+12
-15
core/src/wrapper/ConfAdapter.h
core/src/wrapper/ConfAdapter.h
+0
-3
core/unittest/wrapper/test_wrapper.cpp
core/unittest/wrapper/test_wrapper.cpp
+117
-4
core/unittest/wrapper/utils.cpp
core/unittest/wrapper/utils.cpp
+5
-0
core/unittest/wrapper/utils.h
core/unittest/wrapper/utils.h
+6
-4
未找到文件。
core/src/index/CMakeLists.txt
浏览文件 @
d5eb5675
...
...
@@ -88,14 +88,14 @@ endif ()
include
(
ThirdPartyPackagesCore
)
if
(
CMAKE_BUILD_TYPE STREQUAL
"Release"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O3 -fPIC -DELPP_THREAD_SAFE -fopenmp"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O3 -fPIC -DELPP_THREAD_SAFE -fopenmp
-mavx -mf16c -msse4 -mpopcnt
"
)
if
(
KNOWHERE_GPU_VERSION
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-O3"
)
endif
()
else
()
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O
0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp
"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O
3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt
"
)
if
(
KNOWHERE_GPU_VERSION
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-O
0
-g"
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-O
3
-g"
)
endif
()
endif
()
...
...
core/src/index/knowhere/knowhere/common/Config.h
浏览文件 @
d5eb5675
...
...
@@ -18,6 +18,8 @@
#pragma once
#include <memory>
#include <sstream>
#include "Log.h"
namespace
knowhere
{
...
...
@@ -50,6 +52,18 @@ struct Cfg {
CheckValid
()
{
return
true
;
}
void
Dump
()
{
KNOWHERE_LOG_DEBUG
<<
DumpImpl
().
str
();
}
virtual
std
::
stringstream
DumpImpl
()
{
std
::
stringstream
ss
;
ss
<<
"dim: "
<<
d
<<
", metric: "
<<
int
(
metric_type
)
<<
", gpuid: "
<<
gpu_id
<<
", k: "
<<
k
;
return
ss
;
}
};
using
Config
=
std
::
shared_ptr
<
Cfg
>
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp
浏览文件 @
d5eb5675
...
...
@@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) {
KNOWHERE_THROW_MSG
(
"Metric type is invalid"
);
}
std
::
stringstream
IVFCfg
::
DumpImpl
()
{
auto
ss
=
Cfg
::
DumpImpl
();
ss
<<
", nlist: "
<<
nlist
<<
", nprobe: "
<<
nprobe
;
return
ss
;
}
std
::
stringstream
IVFSQCfg
::
DumpImpl
()
{
auto
ss
=
IVFCfg
::
DumpImpl
();
ss
<<
", nbits: "
<<
nbits
;
return
ss
;
}
std
::
stringstream
NSGCfg
::
DumpImpl
()
{
auto
ss
=
IVFCfg
::
DumpImpl
();
ss
<<
", knng: "
<<
knng
<<
", search_length: "
<<
search_length
<<
", out_degree: "
<<
out_degree
<<
", candidate: "
<<
candidate_pool_size
;
return
ss
;
}
}
// namespace knowhere
core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h
浏览文件 @
d5eb5675
...
...
@@ -53,6 +53,9 @@ struct IVFCfg : public Cfg {
IVFCfg
()
=
default
;
std
::
stringstream
DumpImpl
()
override
;
bool
CheckValid
()
override
{
return
true
;
...
...
@@ -69,6 +72,9 @@ struct IVFSQCfg : public IVFCfg {
:
IVFCfg
(
dim
,
k
,
gpu_id
,
nlist
,
nprobe
,
type
),
nbits
(
nbits
)
{
}
std
::
stringstream
DumpImpl
()
override
;
IVFSQCfg
()
=
default
;
bool
...
...
@@ -119,6 +125,9 @@ struct NSGCfg : public IVFCfg {
NSGCfg
()
=
default
;
std
::
stringstream
DumpImpl
()
override
;
bool
CheckValid
()
override
{
return
true
;
...
...
core/src/wrapper/ConfAdapter.cpp
浏览文件 @
d5eb5675
...
...
@@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) {
auto
conf
=
std
::
make_shared
<
knowhere
::
Cfg
>
();
conf
->
d
=
metaconf
.
dim
;
conf
->
metric_type
=
metaconf
.
metric_type
;
conf
->
gpu_id
=
conf
->
gpu_id
;
conf
->
gpu_id
=
metaconf
.
gpu_id
;
conf
->
k
=
metaconf
.
k
;
MatchBase
(
conf
);
return
conf
;
}
...
...
@@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) {
conf
->
nlist
=
MatchNlist
(
metaconf
.
size
,
metaconf
.
nlist
);
conf
->
d
=
metaconf
.
dim
;
conf
->
metric_type
=
metaconf
.
metric_type
;
conf
->
gpu_id
=
conf
->
gpu_id
;
conf
->
gpu_id
=
metaconf
.
gpu_id
;
MatchBase
(
conf
);
return
conf
;
}
...
...
@@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) {
conf
->
nlist
=
MatchNlist
(
metaconf
.
size
,
metaconf
.
nlist
);
conf
->
d
=
metaconf
.
dim
;
conf
->
metric_type
=
metaconf
.
metric_type
;
conf
->
gpu_id
=
conf
->
gpu_id
;
conf
->
gpu_id
=
metaconf
.
gpu_id
;
conf
->
nbits
=
8
;
MatchBase
(
conf
);
return
conf
;
...
...
@@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) {
conf
->
nlist
=
MatchNlist
(
metaconf
.
size
,
metaconf
.
nlist
);
conf
->
d
=
metaconf
.
dim
;
conf
->
metric_type
=
metaconf
.
metric_type
;
conf
->
gpu_id
=
conf
->
gpu_id
;
conf
->
gpu_id
=
metaconf
.
gpu_id
;
conf
->
nbits
=
8
;
if
(
!
(
conf
->
d
%
4
))
...
...
@@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) {
conf
->
nlist
=
MatchNlist
(
metaconf
.
size
,
metaconf
.
nlist
);
conf
->
d
=
metaconf
.
dim
;
conf
->
metric_type
=
metaconf
.
metric_type
;
conf
->
gpu_id
=
conf
->
gpu_id
;
conf
->
gpu_id
=
metaconf
.
gpu_id
;
conf
->
k
=
metaconf
.
k
;
double
factor
=
metaconf
.
size
/
TYPICAL_COUNT
;
auto
scale_factor
=
round
(
metaconf
.
dim
/
128.0
);
scale_factor
=
scale_factor
>=
4
?
4
:
scale_factor
;
conf
->
nprobe
=
conf
->
nlist
>
10000
?
conf
->
nlist
*
0.02
:
conf
->
nlist
*
0.1
;
conf
->
knng
=
(
100
+
100
*
scale_factor
)
*
factor
;
conf
->
search_length
=
(
40
+
5
*
scale_factor
)
*
factor
;
conf
->
out_degree
=
(
50
+
5
*
scale_factor
)
*
factor
;
conf
->
candidate_pool_size
=
(
200
+
100
*
scale_factor
)
*
factor
;
conf
->
nprobe
=
int64_t
(
conf
->
nlist
*
0.01
)
;
conf
->
knng
=
40
+
10
*
scale_factor
;
// the size of knng
conf
->
search_length
=
40
+
5
*
scale_
factor
;
conf
->
out_degree
=
50
+
5
*
scale_
factor
;
conf
->
candidate_pool_size
=
200
+
100
*
scale_
factor
;
MatchBase
(
conf
);
// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist
// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d
// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng;
return
conf
;
}
...
...
core/src/wrapper/ConfAdapter.h
浏览文件 @
d5eb5675
...
...
@@ -46,9 +46,6 @@ class ConfAdapter {
virtual
knowhere
::
Config
MatchSearch
(
const
TempMetaConf
&
metaconf
,
const
IndexType
&
type
);
// virtual void
// Dump(){}
protected:
static
void
MatchBase
(
knowhere
::
Config
conf
);
...
...
core/unittest/wrapper/test_wrapper.cpp
浏览文件 @
d5eb5675
...
...
@@ -56,10 +56,6 @@ class KnowhereWrapperTest
index_
=
GetVecIndexFactory
(
index_type
);
conf
=
ParamGenerator
::
GetInstance
().
GenBuild
(
index_type
,
tempconf
);
searchconf
=
ParamGenerator
::
GetInstance
().
GenSearchConf
(
index_type
,
tempconf
);
// conf->k = k;
// conf->d = dim;
// conf->gpu_id = DEVICEID;
}
void
TearDown
()
override
{
...
...
@@ -97,6 +93,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
TEST_P
(
KnowhereWrapperTest
,
BASE_TEST
)
{
EXPECT_EQ
(
index_
->
GetType
(),
index_type
);
// conf->Dump();
auto
elems
=
nq
*
k
;
std
::
vector
<
int64_t
>
res_ids
(
elems
);
...
...
@@ -191,3 +188,119 @@ TEST(whatever, test_config) {
auto
pq_conf
=
std
::
make_shared
<
milvus
::
engine
::
IVFPQConfAdapter
>
();
pq_conf
->
Match
(
conf
);
}
// #include "knowhere/index/vector_index/IndexIDMAP.h"
// #include "src/wrapper/VecImpl.h"
// #include "src/index/unittest/utils.h"
// The two case below prove NSG is concern with data distribution
// Further work: 1. Use right basedata and pass it by milvus
// a. batch size is 100000 [Pass]
// b. transfer all at once [Pass]
// 2. Use SIFT1M in test and check time cost []
// TEST_P(KnowhereWrapperTest, nsgwithidmap) {
// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP);
// auto ori_xb = xb;
// auto ori_ids = ids;
// std::vector<float> temp_xb;
// std::vector<int64_t> temp_ids;
// nb = 50000;
// for (int i = 0; i < 20; ++i) {
// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis);
// assert(xb.size() == nb*dim);
// //#define IDMAP
// #ifdef IDMAP
// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim);
// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1));
// if (i == 0) {
// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf);
// } else {
// idmap->Add(nb, temp_xb.data(), temp_ids.data());
// }
// temp_xb.clear();
// temp_ids.clear();
// #else
// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim);
// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1));
// #endif
// }
// #ifdef IDMAP
// auto idmap_idx = std::dynamic_pointer_cast<milvus::engine::BFIndex>(idmap);
// auto x = idmap_idx->Count();
// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf);
// #else
// assert(temp_xb.size() == 1000000*128);
// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf);
// #endif
// }
// TEST_P(KnowhereWrapperTest, nsgwithsidmap) {
// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP);
// auto ori_xb = xb;
// std::vector<float> temp_xb;
// std::vector<int64_t> temp_ids;
// nb = 50000;
// for (int i = 0; i < 20; ++i) {
// #define IDMAP
// #ifdef IDMAP
// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1));
// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1));
// if (i == 0) {
// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf);
// } else {
// idmap->Add(nb, temp_xb.data(), temp_ids.data());
// }
// temp_xb.clear();
// temp_ids.clear();
// #else
// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1));
// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1));
// #endif
// }
// #ifdef IDMAP
// auto idmap_idx = std::dynamic_pointer_cast<milvus::engine::BFIndex>(idmap);
// auto x = idmap_idx->Count();
// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf);
// #else
// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf);
// #endif
// // The code use to store raw base data
// FileIOWriter writer("/tmp/newraw");
// ori_xb.shrink_to_fit();
// std::cout << "size" << ori_xb.size();
// writer(static_cast<void*>(ori_xb.data()), ori_xb.size()* sizeof(float));
// std::cout << "Finish!" << std::endl;
// }
// void load_data(char* filename, float*& data, unsigned& num,
// unsigned& dim) { // load data with sift10K pattern
// std::ifstream in(filename, std::ios::binary);
// if (!in.is_open()) {
// std::cout << "open file error" << std::endl;
// exit(-1);
// }
// in.read((char*)&dim, 4);
// in.seekg(0, std::ios::end);
// std::ios::pos_type ss = in.tellg();
// size_t fsize = (size_t)ss;
// num = (unsigned)(fsize / (dim + 1) / 4);
// data = new float[(size_t)num * (size_t)dim];
// in.seekg(0, std::ios::beg);
// for (size_t i = 0; i < num; i++) {
// in.seekg(4, std::ios::cur);
// in.read((char*)(data + i * dim), dim * 4);
// }
// in.close();
// }
// TEST_P(KnowhereWrapperTest, Sift1M) {
// float* data = nullptr;
// unsigned points_num, dim;
// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num,
// dim); std::cout << points_num << " " << dim << std::endl;
// index_->BuildAll(points_num, data, ids.data(), conf);
// }
\ No newline at end of file
core/unittest/wrapper/utils.cpp
浏览文件 @
d5eb5675
...
...
@@ -117,6 +117,11 @@ void
DataGenBase
::
GenData
(
const
int
&
dim
,
const
int
&
nb
,
const
int
&
nq
,
std
::
vector
<
float
>&
xb
,
std
::
vector
<
float
>&
xq
,
std
::
vector
<
int64_t
>&
ids
,
const
int
&
k
,
std
::
vector
<
int64_t
>&
gt_ids
,
std
::
vector
<
float
>&
gt_dis
)
{
xb
.
clear
();
xq
.
clear
();
ids
.
clear
();
gt_ids
.
clear
();
gt_dis
.
clear
();
xb
.
resize
(
nb
*
dim
);
xq
.
resize
(
nq
*
dim
);
ids
.
resize
(
nb
);
...
...
core/unittest/wrapper/utils.h
浏览文件 @
d5eb5675
...
...
@@ -25,7 +25,6 @@
#include <cstdio>
#include <fstream>
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
...
...
@@ -90,17 +89,20 @@ class ParamGenerator {
return
instance
;
}
knowhere
::
Config
GenSearchConf
(
const
milvus
::
engine
::
IndexType
&
type
,
const
milvus
::
engine
::
TempMetaConf
&
conf
)
{
knowhere
::
Config
GenSearchConf
(
const
milvus
::
engine
::
IndexType
&
type
,
const
milvus
::
engine
::
TempMetaConf
&
conf
)
{
auto
adapter
=
milvus
::
engine
::
AdapterMgr
::
GetInstance
().
GetAdapter
(
type
);
return
adapter
->
MatchSearch
(
conf
,
type
);
}
knowhere
::
Config
GenBuild
(
const
milvus
::
engine
::
IndexType
&
type
,
const
milvus
::
engine
::
TempMetaConf
&
conf
)
{
knowhere
::
Config
GenBuild
(
const
milvus
::
engine
::
IndexType
&
type
,
const
milvus
::
engine
::
TempMetaConf
&
conf
)
{
auto
adapter
=
milvus
::
engine
::
AdapterMgr
::
GetInstance
().
GetAdapter
(
type
);
return
adapter
->
Match
(
conf
);
}
knowhere
::
Config
Gen
(
const
milvus
::
engine
::
IndexType
&
type
)
{
knowhere
::
Config
Gen
(
const
milvus
::
engine
::
IndexType
&
type
)
{
switch
(
type
)
{
case
milvus
::
engine
::
IndexType
::
FAISS_IDMAP
:
{
auto
tempconf
=
std
::
make_shared
<
knowhere
::
Cfg
>
();
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录