Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
fae3220c
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
fae3220c
编写于
12月 05, 2019
作者:
J
Jin Hai
提交者:
GitHub
12月 05, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #711 from milvus-io/0.6.0
Merge from 0.6.0
上级
ae630a80
ebb4cede
变更
15
隐藏空白更改
内联
并排
Showing
15 changed file
with
59 addition
and
78 deletion
+59
-78
CHANGELOG.md
CHANGELOG.md
+3
-0
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+4
-2
core/src/db/meta/MySQLMetaImpl.cpp
core/src/db/meta/MySQLMetaImpl.cpp
+0
-2
core/src/db/meta/SqliteMetaImpl.cpp
core/src/db/meta/SqliteMetaImpl.cpp
+1
-3
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp
...knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp
+0
-56
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h
...x/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
...x/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
+2
-0
core/src/index/unittest/Helper.h
core/src/index/unittest/Helper.h
+2
-0
core/src/scheduler/task/BuildIndexTask.cpp
core/src/scheduler/task/BuildIndexTask.cpp
+3
-2
core/src/scheduler/task/SearchTask.cpp
core/src/scheduler/task/SearchTask.cpp
+2
-2
core/src/wrapper/VecImpl.h
core/src/wrapper/VecImpl.h
+1
-1
core/src/wrapper/VecIndex.cpp
core/src/wrapper/VecIndex.cpp
+17
-7
core/src/wrapper/gpu/GPUVecImpl.cpp
core/src/wrapper/gpu/GPUVecImpl.cpp
+2
-0
core/unittest/db/test_db.cpp
core/unittest/db/test_db.cpp
+20
-1
install.md
install.md
+1
-1
未找到文件。
CHANGELOG.md
浏览文件 @
fae3220c
...
...
@@ -52,6 +52,9 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
658 - Milvus error out when building SQ8H index without GPU resources
-
\#
668 - Update badge of README
-
\#
670 - Random failure of unittest db_test::SEARCH_TEST
-
\#
674 - Server down in stability test
-
\#
696 - Metric_type changed from IP to L2
-
\#
705 - Fix search SQ8H crash without GPU resource
## Feature
-
\#
12 - Pure CPU version for Milvus
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
fae3220c
...
...
@@ -400,10 +400,11 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
Status
ExecutionEngineImpl
::
CopyToIndexFileToGpu
(
uint64_t
device_id
)
{
#ifdef MILVUS_GPU_VERSION
// the ToIndexData is only a placeholder, cpu-copy-to-gpu action is performed in
gpu_num_
=
device_id
;
auto
to_index_data
=
std
::
make_shared
<
ToIndexData
>
(
PhysicalSize
());
cache
::
DataObjPtr
obj
=
std
::
static_pointer_cast
<
cache
::
DataObj
>
(
to_index_data
);
milvus
::
cache
::
GpuCacheMgr
::
GetInstance
(
device_id
)
->
InsertItem
(
location_
,
obj
);
milvus
::
cache
::
GpuCacheMgr
::
GetInstance
(
device_id
)
->
InsertItem
(
location_
+
"_placeholder"
,
obj
);
#endif
return
Status
::
OK
();
}
...
...
@@ -603,7 +604,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr
}
if
(
!
status
.
ok
())
{
ENGINE_LOG_ERROR
<<
"Search error
"
;
ENGINE_LOG_ERROR
<<
"Search error
:"
<<
status
.
message
()
;
}
return
status
;
}
...
...
@@ -634,6 +635,7 @@ ExecutionEngineImpl::Init() {
Status
s
=
config
.
GetGpuResourceConfigBuildIndexResources
(
gpu_ids
);
if
(
!
s
.
ok
())
{
gpu_num_
=
knowhere
::
INVALID_VALUE
;
return
s
;
}
for
(
auto
id
:
gpu_ids
)
{
if
(
gpu_num_
==
id
)
{
...
...
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
fae3220c
...
...
@@ -1138,8 +1138,6 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) {
// set table index type to raw
dropTableIndexQuery
<<
"UPDATE "
<<
META_TABLES
<<
" SET engine_type = "
<<
std
::
to_string
(
DEFAULT_ENGINE_TYPE
)
<<
" ,nlist = "
<<
std
::
to_string
(
DEFAULT_NLIST
)
<<
" ,metric_type = "
<<
std
::
to_string
(
DEFAULT_METRIC_TYPE
)
<<
" WHERE table_id = "
<<
mysqlpp
::
quote
<<
table_id
<<
";"
;
ENGINE_LOG_DEBUG
<<
"MySQLMetaImpl::DropTableIndex: "
<<
dropTableIndexQuery
.
str
();
...
...
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
fae3220c
...
...
@@ -733,9 +733,7 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
//set table index type to raw
ConnectorPtr
->
update_all
(
set
(
c
(
&
TableSchema
::
engine_type_
)
=
DEFAULT_ENGINE_TYPE
,
c
(
&
TableSchema
::
nlist_
)
=
DEFAULT_NLIST
,
c
(
&
TableSchema
::
metric_type_
)
=
DEFAULT_METRIC_TYPE
),
c
(
&
TableSchema
::
engine_type_
)
=
DEFAULT_ENGINE_TYPE
),
where
(
c
(
&
TableSchema
::
table_id_
)
==
table_id
));
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp
浏览文件 @
fae3220c
...
...
@@ -295,61 +295,5 @@ FaissIVFQuantizer::~FaissIVFQuantizer() {
// else do nothing
}
#else
QuantizerPtr
IVFSQHybrid
::
LoadQuantizer
(
const
Config
&
conf
)
{
return
knowhere
::
QuantizerPtr
();
}
void
IVFSQHybrid
::
SetQuantizer
(
const
QuantizerPtr
&
q
)
{
}
void
IVFSQHybrid
::
UnsetQuantizer
()
{
}
VectorIndexPtr
IVFSQHybrid
::
LoadData
(
const
knowhere
::
QuantizerPtr
&
q
,
const
Config
&
conf
)
{
return
nullptr
;
}
std
::
pair
<
VectorIndexPtr
,
QuantizerPtr
>
IVFSQHybrid
::
CopyCpuToGpuWithQuantizer
(
const
int64_t
&
device_id
,
const
Config
&
config
)
{
KNOWHERE_THROW_MSG
(
"Not yet implemented"
);
}
IndexModelPtr
IVFSQHybrid
::
Train
(
const
DatasetPtr
&
dataset
,
const
Config
&
config
)
{
return
GPUIVFSQ
::
Train
(
dataset
,
config
);
}
VectorIndexPtr
IVFSQHybrid
::
CopyGpuToCpu
(
const
Config
&
config
)
{
return
GPUIVFSQ
::
CopyGpuToCpu
(
config
);
}
VectorIndexPtr
IVFSQHybrid
::
CopyCpuToGpu
(
const
int64_t
&
device_id
,
const
Config
&
config
)
{
return
IVF
::
CopyCpuToGpu
(
device_id
,
config
);
}
void
IVFSQHybrid
::
search_impl
(
int64_t
n
,
const
float
*
data
,
int64_t
k
,
float
*
distances
,
int64_t
*
labels
,
const
Config
&
cfg
)
{
GPUIVF
::
search_impl
(
n
,
data
,
k
,
distances
,
labels
,
cfg
);
}
void
IVFSQHybrid
::
LoadImpl
(
const
BinarySet
&
index_binary
)
{
GPUIVF
::
LoadImpl
(
index_binary
);
}
void
IVFSQHybrid
::
set_index_model
(
IndexModelPtr
model
)
{
GPUIVF
::
set_index_model
(
model
);
}
#endif
}
// namespace knowhere
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h
浏览文件 @
fae3220c
...
...
@@ -36,7 +36,6 @@ struct FaissIVFQuantizer : public Quantizer {
~
FaissIVFQuantizer
()
override
;
};
using
FaissIVFQuantizerPtr
=
std
::
shared_ptr
<
FaissIVFQuantizer
>
;
#endif
class
IVFSQHybrid
:
public
GPUIVFSQ
{
public:
...
...
@@ -93,5 +92,6 @@ class IVFSQHybrid : public GPUIVFSQ {
int64_t
gpu_mode
=
0
;
// 0,1,2
int64_t
quantizer_gpu_id_
=
-
1
;
};
#endif
}
// namespace knowhere
core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
浏览文件 @
fae3220c
...
...
@@ -38,9 +38,11 @@ CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) {
VectorIndexPtr
CopyCpuToGpu
(
const
VectorIndexPtr
&
index
,
const
int64_t
&
device_id
,
const
Config
&
config
)
{
#ifdef CUSTOMIZATION
if
(
auto
device_index
=
std
::
dynamic_pointer_cast
<
IVFSQHybrid
>
(
index
))
{
return
device_index
->
CopyCpuToGpu
(
device_id
,
config
);
}
#endif
if
(
auto
device_index
=
std
::
dynamic_pointer_cast
<
GPUIndex
>
(
index
))
{
return
device_index
->
CopyGpuToGpu
(
device_id
,
config
);
...
...
core/src/index/unittest/Helper.h
浏览文件 @
fae3220c
...
...
@@ -53,8 +53,10 @@ IndexFactory(const std::string& type) {
return
std
::
make_shared
<
knowhere
::
GPUIVFPQ
>
(
DEVICEID
);
}
else
if
(
type
==
"GPUIVFSQ"
)
{
return
std
::
make_shared
<
knowhere
::
GPUIVFSQ
>
(
DEVICEID
);
#ifdef CUSTOMIZATION
}
else
if
(
type
==
"IVFSQHybrid"
)
{
return
std
::
make_shared
<
knowhere
::
IVFSQHybrid
>
(
DEVICEID
);
#endif
#endif
}
}
...
...
core/src/scheduler/task/BuildIndexTask.cpp
浏览文件 @
fae3220c
...
...
@@ -55,7 +55,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
type_str
=
"DISK2CPU"
;
}
else
if
(
type
==
LoadType
::
CPU2GPU
)
{
stat
=
to_index_engine_
->
CopyToIndexFileToGpu
(
device_id
);
type_str
=
"CPU2GPU
"
;
type_str
=
"CPU2GPU
:"
+
std
::
to_string
(
device_id
)
;
}
else
{
error_msg
=
"Wrong load type"
;
stat
=
Status
(
SERVER_UNEXPECTED_ERROR
,
error_msg
);
...
...
@@ -86,7 +86,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
size_t
file_size
=
to_index_engine_
->
PhysicalSize
();
std
::
string
info
=
"
L
oad file id:"
+
std
::
to_string
(
file_
->
id_
)
+
" "
+
type_str
+
std
::
string
info
=
"
Build index task l
oad file id:"
+
std
::
to_string
(
file_
->
id_
)
+
" "
+
type_str
+
" file type:"
+
std
::
to_string
(
file_
->
file_type_
)
+
" size:"
+
std
::
to_string
(
file_size
)
+
" bytes from location: "
+
file_
->
location_
+
" totally cost"
;
double
span
=
rc
.
ElapseFromBegin
(
info
);
...
...
@@ -128,6 +128,7 @@ XBuildIndexTask::Execute() {
// step 3: build index
try
{
ENGINE_LOG_DEBUG
<<
"Begin build index for file:"
+
table_file
.
location_
;
index
=
to_index_engine_
->
BuildIndex
(
table_file
.
location_
,
(
EngineType
)
table_file
.
engine_type_
);
if
(
index
==
nullptr
)
{
throw
Exception
(
DB_ERROR
,
"index NULL"
);
...
...
core/src/scheduler/task/SearchTask.cpp
浏览文件 @
fae3220c
...
...
@@ -125,7 +125,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
hybrid
=
true
;
}
stat
=
index_engine_
->
CopyToGpu
(
device_id
,
hybrid
);
type_str
=
"CPU2GPU
"
;
type_str
=
"CPU2GPU
:"
+
std
::
to_string
(
device_id
)
;
}
else
if
(
type
==
LoadType
::
GPU2CPU
)
{
stat
=
index_engine_
->
CopyToCpu
();
type_str
=
"GPU2CPU"
;
...
...
@@ -160,7 +160,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
size_t
file_size
=
index_engine_
->
PhysicalSize
();
std
::
string
info
=
"
Load file id:"
+
std
::
to_string
(
file_
->
id_
)
+
std
::
string
info
=
"
Search task load file id:"
+
std
::
to_string
(
file_
->
id_
)
+
" "
+
type_str
+
" file type:"
+
std
::
to_string
(
file_
->
file_type_
)
+
" size:"
+
std
::
to_string
(
file_size
)
+
" bytes from location: "
+
file_
->
location_
+
" totally cost"
;
double
span
=
rc
.
ElapseFromBegin
(
info
);
...
...
core/src/wrapper/VecImpl.h
浏览文件 @
fae3220c
...
...
@@ -108,7 +108,7 @@ class ToIndexData : public cache::DataObj {
}
private:
int64_t
size_
;
int64_t
size_
=
0
;
};
}
// namespace engine
...
...
core/src/wrapper/VecIndex.cpp
浏览文件 @
fae3220c
...
...
@@ -24,6 +24,8 @@
#include "knowhere/index/vector_index/IndexIVFSQ.h"
#include "knowhere/index/vector_index/IndexNSG.h"
#include "knowhere/index/vector_index/IndexSPTAG.h"
#include "server/Config.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#ifdef MILVUS_GPU_VERSION
...
...
@@ -145,6 +147,10 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index
=
std
::
make_shared
<
knowhere
::
GPUIVF
>
(
gpu_device
);
break
;
}
case
IndexType
::
FAISS_IVFFLAT_MIX
:
{
index
=
std
::
make_shared
<
knowhere
::
GPUIVF
>
(
gpu_device
);
return
std
::
make_shared
<
IVFMixIndex
>
(
index
,
IndexType
::
FAISS_IVFFLAT_MIX
);
}
case
IndexType
::
FAISS_IVFPQ_GPU
:
{
index
=
std
::
make_shared
<
knowhere
::
GPUIVFPQ
>
(
gpu_device
);
break
;
...
...
@@ -161,16 +167,20 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index
=
std
::
make_shared
<
knowhere
::
GPUIVFSQ
>
(
gpu_device
);
break
;
}
case
IndexType
::
FAISS_IVFFLAT_MIX
:
{
index
=
std
::
make_shared
<
knowhere
::
GPUIVF
>
(
gpu_device
);
return
std
::
make_shared
<
IVFMixIndex
>
(
index
,
IndexType
::
FAISS_IVFFLAT_MIX
);
}
#endif
#ifdef CUSTOMIZATION
case
IndexType
::
FAISS_IVFSQ8_HYBRID
:
{
index
=
std
::
make_shared
<
knowhere
::
IVFSQHybrid
>
(
gpu_device
);
return
std
::
make_shared
<
IVFHybridIndex
>
(
index
,
IndexType
::
FAISS_IVFSQ8_HYBRID
);
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
bool
gpu_resource_enable
=
true
;
config
.
GetGpuResourceConfigEnable
(
gpu_resource_enable
);
if
(
gpu_resource_enable
)
{
index
=
std
::
make_shared
<
knowhere
::
IVFSQHybrid
>
(
gpu_device
);
return
std
::
make_shared
<
IVFHybridIndex
>
(
index
,
IndexType
::
FAISS_IVFSQ8_HYBRID
);
}
else
{
throw
Exception
(
DB_ERROR
,
"No GPU resources for IndexType::FAISS_IVFSQ8_HYBRID"
);
}
}
#endif
#endif
case
IndexType
::
NSG_MIX
:
{
// TODO(linxj): bug.
index
=
std
::
make_shared
<
knowhere
::
NSG
>
(
gpu_device
);
...
...
core/src/wrapper/gpu/GPUVecImpl.cpp
浏览文件 @
fae3220c
...
...
@@ -73,6 +73,7 @@ IVFMixIndex::Load(const knowhere::BinarySet& index_binary) {
return
Status
::
OK
();
}
#ifdef CUSTOMIZATION
knowhere
::
QuantizerPtr
IVFHybridIndex
::
LoadQuantizer
(
const
Config
&
conf
)
{
// TODO(linxj): Hardcode here
...
...
@@ -158,6 +159,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c
}
return
std
::
make_pair
(
nullptr
,
nullptr
);
}
#endif
}
// namespace engine
}
// namespace milvus
core/unittest/db/test_db.cpp
浏览文件 @
fae3220c
...
...
@@ -351,6 +351,19 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE
(
stat
.
ok
());
}
index
.
engine_type_
=
(
int
)
milvus
::
engine
::
EngineType
::
FAISS_PQ
;
db_
->
CreateIndex
(
TABLE_NAME
,
index
);
// wait until build index finish
{
std
::
vector
<
std
::
string
>
tags
;
milvus
::
engine
::
ResultIds
result_ids
;
milvus
::
engine
::
ResultDistances
result_distances
;
stat
=
db_
->
Query
(
TABLE_NAME
,
tags
,
k
,
nq
,
10
,
xq
.
data
(),
result_ids
,
result_distances
);
ASSERT_TRUE
(
stat
.
ok
());
stat
=
db_
->
Query
(
TABLE_NAME
,
tags
,
k
,
1100
,
10
,
xq
.
data
(),
result_ids
,
result_distances
);
ASSERT_TRUE
(
stat
.
ok
());
}
#ifdef CUSTOMIZATION
// test FAISS_IVFSQ8H optimizer
index
.
engine_type_
=
(
int
)
milvus
::
engine
::
EngineType
::
FAISS_IVFSQ8H
;
...
...
@@ -375,7 +388,13 @@ TEST_F(DBTest, SEARCH_TEST) {
{
// search by specify index file
milvus
::
engine
::
meta
::
DatesT
dates
;
std
::
vector
<
std
::
string
>
file_ids
=
{
"1"
,
"2"
,
"3"
,
"4"
,
"5"
,
"6"
};
std
::
vector
<
std
::
string
>
file_ids
;
// sometimes this case run fast to merge file and build index, old file will be deleted immediately,
// so the QueryByFileID cannot get files to search
// input 100 files ids to avoid random failure of this case
for
(
int
i
=
0
;
i
<
100
;
i
++
)
{
file_ids
.
push_back
(
std
::
to_string
(
i
));
}
result_ids
.
clear
();
result_dists
.
clear
();
stat
=
db_
->
QueryByFileID
(
TABLE_NAME
,
file_ids
,
k
,
nq
,
10
,
xq
.
data
(),
dates
,
result_ids
,
result_dists
);
...
...
install.md
浏览文件 @
fae3220c
...
...
@@ -103,7 +103,7 @@ $ docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18
Start a GPU container:
```
shell
$
docker run
—
runtime
=
nvidia
-it
-p
19530:19530
-d
milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04
$
docker run
--
runtime
=
nvidia
-it
-p
19530:19530
-d
milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04
```
To enter the container:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录