Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
qq_34331598
milvus
提交
f330d9f9
milvus
项目概览
qq_34331598
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f330d9f9
编写于
12月 03, 2019
作者:
Y
yudong.cai
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
#658 Milvus error out when building SQ8H index without GPU resources
上级
c974392b
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
14 addition
and
24 deletion
+14
-24
CHANGELOG.md
CHANGELOG.md
+1
-0
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+5
-1
core/src/scheduler/task/BuildIndexTask.cpp
core/src/scheduler/task/BuildIndexTask.cpp
+4
-12
core/src/sdk/examples/partition/src/ClientTest.cpp
core/src/sdk/examples/partition/src/ClientTest.cpp
+2
-5
core/src/sdk/examples/simple/src/ClientTest.cpp
core/src/sdk/examples/simple/src/ClientTest.cpp
+1
-4
core/src/sdk/examples/utils/Utils.cpp
core/src/sdk/examples/utils/Utils.cpp
+1
-2
未找到文件。
CHANGELOG.md
浏览文件 @
f330d9f9
...
...
@@ -47,6 +47,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
631 - FAISS isn't compiled with O3 option
-
\#
649 - Typo "partiton" should be "partition"
-
\#
654 - Random crash when frequently insert vector one by one
-
\#
658 - Milvus error out when building SQ8H index without GPU resources
## Feature
-
\#
12 - Pure CPU version for Milvus
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
f330d9f9
...
...
@@ -121,7 +121,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
}
#ifdef CUSTOMIZATION
case
EngineType
::
FAISS_IVFSQ8H
:
{
index
=
GetVecIndexFactory
(
IndexType
::
FAISS_IVFSQ8_HYBRID
);
if
(
gpu_resource_enable
)
{
index
=
GetVecIndexFactory
(
IndexType
::
FAISS_IVFSQ8_HYBRID
);
}
else
{
throw
Exception
(
DB_ERROR
,
"No GPU resources for IVFSQ8H"
);
}
break
;
}
#endif
...
...
core/src/scheduler/task/BuildIndexTask.cpp
浏览文件 @
f330d9f9
...
...
@@ -19,6 +19,7 @@
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/job/BuildIndexJob.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
...
...
@@ -129,24 +130,15 @@ XBuildIndexTask::Execute() {
try
{
index
=
to_index_engine_
->
BuildIndex
(
table_file
.
location_
,
(
EngineType
)
table_file
.
engine_type_
);
if
(
index
==
nullptr
)
{
table_file
.
file_type_
=
engine
::
meta
::
TableFileSchema
::
TO_DELETE
;
status
=
meta_ptr
->
UpdateTableFile
(
table_file
);
ENGINE_LOG_DEBUG
<<
"Failed to update file to index, mark file: "
<<
table_file
.
file_id_
<<
" to to_delete"
;
build_index_job
->
BuildIndexDone
(
to_index_id_
);
to_index_engine_
=
nullptr
;
return
;
throw
Exception
(
DB_ERROR
,
"index NULL"
);
}
}
catch
(
std
::
exception
&
ex
)
{
std
::
string
msg
=
"Build
Index encounter
exception: "
+
std
::
string
(
ex
.
what
());
std
::
string
msg
=
"Build
index
exception: "
+
std
::
string
(
ex
.
what
());
ENGINE_LOG_ERROR
<<
msg
;
table_file
.
file_type_
=
engine
::
meta
::
TableFileSchema
::
TO_DELETE
;
status
=
meta_ptr
->
UpdateTableFile
(
table_file
);
ENGINE_LOG_DEBUG
<<
"Failed to update file to index, mark file: "
<<
table_file
.
file_id_
<<
" to to_delete"
;
ENGINE_LOG_ERROR
<<
"Failed to build index, index file is too large or gpu memory is not enough"
;
ENGINE_LOG_DEBUG
<<
"Build index fail, mark file: "
<<
table_file
.
file_id_
<<
" to to_delete"
;
build_index_job
->
BuildIndexDone
(
to_index_id_
);
build_index_job
->
GetStatus
()
=
Status
(
DB_ERROR
,
msg
);
...
...
core/src/sdk/examples/partition/src/ClientTest.cpp
浏览文件 @
f330d9f9
...
...
@@ -17,15 +17,12 @@
#include "sdk/examples/partition/src/ClientTest.h"
#include "MilvusApi.h"
#include "sdk/examples/utils/TimeRecorder.h"
#include "sdk/examples/utils/Utils.h"
#include <src/sdk/examples/utils/TimeRecorder.h>
#include <time.h>
#include <unistd.h>
#include <chrono>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
...
...
@@ -42,7 +39,7 @@ constexpr int64_t TOP_K = 10;
constexpr
int64_t
NPROBE
=
32
;
constexpr
int64_t
SEARCH_TARGET
=
5000
;
// change this value, result is different
constexpr
milvus
::
IndexType
INDEX_TYPE
=
milvus
::
IndexType
::
IVFSQ8
;
constexpr
int32_t
N_LIST
=
1
5000
;
constexpr
int32_t
N_LIST
=
1
6384
;
constexpr
int32_t
PARTITION_COUNT
=
5
;
constexpr
int32_t
TARGET_PARTITION
=
3
;
...
...
core/src/sdk/examples/simple/src/ClientTest.cpp
浏览文件 @
f330d9f9
...
...
@@ -20,11 +20,8 @@
#include "sdk/examples/utils/TimeRecorder.h"
#include "sdk/examples/utils/Utils.h"
#include <time.h>
#include <unistd.h>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
...
...
@@ -42,7 +39,7 @@ constexpr int64_t NPROBE = 32;
constexpr
int64_t
SEARCH_TARGET
=
5000
;
// change this value, result is different
constexpr
int64_t
ADD_VECTOR_LOOP
=
5
;
constexpr
milvus
::
IndexType
INDEX_TYPE
=
milvus
::
IndexType
::
IVFSQ8
;
constexpr
int32_t
N_LIST
=
1
5000
;
constexpr
int32_t
N_LIST
=
1
6384
;
milvus
::
TableSchema
BuildTableSchema
()
{
...
...
core/src/sdk/examples/utils/Utils.cpp
浏览文件 @
f330d9f9
...
...
@@ -22,7 +22,6 @@
#include <unistd.h>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
...
...
@@ -194,7 +193,7 @@ Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>
if
(
result_id
!=
search_id
)
{
std
::
cout
<<
"The top 1 result is wrong: "
<<
result_id
<<
" vs. "
<<
search_id
<<
std
::
endl
;
}
else
{
std
::
cout
<<
"
Check result su
cessfully"
<<
std
::
endl
;
std
::
cout
<<
"
No."
<<
i
<<
" Check result suc
cessfully"
<<
std
::
endl
;
}
}
BLOCK_SPLITER
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录