Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
52963119
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
52963119
编写于
9月 07, 2019
作者:
Y
Yu Kun
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0
Former-commit-id: 407999d3d02b41c5d6b46de8c49fcd53ecceef2b
上级
905191b2
82cd6976
变更
52
展开全部
隐藏空白更改
内联
并排
Showing
52 changed file
with
811 addition
and
839 deletion
+811
-839
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/coverage.sh
cpp/coverage.sh
+1
-0
cpp/src/config/ConfigMgr.h
cpp/src/config/ConfigMgr.h
+1
-1
cpp/src/config/YamlConfigMgr.cpp
cpp/src/config/YamlConfigMgr.cpp
+1
-1
cpp/src/config/YamlConfigMgr.h
cpp/src/config/YamlConfigMgr.h
+1
-1
cpp/src/core/CMakeLists.txt
cpp/src/core/CMakeLists.txt
+4
-0
cpp/src/core/test/CMakeLists.txt
cpp/src/core/test/CMakeLists.txt
+4
-0
cpp/src/core/test/faiss_ori/CMakeLists.txt
cpp/src/core/test/faiss_ori/CMakeLists.txt
+2
-0
cpp/src/core/test/test_nsg/CMakeLists.txt
cpp/src/core/test/test_nsg/CMakeLists.txt
+3
-1
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+21
-21
cpp/src/db/Status.cpp
cpp/src/db/Status.cpp
+43
-38
cpp/src/db/Status.h
cpp/src/db/Status.h
+9
-55
cpp/src/db/Utils.cpp
cpp/src/db/Utils.cpp
+4
-4
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+25
-25
cpp/src/db/insert/MemTable.cpp
cpp/src/db/insert/MemTable.cpp
+2
-2
cpp/src/db/insert/MemTableFile.cpp
cpp/src/db/insert/MemTableFile.cpp
+1
-1
cpp/src/db/meta/MySQLMetaImpl.cpp
cpp/src/db/meta/MySQLMetaImpl.cpp
+252
-285
cpp/src/db/meta/SqliteMetaImpl.cpp
cpp/src/db/meta/SqliteMetaImpl.cpp
+73
-60
cpp/src/db/scheduler/TaskDispatchQueue.cpp
cpp/src/db/scheduler/TaskDispatchQueue.cpp
+2
-2
cpp/src/db/scheduler/task/SearchTask.cpp
cpp/src/db/scheduler/task/SearchTask.cpp
+2
-2
cpp/src/metrics/MetricBase.h
cpp/src/metrics/MetricBase.h
+1
-1
cpp/src/metrics/PrometheusMetrics.cpp
cpp/src/metrics/PrometheusMetrics.cpp
+1
-1
cpp/src/metrics/PrometheusMetrics.h
cpp/src/metrics/PrometheusMetrics.h
+1
-2
cpp/src/scheduler/task/SearchTask.cpp
cpp/src/scheduler/task/SearchTask.cpp
+2
-2
cpp/src/server/DBWrapper.cpp
cpp/src/server/DBWrapper.cpp
+3
-3
cpp/src/server/DBWrapper.h
cpp/src/server/DBWrapper.h
+2
-2
cpp/src/server/Server.cpp
cpp/src/server/Server.cpp
+2
-2
cpp/src/server/Server.h
cpp/src/server/Server.h
+1
-1
cpp/src/server/ServerConfig.cpp
cpp/src/server/ServerConfig.cpp
+3
-3
cpp/src/server/ServerConfig.h
cpp/src/server/ServerConfig.h
+2
-2
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
+10
-10
cpp/src/server/grpc_impl/GrpcRequestScheduler.h
cpp/src/server/grpc_impl/GrpcRequestScheduler.h
+8
-8
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
+36
-36
cpp/src/server/grpc_impl/GrpcRequestTask.h
cpp/src/server/grpc_impl/GrpcRequestTask.h
+14
-14
cpp/src/utils/CommonUtil.cpp
cpp/src/utils/CommonUtil.cpp
+3
-3
cpp/src/utils/CommonUtil.h
cpp/src/utils/CommonUtil.h
+2
-2
cpp/src/utils/Error.h
cpp/src/utils/Error.h
+77
-64
cpp/src/utils/StringHelpFunctions.cpp
cpp/src/utils/StringHelpFunctions.cpp
+7
-7
cpp/src/utils/StringHelpFunctions.h
cpp/src/utils/StringHelpFunctions.h
+7
-7
cpp/src/utils/ValidationUtil.cpp
cpp/src/utils/ValidationUtil.cpp
+10
-10
cpp/src/utils/ValidationUtil.h
cpp/src/utils/ValidationUtil.h
+10
-10
cpp/src/wrapper/knowhere/vec_impl.cpp
cpp/src/wrapper/knowhere/vec_impl.cpp
+35
-35
cpp/src/wrapper/knowhere/vec_impl.h
cpp/src/wrapper/knowhere/vec_impl.h
+8
-8
cpp/src/wrapper/knowhere/vec_index.cpp
cpp/src/wrapper/knowhere/vec_index.cpp
+4
-4
cpp/src/wrapper/knowhere/vec_index.h
cpp/src/wrapper/knowhere/vec_index.h
+19
-19
cpp/unittest/db/meta_tests.cpp
cpp/unittest/db/meta_tests.cpp
+2
-1
cpp/unittest/db/misc_test.cpp
cpp/unittest/db/misc_test.cpp
+12
-7
cpp/unittest/db/mysql_meta_test.cpp
cpp/unittest/db/mysql_meta_test.cpp
+2
-1
cpp/unittest/metrics/metrics_test.cpp
cpp/unittest/metrics/metrics_test.cpp
+1
-1
cpp/unittest/server/cache_test.cpp
cpp/unittest/server/cache_test.cpp
+16
-16
cpp/unittest/server/config_test.cpp
cpp/unittest/server/config_test.cpp
+12
-12
cpp/unittest/server/util_test.cpp
cpp/unittest/server/util_test.cpp
+46
-46
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
52963119
...
...
@@ -92,6 +92,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-487 - Define metric type in CreateTable
-
MS-488 - Improve code format in scheduler
-
MS-495 - cmake: integrated knowhere
-
MS-505 - Install core unit test and add to coverage
## New Feature
-
MS-343 - Implement ResourceMgr
...
...
cpp/coverage.sh
浏览文件 @
52963119
...
...
@@ -95,6 +95,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
"*/boost/*"
\
"*/cmake_build/*_ep-prefix/*"
\
"src/core/cmake_build*"
\
"src/core/thirdparty*"
\
# gen html report
${
LCOV_GEN_CMD
}
"
${
FILE_INFO_OUTPUT_NEW
}
"
--output-directory
${
DIR_LCOV_OUTPUT
}
/
\ No newline at end of file
cpp/src/config/ConfigMgr.h
浏览文件 @
52963119
...
...
@@ -30,7 +30,7 @@ class ConfigMgr {
public:
static
ConfigMgr
*
GetInstance
();
virtual
ServerError
LoadConfigFile
(
const
std
::
string
&
filename
)
=
0
;
virtual
ErrorCode
LoadConfigFile
(
const
std
::
string
&
filename
)
=
0
;
virtual
void
Print
()
const
=
0
;
//will be deleted
virtual
std
::
string
DumpString
()
const
=
0
;
...
...
cpp/src/config/YamlConfigMgr.cpp
浏览文件 @
52963119
...
...
@@ -12,7 +12,7 @@ namespace zilliz {
namespace
milvus
{
namespace
server
{
ServerError
YamlConfigMgr
::
LoadConfigFile
(
const
std
::
string
&
filename
)
{
ErrorCode
YamlConfigMgr
::
LoadConfigFile
(
const
std
::
string
&
filename
)
{
struct
stat
directoryStat
;
int
statOK
=
stat
(
filename
.
c_str
(),
&
directoryStat
);
if
(
statOK
!=
0
)
{
...
...
cpp/src/config/YamlConfigMgr.h
浏览文件 @
52963119
...
...
@@ -17,7 +17,7 @@ namespace server {
class
YamlConfigMgr
:
public
ConfigMgr
{
public:
virtual
ServerError
LoadConfigFile
(
const
std
::
string
&
filename
);
virtual
ErrorCode
LoadConfigFile
(
const
std
::
string
&
filename
);
virtual
void
Print
()
const
;
virtual
std
::
string
DumpString
()
const
;
...
...
cpp/src/core/CMakeLists.txt
浏览文件 @
52963119
...
...
@@ -72,6 +72,10 @@ include(ThirdPartyPackagesCore)
add_subdirectory
(
src
)
if
(
BUILD_COVERAGE STREQUAL
"ON"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-fprofile-arcs -ftest-coverage"
)
endif
()
set
(
CORE_INCLUDE_DIRS
${
CORE_INCLUDE_DIRS
}
PARENT_SCOPE
)
if
(
BUILD_UNIT_TEST STREQUAL
"ON"
)
...
...
cpp/src/core/test/CMakeLists.txt
浏览文件 @
52963119
...
...
@@ -74,6 +74,10 @@ target_link_libraries(test_kdt
SPTAGLibStatic
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
install
(
TARGETS test_ivf DESTINATION unittest
)
install
(
TARGETS test_idmap DESTINATION unittest
)
install
(
TARGETS test_kdt DESTINATION unittest
)
add_subdirectory
(
faiss_ori
)
add_subdirectory
(
test_nsg
)
cpp/src/core/test/faiss_ori/CMakeLists.txt
浏览文件 @
52963119
...
...
@@ -24,3 +24,5 @@ if(NOT TARGET test_gpu)
add_executable
(
test_gpu gpuresource_test.cpp
)
endif
()
target_link_libraries
(
test_gpu
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
install
(
TARGETS test_gpu DESTINATION unittest
)
\ No newline at end of file
cpp/src/core/test/test_nsg/CMakeLists.txt
浏览文件 @
52963119
...
...
@@ -38,4 +38,6 @@ if(NOT TARGET test_nsg)
endif
()
target_link_libraries
(
test_nsg
${
depend_libs
}
${
unittest_libs
}
${
basic_libs
}
)
##############################
\ No newline at end of file
##############################
install
(
TARGETS test_nsg DESTINATION unittest
)
\ No newline at end of file
cpp/src/db/DBImpl.cpp
浏览文件 @
52963119
...
...
@@ -100,7 +100,7 @@ Status DBImpl::DropAll() {
Status
DBImpl
::
CreateTable
(
meta
::
TableSchema
&
table_schema
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
meta
::
TableSchema
temp_schema
=
table_schema
;
...
...
@@ -110,7 +110,7 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
Status
DBImpl
::
DeleteTable
(
const
std
::
string
&
table_id
,
const
meta
::
DatesT
&
dates
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
//dates partly delete files of the table but currently we don't support
...
...
@@ -136,7 +136,7 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
Status
DBImpl
::
DescribeTable
(
meta
::
TableSchema
&
table_schema
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
auto
stat
=
meta_ptr_
->
DescribeTable
(
table_schema
);
...
...
@@ -146,7 +146,7 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
Status
DBImpl
::
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
return
meta_ptr_
->
HasTable
(
table_id
,
has_or_not
);
...
...
@@ -154,7 +154,7 @@ Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
Status
DBImpl
::
AllTables
(
std
::
vector
<
meta
::
TableSchema
>&
table_schema_array
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
return
meta_ptr_
->
AllTables
(
table_schema_array
);
...
...
@@ -162,7 +162,7 @@ Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
Status
DBImpl
::
PreloadTable
(
const
std
::
string
&
table_id
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
meta
::
DatePartionedTableFilesSchema
files
;
...
...
@@ -184,7 +184,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
ExecutionEnginePtr
engine
=
EngineFactory
::
Build
(
file
.
dimension_
,
file
.
location_
,
(
EngineType
)
file
.
engine_type_
,
(
MetricType
)
file
.
metric_type_
,
file
.
nlist_
);
if
(
engine
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Invalid engine type"
;
return
Status
::
Error
(
"Invalid engine type"
);
return
Status
(
DB_ERROR
,
"Invalid engine type"
);
}
size
+=
engine
->
PhysicalSize
();
...
...
@@ -197,7 +197,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
}
catch
(
std
::
exception
&
ex
)
{
std
::
string
msg
=
"Pre-load table encounter exception: "
+
std
::
string
(
ex
.
what
());
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
}
}
...
...
@@ -207,7 +207,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
Status
DBImpl
::
UpdateTableFlag
(
const
std
::
string
&
table_id
,
int64_t
flag
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
return
meta_ptr_
->
UpdateTableFlag
(
table_id
,
flag
);
...
...
@@ -215,7 +215,7 @@ Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
Status
DBImpl
::
GetTableRowCount
(
const
std
::
string
&
table_id
,
uint64_t
&
row_count
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
return
meta_ptr_
->
Count
(
table_id
,
row_count
);
...
...
@@ -225,7 +225,7 @@ Status DBImpl::InsertVectors(const std::string& table_id_,
uint64_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids_
)
{
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
Status
status
;
...
...
@@ -314,7 +314,7 @@ Status DBImpl::DropIndex(const std::string& table_id) {
Status
DBImpl
::
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
QueryResults
&
results
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
meta
::
DatesT
dates
=
{
utils
::
GetDate
()};
...
...
@@ -326,7 +326,7 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6
Status
DBImpl
::
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
ENGINE_LOG_DEBUG
<<
"Query by dates for table: "
<<
table_id
;
...
...
@@ -354,7 +354,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
ENGINE_LOG_DEBUG
<<
"Query by file ids for table: "
<<
table_id
;
...
...
@@ -382,7 +382,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
}
if
(
file_id_array
.
empty
())
{
return
Status
::
Error
(
"Invalid file id"
);
return
Status
(
DB_ERROR
,
"Invalid file id"
);
}
cache
::
CpuCacheMgr
::
GetInstance
()
->
PrintInfo
();
//print cache info before query
...
...
@@ -393,7 +393,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
Status
DBImpl
::
Size
(
uint64_t
&
result
)
{
if
(
shutting_down_
.
load
(
std
::
memory_order_acquire
)){
return
Status
::
Error
(
"Milsvus server is shutdown!"
);
return
Status
(
DB_ERROR
,
"Milsvus server is shutdown!"
);
}
return
meta_ptr_
->
Size
(
result
);
...
...
@@ -600,7 +600,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
std
::
cout
<<
"ERROR: failed to persist merged index file: "
<<
table_file
.
location_
<<
", possible out of disk space"
<<
std
::
endl
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
//step 4: update table files state
...
...
@@ -708,7 +708,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
(
MetricType
)
file
.
metric_type_
,
file
.
nlist_
);
if
(
to_index
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Invalid engine type"
;
return
Status
::
Error
(
"Invalid engine type"
);
return
Status
(
DB_ERROR
,
"Invalid engine type"
);
}
try
{
...
...
@@ -755,7 +755,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
std
::
cout
<<
"ERROR: failed to build index, index file is too large or gpu memory is not enough"
<<
std
::
endl
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
//step 4: if table has been deleted, dont save index file
...
...
@@ -781,7 +781,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
std
::
cout
<<
"ERROR: failed to persist index file: "
<<
table_file
.
location_
<<
", possible out of disk space"
<<
std
::
endl
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
//step 6: update meta
...
...
@@ -816,7 +816,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
}
catch
(
std
::
exception
&
ex
)
{
std
::
string
msg
=
"Build index encounter exception: "
+
std
::
string
(
ex
.
what
());
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
return
Status
::
OK
();
...
...
cpp/src/db/Status.cpp
浏览文件 @
52963119
...
...
@@ -12,57 +12,62 @@ namespace zilliz {
namespace
milvus
{
namespace
engine
{
const
char
*
Status
::
CopyState
(
const
char
*
state
)
{
uint32_t
size
;
std
::
memcpy
(
&
size
,
state
,
sizeof
(
size
));
char
*
result
=
new
char
[
size
+
5
];
memcpy
(
result
,
state
,
size
+
5
);
return
result
;
}
constexpr
int
CODE_WIDTH
=
sizeof
(
ErrorCode
);
Status
::
Status
(
ErrorCode
code
,
const
std
::
string
&
msg
)
{
//4 bytes store code
//4 bytes store message length
//the left bytes store message string
const
uint32_t
length
=
(
uint32_t
)
msg
.
size
();
char
*
result
=
new
char
[
length
+
sizeof
(
length
)
+
CODE_WIDTH
];
std
::
memcpy
(
result
,
&
code
,
CODE_WIDTH
);
std
::
memcpy
(
result
+
CODE_WIDTH
,
&
length
,
sizeof
(
length
));
memcpy
(
result
+
sizeof
(
length
)
+
CODE_WIDTH
,
msg
.
data
(),
length
);
Status
::
Status
(
Code
code
,
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
)
{
assert
(
code
!=
kOK
);
const
uint32_t
len1
=
msg
.
size
();
const
uint32_t
len2
=
msg2
.
size
();
const
uint32_t
size
=
len1
+
(
len2
?
(
2
+
len2
)
:
0
);
char
*
result
=
new
char
[
size
+
5
];
std
::
memcpy
(
result
,
&
size
,
sizeof
(
size
));
result
[
4
]
=
static_cast
<
char
>
(
code
);
memcpy
(
result
+
5
,
msg
.
data
(),
len1
);
if
(
len2
)
{
result
[
5
+
len1
]
=
':'
;
result
[
6
+
len1
]
=
' '
;
memcpy
(
result
+
7
+
len1
,
msg2
.
data
(),
len2
);
}
state_
=
result
;
}
Status
::
Status
()
:
state_
(
nullptr
)
{
}
Status
::~
Status
()
{
delete
[]
state_
;
}
const
char
*
Status
::
CopyState
(
const
char
*
state
)
{
uint32_t
length
=
0
;
std
::
memcpy
(
&
length
,
state
+
CODE_WIDTH
,
sizeof
(
length
));
int
buff_len
=
length
+
sizeof
(
length
)
+
CODE_WIDTH
;
char
*
result
=
new
char
[
buff_len
];
memcpy
(
result
,
state
,
buff_len
);
return
result
;
}
std
::
string
Status
::
ToString
()
const
{
if
(
state_
==
nullptr
)
return
"OK"
;
char
tmp
[
3
0
];
char
tmp
[
3
2
];
const
char
*
type
;
switch
(
code
())
{
case
kOK
:
case
DB_SUCCESS
:
type
=
"OK"
;
break
;
case
kNotFound
:
type
=
"NotFound: "
;
break
;
case
kError
:
case
DB_ERROR
:
type
=
"Error: "
;
break
;
case
kInvalidDBPath
:
type
=
"InvalidDBPath: "
;
break
;
case
kGroupError
:
type
=
"GroupError: "
;
break
;
case
kDBTransactionError
:
case
DB_META_TRANSACTION_FAILED
:
type
=
"DBTransactionError: "
;
break
;
case
kAlreadyExist
:
case
DB_NOT_FOUND
:
type
=
"NotFound: "
;
break
;
case
DB_ALREADY_EXIST
:
type
=
"AlreadyExist: "
;
break
;
case
DB_INVALID_PATH
:
type
=
"InvalidPath: "
;
break
;
default:
snprintf
(
tmp
,
sizeof
(
tmp
),
"Unkown code(%d): "
,
static_cast
<
int
>
(
code
()));
...
...
@@ -71,9 +76,9 @@ std::string Status::ToString() const {
}
std
::
string
result
(
type
);
uint32_t
length
;
memcpy
(
&
length
,
state_
,
sizeof
(
length
));
result
.
append
(
state_
+
5
,
length
);
uint32_t
length
=
0
;
memcpy
(
&
length
,
state_
+
CODE_WIDTH
,
sizeof
(
length
));
result
.
append
(
state_
+
sizeof
(
length
)
+
CODE_WIDTH
,
length
);
return
result
;
}
...
...
cpp/src/db/Status.h
浏览文件 @
52963119
...
...
@@ -5,8 +5,9 @@
******************************************************************************/
#pragma once
#include
<string>
#include
"utils/Error.h"
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -14,9 +15,9 @@ namespace engine {
class
Status
{
public:
Status
(
)
noexcept
:
state_
(
nullptr
)
{}
~
Status
()
{
delete
[]
state_
;
}
Status
(
ErrorCode
code
,
const
std
::
string
&
msg
);
Status
();
~
Status
()
;
Status
(
const
Status
&
rhs
);
...
...
@@ -31,64 +32,17 @@ class Status {
static
Status
OK
()
{
return
Status
();
}
static
Status
NotFound
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kNotFound
,
msg
,
msg2
);
}
static
Status
Error
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kError
,
msg
,
msg2
);
}
bool
ok
()
const
{
return
state_
==
nullptr
||
code
()
==
DB_SUCCESS
;
}
static
Status
InvalidDBPath
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kInvalidDBPath
,
msg
,
msg2
);
}
static
Status
GroupError
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kGroupError
,
msg
,
msg2
);
}
static
Status
DBTransactionError
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kDBTransactionError
,
msg
,
msg2
);
}
std
::
string
ToString
()
const
;
static
Status
AlreadyExist
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kAlreadyExist
,
msg
,
msg2
);
ErrorCode
code
()
const
{
return
(
state_
==
nullptr
)
?
DB_SUCCESS
:
*
(
ErrorCode
*
)(
state_
);
}
bool
ok
()
const
{
return
state_
==
nullptr
;
}
bool
IsNotFound
()
const
{
return
code
()
==
kNotFound
;
}
bool
IsError
()
const
{
return
code
()
==
kError
;
}
bool
IsInvalidDBPath
()
const
{
return
code
()
==
kInvalidDBPath
;
}
bool
IsGroupError
()
const
{
return
code
()
==
kGroupError
;
}
bool
IsDBTransactionError
()
const
{
return
code
()
==
kDBTransactionError
;
}
bool
IsAlreadyExist
()
const
{
return
code
()
==
kAlreadyExist
;
}
std
::
string
ToString
()
const
;
private:
const
char
*
state_
=
nullptr
;
enum
Code
{
kOK
=
0
,
kNotFound
,
kError
,
kInvalidDBPath
,
kGroupError
,
kDBTransactionError
,
kAlreadyExist
,
};
Code
code
()
const
{
return
(
state_
==
nullptr
)
?
kOK
:
static_cast
<
Code
>
(
state_
[
4
]);
}
Status
(
Code
code
,
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
);
static
const
char
*
CopyState
(
const
char
*
s
);
};
// Status
...
...
cpp/src/db/Utils.cpp
浏览文件 @
52963119
...
...
@@ -70,7 +70,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
auto
status
=
server
::
CommonUtil
::
CreateDirectory
(
table_path
);
if
(
status
!=
0
)
{
ENGINE_LOG_ERROR
<<
"Create directory "
<<
table_path
<<
" Error"
;
return
Status
::
Error
(
"Failed to create table path"
);
return
Status
(
DB_ERROR
,
"Failed to create table path"
);
}
for
(
auto
&
path
:
options
.
slave_paths
)
{
...
...
@@ -78,7 +78,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
status
=
server
::
CommonUtil
::
CreateDirectory
(
table_path
);
if
(
status
!=
0
)
{
ENGINE_LOG_ERROR
<<
"Create directory "
<<
table_path
<<
" Error"
;
return
Status
::
Error
(
"Failed to create table path"
);
return
Status
(
DB_ERROR
,
"Failed to create table path"
);
}
}
...
...
@@ -110,7 +110,7 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
auto
status
=
server
::
CommonUtil
::
CreateDirectory
(
parent_path
);
if
(
status
!=
0
)
{
ENGINE_LOG_ERROR
<<
"Create directory "
<<
parent_path
<<
" Error"
;
return
Status
::
DBTransactionError
(
"Failed to create partition directory"
);
return
Status
(
DB_ERROR
,
"Failed to create partition directory"
);
}
table_file
.
location_
=
parent_path
+
"/"
+
table_file
.
file_id_
;
...
...
@@ -137,7 +137,7 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
std
::
string
msg
=
"Table file doesn't exist: "
+
table_file
.
file_id_
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
Status
DeleteTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
...
...
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
52963119
...
...
@@ -41,7 +41,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
build_cfg
[
"metric_type"
]
=
(
metric_type_
==
MetricType
::
IP
)
?
"IP"
:
"L2"
;
AutoGenParams
(
index_
->
GetType
(),
0
,
build_cfg
);
auto
ec
=
std
::
static_pointer_cast
<
BFIndex
>
(
index_
)
->
Build
(
build_cfg
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
throw
Exception
(
"Build index error"
);
}
if
(
ec
!=
KNOWHERE_SUCCESS
)
{
throw
Exception
(
"Build index error"
);
}
}
ExecutionEngineImpl
::
ExecutionEngineImpl
(
VecIndexPtr
index
,
...
...
@@ -85,8 +85,8 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
Status
ExecutionEngineImpl
::
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
{
auto
ec
=
index_
->
Add
(
n
,
xdata
,
xids
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
return
Status
::
Error
(
"Add error"
);
if
(
ec
!=
KNOWHERE_SUCCESS
)
{
return
Status
(
DB_ERROR
,
"Add error"
);
}
return
Status
::
OK
();
}
...
...
@@ -117,8 +117,8 @@ size_t ExecutionEngineImpl::PhysicalSize() const {
Status
ExecutionEngineImpl
::
Serialize
()
{
auto
ec
=
write_index
(
index_
,
location_
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
return
Status
::
Error
(
"Serialize: write to disk error"
);
if
(
ec
!=
KNOWHERE_SUCCESS
)
{
return
Status
(
DB_ERROR
,
"Serialize: write to disk error"
);
}
return
Status
::
OK
();
}
...
...
@@ -134,15 +134,15 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
if
(
index_
==
nullptr
)
{
std
::
string
msg
=
"Failed to load index from "
+
location_
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
else
{
ENGINE_LOG_DEBUG
<<
"Disk io from: "
<<
location_
;
}
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
ENGINE_LOG_ERROR
<<
e
.
what
();
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
catch
(
std
::
exception
&
e
)
{
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
}
...
...
@@ -160,7 +160,7 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
}
else
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to copy to gpu"
;
return
Status
::
Error
(
"index is null"
);
return
Status
(
DB_ERROR
,
"index is null"
);
}
try
{
...
...
@@ -168,9 +168,9 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
ENGINE_LOG_DEBUG
<<
"CPU to GPU"
<<
device_id
;
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
ENGINE_LOG_ERROR
<<
e
.
what
();
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
catch
(
std
::
exception
&
e
)
{
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
}
...
...
@@ -189,7 +189,7 @@ Status ExecutionEngineImpl::CopyToCpu() {
}
else
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to copy to cpu"
;
return
Status
::
Error
(
"index is null"
);
return
Status
(
DB_ERROR
,
"index is null"
);
}
try
{
...
...
@@ -197,9 +197,9 @@ Status ExecutionEngineImpl::CopyToCpu() {
ENGINE_LOG_DEBUG
<<
"GPU to CPU"
;
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
ENGINE_LOG_ERROR
<<
e
.
what
();
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
catch
(
std
::
exception
&
e
)
{
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
}
...
...
@@ -223,7 +223,7 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() {
Status
ExecutionEngineImpl
::
Merge
(
const
std
::
string
&
location
)
{
if
(
location
==
location_
)
{
return
Status
::
Error
(
"Cannot Merge Self"
);
return
Status
(
DB_ERROR
,
"Cannot Merge Self"
);
}
ENGINE_LOG_DEBUG
<<
"Merge index file: "
<<
location
<<
" to: "
<<
location_
;
...
...
@@ -235,26 +235,26 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
to_merge
=
read_index
(
location
);
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
ENGINE_LOG_ERROR
<<
e
.
what
();
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
catch
(
std
::
exception
&
e
)
{
return
Status
::
Error
(
e
.
what
());
return
Status
(
DB_ERROR
,
e
.
what
());
}
}
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to merge"
;
return
Status
::
Error
(
"index is null"
);
return
Status
(
DB_ERROR
,
"index is null"
);
}
if
(
auto
file_index
=
std
::
dynamic_pointer_cast
<
BFIndex
>
(
to_merge
))
{
auto
ec
=
index_
->
Add
(
file_index
->
Count
(),
file_index
->
GetRawVectors
(),
file_index
->
GetRawIds
());
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
if
(
ec
!=
KNOWHERE_SUCCESS
)
{
ENGINE_LOG_ERROR
<<
"Merge: Add Error"
;
return
Status
::
Error
(
"Merge: Add Error"
);
return
Status
(
DB_ERROR
,
"Merge: Add Error"
);
}
return
Status
::
OK
();
}
else
{
return
Status
::
Error
(
"file index type is not idmap"
);
return
Status
(
DB_ERROR
,
"file index type is not idmap"
);
}
}
...
...
@@ -284,7 +284,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
from_index
->
GetRawVectors
(),
from_index
->
GetRawIds
(),
build_cfg
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
throw
Exception
(
"Build index error"
);
}
if
(
ec
!=
KNOWHERE_SUCCESS
)
{
throw
Exception
(
"Build index error"
);
}
return
std
::
make_shared
<
ExecutionEngineImpl
>
(
to_index
,
location
,
engine_type
,
metric_type_
,
nlist_
);
}
...
...
@@ -297,15 +297,15 @@ Status ExecutionEngineImpl::Search(long n,
long
*
labels
)
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to search"
;
return
Status
::
Error
(
"index is null"
);
return
Status
(
DB_ERROR
,
"index is null"
);
}
ENGINE_LOG_DEBUG
<<
"Search Params: [k] "
<<
k
<<
" [nprobe] "
<<
nprobe
;
auto
cfg
=
Config
::
object
{{
"k"
,
k
},
{
"nprobe"
,
nprobe
}};
auto
ec
=
index_
->
Search
(
n
,
data
,
distances
,
labels
,
cfg
);
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
if
(
ec
!=
KNOWHERE_SUCCESS
)
{
ENGINE_LOG_ERROR
<<
"Search error"
;
return
Status
::
Error
(
"Search: Search Error"
);
return
Status
(
DB_ERROR
,
"Search: Search Error"
);
}
return
Status
::
OK
();
}
...
...
cpp/src/db/insert/MemTable.cpp
浏览文件 @
52963119
...
...
@@ -38,7 +38,7 @@ Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) {
if
(
!
status
.
ok
())
{
std
::
string
err_msg
=
"MemTable::Add failed: "
+
status
.
ToString
();
ENGINE_LOG_ERROR
<<
err_msg
;
return
Status
::
Error
(
err_msg
);
return
Status
(
DB_ERROR
,
err_msg
);
}
}
return
Status
::
OK
();
...
...
@@ -58,7 +58,7 @@ Status MemTable::Serialize() {
if
(
!
status
.
ok
())
{
std
::
string
err_msg
=
"MemTable::Serialize failed: "
+
status
.
ToString
();
ENGINE_LOG_ERROR
<<
err_msg
;
return
Status
::
Error
(
err_msg
);
return
Status
(
DB_ERROR
,
err_msg
);
}
std
::
lock_guard
<
std
::
mutex
>
lock
(
mutex_
);
mem_table_file
=
mem_table_file_list_
.
erase
(
mem_table_file
);
...
...
cpp/src/db/insert/MemTableFile.cpp
浏览文件 @
52963119
...
...
@@ -49,7 +49,7 @@ Status MemTableFile::Add(const VectorSource::Ptr &source, IDNumbers& vector_ids)
std
::
string
err_msg
=
"MemTableFile::Add: table_file_schema dimension = "
+
std
::
to_string
(
table_file_schema_
.
dimension_
)
+
", table_id = "
+
table_file_schema_
.
table_id_
;
ENGINE_LOG_ERROR
<<
err_msg
;
return
Status
::
Error
(
err_msg
);
return
Status
(
DB_ERROR
,
err_msg
);
}
size_t
single_vector_mem_size
=
table_file_schema_
.
dimension_
*
VECTOR_TYPE_SIZE
;
...
...
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
52963119
此差异已折叠。
点击以展开。
cpp/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
52963119
...
...
@@ -29,9 +29,15 @@ using namespace sqlite_orm;
namespace
{
Status
HandleException
(
const
std
::
string
&
desc
,
std
::
exception
&
e
)
{
ENGINE_LOG_ERROR
<<
desc
<<
": "
<<
e
.
what
();
return
Status
::
DBTransactionError
(
desc
,
e
.
what
());
Status
HandleException
(
const
std
::
string
&
desc
,
const
char
*
what
=
nullptr
)
{
if
(
what
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
desc
;
return
Status
(
DB_META_TRANSACTION_FAILED
,
desc
);
}
else
{
std
::
string
msg
=
desc
+
":"
+
what
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
(
DB_META_TRANSACTION_FAILED
,
msg
);
}
}
}
...
...
@@ -97,8 +103,9 @@ Status SqliteMetaImpl::Initialize() {
if
(
!
boost
::
filesystem
::
is_directory
(
options_
.
path
))
{
auto
ret
=
boost
::
filesystem
::
create_directory
(
options_
.
path
);
if
(
!
ret
)
{
ENGINE_LOG_ERROR
<<
"Failed to create db directory "
<<
options_
.
path
;
return
Status
::
InvalidDBPath
(
"Failed to create db directory"
,
options_
.
path
);
std
::
string
msg
=
"Failed to create db directory "
+
options_
.
path
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
(
DB_INVALID_PATH
,
msg
);
}
}
...
...
@@ -141,7 +148,7 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id,
in
(
&
TableFileSchema
::
date_
,
dates
)
));
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when drop partition"
,
e
);
return
HandleException
(
"Encounter exception when drop partition"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -162,10 +169,10 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) {
where
(
c
(
&
TableSchema
::
table_id_
)
==
table_schema
.
table_id_
));
if
(
table
.
size
()
==
1
)
{
if
(
TableSchema
::
TO_DELETE
==
std
::
get
<
0
>
(
table
[
0
]))
{
return
Status
::
Error
(
"Table already exists and it is in delete state, please wait a second"
);
return
Status
(
DB_ERROR
,
"Table already exists and it is in delete state, please wait a second"
);
}
else
{
// Change from no error to already exist.
return
Status
::
AlreadyExist
(
"Table already exists"
);
return
Status
(
DB_ALREADY_EXIST
,
"Table already exists"
);
}
}
}
...
...
@@ -176,15 +183,14 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) {
try
{
auto
id
=
ConnectorPtr
->
insert
(
table_schema
);
table_schema
.
id_
=
id
;
}
catch
(...)
{
ENGINE_LOG_ERROR
<<
"sqlite transaction failed"
;
return
Status
::
DBTransactionError
(
"Add Table Error"
);
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when create table"
,
e
.
what
());
}
return
utils
::
CreateTablePath
(
options_
,
table_schema
.
table_id_
);
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when create table"
,
e
);
return
HandleException
(
"Encounter exception when create table"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -208,7 +214,7 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) {
));
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table"
,
e
);
return
HandleException
(
"Encounter exception when delete table"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -233,7 +239,7 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) {
));
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table files"
,
e
);
return
HandleException
(
"Encounter exception when delete table files"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -266,11 +272,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
table_schema
.
nlist_
=
std
::
get
<
7
>
(
groups
[
0
]);
table_schema
.
metric_type_
=
std
::
get
<
8
>
(
groups
[
0
]);
}
else
{
return
Status
::
NotFound
(
"Table "
+
table_schema
.
table_id_
+
" not found"
);
return
Status
(
DB_NOT_FOUND
,
"Table "
+
table_schema
.
table_id_
+
" not found"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when describe table"
,
e
);
return
HandleException
(
"Encounter exception when describe table"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -280,7 +286,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id,
const
std
::
vector
<
int
>&
file_types
,
std
::
vector
<
std
::
string
>&
file_ids
)
{
if
(
file_types
.
empty
())
{
return
Status
::
Error
(
"file types array is empty"
);
return
Status
(
DB_ERROR
,
"file types array is empty"
);
}
try
{
...
...
@@ -330,7 +336,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id,
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when check non index files"
,
e
);
return
HandleException
(
"Encounter exception when check non index files"
,
e
.
what
()
);
}
return
Status
::
OK
();
}
...
...
@@ -366,7 +372,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
ConnectorPtr
->
update
(
table_schema
);
}
else
{
return
Status
::
NotFound
(
"Table "
+
table_id
+
" not found"
);
return
Status
(
DB_NOT_FOUND
,
"Table "
+
table_id
+
" not found"
);
}
//set all backup file to raw
...
...
@@ -382,7 +388,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
}
catch
(
std
::
exception
&
e
)
{
std
::
string
msg
=
"Encounter exception when update table index: table_id = "
+
table_id
;
return
HandleException
(
msg
,
e
);
return
HandleException
(
msg
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -403,7 +409,7 @@ Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag
}
catch
(
std
::
exception
&
e
)
{
std
::
string
msg
=
"Encounter exception when update table flag: table_id = "
+
table_id
;
return
HandleException
(
msg
,
e
);
return
HandleException
(
msg
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -424,11 +430,11 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde
index
.
nlist_
=
std
::
get
<
1
>
(
groups
[
0
]);
index
.
metric_type_
=
std
::
get
<
2
>
(
groups
[
0
]);
}
else
{
return
Status
::
NotFound
(
"Table "
+
table_id
+
" not found"
);
return
Status
(
DB_NOT_FOUND
,
"Table "
+
table_id
+
" not found"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when describe index"
,
e
);
return
HandleException
(
"Encounter exception when describe index"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -475,7 +481,7 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) {
));
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table index files"
,
e
);
return
HandleException
(
"Encounter exception when delete table index files"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -496,7 +502,7 @@ Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when lookup table"
,
e
);
return
HandleException
(
"Encounter exception when lookup table"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -532,7 +538,7 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when lookup all tables"
,
e
);
return
HandleException
(
"Encounter exception when lookup all tables"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -571,8 +577,8 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
return
utils
::
CreateTableFilePath
(
options_
,
file_schema
);
}
catch
(
std
::
exception
&
e
x
)
{
return
HandleException
(
"Encounter exception when create table file"
,
e
x
);
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when create table file"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -629,7 +635,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when iterate raw files"
,
e
);
return
HandleException
(
"Encounter exception when iterate raw files"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -654,8 +660,12 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
auto
match_tableid
=
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
;
std
::
vector
<
int
>
file_type
=
{(
int
)
TableFileSchema
::
RAW
,
(
int
)
TableFileSchema
::
TO_INDEX
,
(
int
)
TableFileSchema
::
INDEX
};
auto
match_type
=
in
(
&
TableFileSchema
::
file_type_
,
file_type
);
std
::
vector
<
int
>
file_types
=
{
(
int
)
TableFileSchema
::
RAW
,
(
int
)
TableFileSchema
::
TO_INDEX
,
(
int
)
TableFileSchema
::
INDEX
};
auto
match_type
=
in
(
&
TableFileSchema
::
file_type_
,
file_types
);
TableSchema
table_schema
;
table_schema
.
table_id_
=
table_id
;
...
...
@@ -710,7 +720,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
ENGINE_LOG_ERROR
<<
"No file to search for table: "
<<
table_id
;
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when iterate index files"
,
e
);
return
HandleException
(
"Encounter exception when iterate index files"
,
e
.
what
()
);
}
...
...
@@ -773,7 +783,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
files
[
table_file
.
date_
].
push_back
(
table_file
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when iterate merge files"
,
e
);
return
HandleException
(
"Encounter exception when iterate merge files"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -824,7 +834,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
table_files
.
emplace_back
(
file_schema
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when lookup table files"
,
e
);
return
HandleException
(
"Encounter exception when lookup table files"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -856,7 +866,7 @@ Status SqliteMetaImpl::Archive() {
c
(
&
TableFileSchema
::
file_type_
)
!=
(
int
)
TableFileSchema
::
TO_DELETE
));
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when update table files"
,
e
);
return
HandleException
(
"Encounter exception when update table files"
,
e
.
what
()
);
}
}
if
(
criteria
==
engine
::
ARCHIVE_CONF_DISK
)
{
...
...
@@ -886,7 +896,7 @@ Status SqliteMetaImpl::Size(uint64_t &result) {
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when calculte db size"
,
e
);
return
HandleException
(
"Encounter exception when calculte db size"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -943,12 +953,11 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) {
});
if
(
!
commited
)
{
ENGINE_LOG_ERROR
<<
"sqlite transaction failed"
;
return
Status
::
DBTransactionError
(
"Update table file error"
);
return
HandleException
(
"DiscardFiles error: sqlite transaction failed"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when discard table file"
,
e
);
return
HandleException
(
"Encounter exception when discard table file"
,
e
.
what
()
);
}
return
DiscardFiles
(
to_discard_size
);
...
...
@@ -976,7 +985,7 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
}
catch
(
std
::
exception
&
e
)
{
std
::
string
msg
=
"Exception update table file: table_id = "
+
file_schema
.
table_id_
+
" file_id = "
+
file_schema
.
file_id_
;
return
HandleException
(
msg
,
e
);
return
HandleException
(
msg
,
e
.
what
()
);
}
return
Status
::
OK
();
}
...
...
@@ -997,7 +1006,7 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) {
c
(
&
TableFileSchema
::
file_type_
)
==
(
int
)
TableFileSchema
::
RAW
));
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when update table files to to_index"
,
e
);
return
HandleException
(
"Encounter exception when update table files to to_index"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -1038,12 +1047,11 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
});
if
(
!
commited
)
{
ENGINE_LOG_ERROR
<<
"sqlite transaction failed"
;
return
Status
::
DBTransactionError
(
"Update table files error"
);
return
HandleException
(
"UpdateTableFiles error: sqlite transaction failed"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when update table files"
,
e
);
return
HandleException
(
"Encounter exception when update table files"
,
e
.
what
()
);
}
return
Status
::
OK
();
}
...
...
@@ -1088,12 +1096,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
});
if
(
!
commited
)
{
ENGINE_LOG_ERROR
<<
"sqlite transaction failed"
;
return
Status
::
DBTransactionError
(
"Clean files error"
);
return
HandleException
(
"CleanUpFilesWithTTL error: sqlite transaction failed"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when clean table files"
,
e
);
return
HandleException
(
"Encounter exception when clean table files"
,
e
.
what
()
);
}
//remove to_delete tables
...
...
@@ -1117,12 +1124,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
});
if
(
!
commited
)
{
ENGINE_LOG_ERROR
<<
"sqlite transaction failed"
;
return
Status
::
DBTransactionError
(
"Clean files error"
);
return
HandleException
(
"CleanUpFilesWithTTL error: sqlite transaction failed"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when clean table files"
,
e
);
return
HandleException
(
"Encounter exception when clean table files"
,
e
.
what
()
);
}
//remove deleted table folder
...
...
@@ -1139,7 +1145,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table folder"
,
e
);
return
HandleException
(
"Encounter exception when delete table folder"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -1152,8 +1158,12 @@ Status SqliteMetaImpl::CleanUp() {
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std
::
lock_guard
<
std
::
mutex
>
meta_lock
(
meta_mutex_
);
std
::
vector
<
int
>
file_type
=
{(
int
)
TableFileSchema
::
NEW
,
(
int
)
TableFileSchema
::
NEW_INDEX
,
(
int
)
TableFileSchema
::
NEW_MERGE
};
auto
files
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
id_
),
where
(
in
(
&
TableFileSchema
::
file_type_
,
file_type
)));
std
::
vector
<
int
>
file_types
=
{
(
int
)
TableFileSchema
::
NEW
,
(
int
)
TableFileSchema
::
NEW_INDEX
,
(
int
)
TableFileSchema
::
NEW_MERGE
};
auto
files
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
id_
),
where
(
in
(
&
TableFileSchema
::
file_type_
,
file_types
)));
auto
commited
=
ConnectorPtr
->
transaction
([
&
]()
mutable
{
for
(
auto
&
file
:
files
)
{
...
...
@@ -1164,12 +1174,11 @@ Status SqliteMetaImpl::CleanUp() {
});
if
(
!
commited
)
{
ENGINE_LOG_ERROR
<<
"sqlite transaction failed"
;
return
Status
::
DBTransactionError
(
"Clean files error"
);
return
HandleException
(
"CleanUp error: sqlite transaction failed"
);
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when clean table file"
,
e
);
return
HandleException
(
"Encounter exception when clean table file"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
@@ -1180,9 +1189,13 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) {
try
{
server
::
MetricCollector
metric
;
std
::
vector
<
int
>
file_type
=
{(
int
)
TableFileSchema
::
RAW
,
(
int
)
TableFileSchema
::
TO_INDEX
,
(
int
)
TableFileSchema
::
INDEX
};
std
::
vector
<
int
>
file_types
=
{
(
int
)
TableFileSchema
::
RAW
,
(
int
)
TableFileSchema
::
TO_INDEX
,
(
int
)
TableFileSchema
::
INDEX
};
auto
selected
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
row_count_
),
where
(
in
(
&
TableFileSchema
::
file_type_
,
file_type
)
where
(
in
(
&
TableFileSchema
::
file_type_
,
file_type
s
)
and
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
));
TableSchema
table_schema
;
...
...
@@ -1199,7 +1212,7 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) {
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when calculate table file size"
,
e
);
return
HandleException
(
"Encounter exception when calculate table file size"
,
e
.
what
()
);
}
return
Status
::
OK
();
}
...
...
@@ -1211,7 +1224,7 @@ Status SqliteMetaImpl::DropAll() {
ConnectorPtr
->
drop_table
(
"Tables"
);
ConnectorPtr
->
drop_table
(
"TableFiles"
);
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when drop all meta"
,
e
);
return
HandleException
(
"Encounter exception when drop all meta"
,
e
.
what
()
);
}
return
Status
::
OK
();
...
...
cpp/src/db/scheduler/TaskDispatchQueue.cpp
浏览文件 @
52963119
...
...
@@ -53,7 +53,7 @@ TaskDispatchQueue::Front() {
if
(
queue_
.
empty
())
{
std
::
string
error_msg
=
"blocking queue empty"
;
SERVER_LOG_ERROR
<<
error_msg
;
throw
server
::
ServerException
(
server
::
SERVER_BLOCKING_QUEUE_EMPTY
,
error_msg
);
throw
server
::
ServerException
(
SERVER_BLOCKING_QUEUE_EMPTY
,
error_msg
);
}
ScheduleTaskPtr
front
(
queue_
.
front
());
return
front
;
...
...
@@ -67,7 +67,7 @@ TaskDispatchQueue::Back() {
if
(
queue_
.
empty
())
{
std
::
string
error_msg
=
"blocking queue empty"
;
SERVER_LOG_ERROR
<<
error_msg
;
throw
server
::
ServerException
(
server
::
SERVER_BLOCKING_QUEUE_EMPTY
,
error_msg
);
throw
server
::
ServerException
(
SERVER_BLOCKING_QUEUE_EMPTY
,
error_msg
);
}
ScheduleTaskPtr
back
(
queue_
.
back
());
...
...
cpp/src/db/scheduler/task/SearchTask.cpp
浏览文件 @
52963119
...
...
@@ -134,7 +134,7 @@ Status SearchTask::ClusterResult(const std::vector<long> &output_ids,
std
::
string
msg
=
"Invalid id array size: "
+
std
::
to_string
(
output_ids
.
size
())
+
" distance array size: "
+
std
::
to_string
(
output_distence
.
size
());
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
result_set
.
clear
();
...
...
@@ -249,7 +249,7 @@ Status SearchTask::TopkResult(SearchContext::ResultSet &result_src,
if
(
result_src
.
size
()
!=
result_target
.
size
())
{
std
::
string
msg
=
"Invalid result set size"
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
std
::
function
<
void
(
size_t
,
size_t
)
>
ReduceWorker
=
[
&
](
size_t
from_index
,
size_t
to_index
)
{
...
...
cpp/src/metrics/MetricBase.h
浏览文件 @
52963119
...
...
@@ -21,7 +21,7 @@ class MetricsBase{
return
instance
;
}
virtual
ServerError
Init
()
{};
virtual
ErrorCode
Init
()
{};
virtual
void
AddVectorsSuccessTotalIncrement
(
double
value
=
1
)
{};
virtual
void
AddVectorsFailTotalIncrement
(
double
value
=
1
)
{};
...
...
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @
52963119
...
...
@@ -14,7 +14,7 @@ namespace zilliz {
namespace
milvus
{
namespace
server
{
ServerError
ErrorCode
PrometheusMetrics
::
Init
()
{
try
{
ConfigNode
&
configNode
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_METRIC
);
...
...
cpp/src/metrics/PrometheusMetrics.h
浏览文件 @
52963119
...
...
@@ -39,8 +39,7 @@ class PrometheusMetrics: public MetricsBase {
return
instance
;
}
ServerError
Init
();
ErrorCode
Init
();
private:
std
::
shared_ptr
<
prometheus
::
Exposer
>
exposer_ptr_
;
...
...
cpp/src/scheduler/task/SearchTask.cpp
浏览文件 @
52963119
...
...
@@ -202,7 +202,7 @@ Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
std
::
string
msg
=
"Invalid id array size: "
+
std
::
to_string
(
output_ids
.
size
())
+
" distance array size: "
+
std
::
to_string
(
output_distence
.
size
());
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
result_set
.
clear
();
...
...
@@ -317,7 +317,7 @@ Status XSearchTask::TopkResult(SearchContext::ResultSet &result_src,
if
(
result_src
.
size
()
!=
result_target
.
size
())
{
std
::
string
msg
=
"Invalid result set size"
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
return
Status
(
DB_ERROR
,
msg
);
}
std
::
function
<
void
(
size_t
,
size_t
)
>
ReduceWorker
=
[
&
](
size_t
from_index
,
size_t
to_index
)
{
...
...
cpp/src/server/DBWrapper.cpp
浏览文件 @
52963119
...
...
@@ -21,7 +21,7 @@ DBWrapper::DBWrapper() {
}
ServerError
DBWrapper
::
StartService
()
{
ErrorCode
DBWrapper
::
StartService
()
{
//db config
zilliz
::
milvus
::
engine
::
Options
opt
;
ConfigNode
&
db_config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_DB
);
...
...
@@ -79,7 +79,7 @@ ServerError DBWrapper::StartService() {
opt
.
meta
.
archive_conf
.
SetCriterias
(
criterial
);
//create db root folder
ServerError
err
=
CommonUtil
::
CreateDirectory
(
opt
.
meta
.
path
);
ErrorCode
err
=
CommonUtil
::
CreateDirectory
(
opt
.
meta
.
path
);
if
(
err
!=
SERVER_SUCCESS
)
{
std
::
cout
<<
"ERROR! Failed to create database root path: "
<<
opt
.
meta
.
path
<<
std
::
endl
;
kill
(
0
,
SIGUSR1
);
...
...
@@ -112,7 +112,7 @@ ServerError DBWrapper::StartService() {
return
SERVER_SUCCESS
;
}
ServerError
DBWrapper
::
StopService
()
{
ErrorCode
DBWrapper
::
StopService
()
{
if
(
db_
)
{
db_
->
Stop
();
}
...
...
cpp/src/server/DBWrapper.h
浏览文件 @
52963119
...
...
@@ -29,8 +29,8 @@ public:
return
GetInstance
().
EngineDB
();
}
ServerError
StartService
();
ServerError
StopService
();
ErrorCode
StartService
();
ErrorCode
StopService
();
std
::
shared_ptr
<
engine
::
DB
>
EngineDB
()
{
return
db_
;
...
...
cpp/src/server/Server.cpp
浏览文件 @
52963119
...
...
@@ -210,10 +210,10 @@ Server::Stop() {
}
ServerError
ErrorCode
Server
::
LoadConfig
()
{
ServerConfig
::
GetInstance
().
LoadConfigFile
(
config_filename_
);
ServerError
err
=
ServerConfig
::
GetInstance
().
ValidateConfig
();
ErrorCode
err
=
ServerConfig
::
GetInstance
().
ValidateConfig
();
if
(
err
!=
SERVER_SUCCESS
){
exit
(
0
);
}
...
...
cpp/src/server/Server.h
浏览文件 @
52963119
...
...
@@ -29,7 +29,7 @@ class Server {
void
Daemonize
();
static
void
HandleSignal
(
int
signal
);
ServerError
LoadConfig
();
ErrorCode
LoadConfig
();
void
StartService
();
void
StopService
();
...
...
cpp/src/server/ServerConfig.cpp
浏览文件 @
52963119
...
...
@@ -28,7 +28,7 @@ ServerConfig::GetInstance() {
return
config
;
}
ServerError
ErrorCode
ServerConfig
::
LoadConfigFile
(
const
std
::
string
&
config_filename
)
{
std
::
string
filename
=
config_filename
;
if
(
filename
.
empty
()){
...
...
@@ -44,7 +44,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
try
{
ConfigMgr
*
mgr
=
const_cast
<
ConfigMgr
*>
(
ConfigMgr
::
GetInstance
());
ServerError
err
=
mgr
->
LoadConfigFile
(
filename
);
ErrorCode
err
=
mgr
->
LoadConfigFile
(
filename
);
if
(
err
!=
0
)
{
std
::
cout
<<
"Server failed to load config file"
<<
std
::
endl
;
exit
(
1
);
//directly exit program if the config file is illegal
...
...
@@ -58,7 +58,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
return
SERVER_SUCCESS
;
}
ServerError
ServerConfig
::
ValidateConfig
()
const
{
ErrorCode
ServerConfig
::
ValidateConfig
()
const
{
//server config validation
ConfigNode
server_config
=
GetConfig
(
CONFIG_SERVER
);
uint32_t
gpu_index
=
(
uint32_t
)
server_config
.
GetInt32Value
(
CONFIG_GPU_INDEX
,
0
);
...
...
cpp/src/server/ServerConfig.h
浏览文件 @
52963119
...
...
@@ -68,8 +68,8 @@ class ServerConfig {
public:
static
ServerConfig
&
GetInstance
();
ServerError
LoadConfigFile
(
const
std
::
string
&
config_filename
);
ServerError
ValidateConfig
()
const
;
ErrorCode
LoadConfigFile
(
const
std
::
string
&
config_filename
);
ErrorCode
ValidateConfig
()
const
;
void
PrintAll
()
const
;
ConfigNode
GetConfig
(
const
std
::
string
&
name
)
const
;
...
...
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
浏览文件 @
52963119
...
...
@@ -16,8 +16,8 @@ namespace grpc {
using
namespace
::
milvus
;
namespace
{
const
std
::
map
<
ServerError
,
::
milvus
::
grpc
::
ErrorCode
>
&
ErrorMap
()
{
static
const
std
::
map
<
ServerError
,
::
milvus
::
grpc
::
ErrorCode
>
code_map
=
{
const
std
::
map
<
ErrorCode
,
::
milvus
::
grpc
::
ErrorCode
>
&
ErrorMap
()
{
static
const
std
::
map
<
ErrorCode
,
::
milvus
::
grpc
::
ErrorCode
>
code_map
=
{
{
SERVER_UNEXPECTED_ERROR
,
::
milvus
::
grpc
::
ErrorCode
::
UNEXPECTED_ERROR
},
{
SERVER_UNSUPPORTED_ERROR
,
::
milvus
::
grpc
::
ErrorCode
::
UNEXPECTED_ERROR
},
{
SERVER_NULL_POINTER
,
::
milvus
::
grpc
::
ErrorCode
::
UNEXPECTED_ERROR
},
...
...
@@ -66,7 +66,7 @@ GrpcBaseTask::~GrpcBaseTask() {
WaitToFinish
();
}
ServerError
GrpcBaseTask
::
Execute
()
{
ErrorCode
GrpcBaseTask
::
Execute
()
{
error_code_
=
OnExecute
();
Done
();
return
error_code_
;
...
...
@@ -77,7 +77,7 @@ void GrpcBaseTask::Done() {
finish_cond_
.
notify_all
();
}
ServerError
GrpcBaseTask
::
SetError
(
ServerError
error_code
,
const
std
::
string
&
error_msg
)
{
ErrorCode
GrpcBaseTask
::
SetError
(
ErrorCode
error_code
,
const
std
::
string
&
error_msg
)
{
error_code_
=
error_code
;
error_msg_
=
error_msg
;
...
...
@@ -85,7 +85,7 @@ ServerError GrpcBaseTask::SetError(ServerError error_code, const std::string &er
return
error_code_
;
}
ServerError
GrpcBaseTask
::
WaitToFinish
()
{
ErrorCode
GrpcBaseTask
::
WaitToFinish
()
{
std
::
unique_lock
<
std
::
mutex
>
lock
(
finish_mtx_
);
finish_cond_
.
wait
(
lock
,
[
this
]
{
return
done_
;
});
...
...
@@ -112,7 +112,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu
if
(
!
task_ptr
->
IsAsync
())
{
task_ptr
->
WaitToFinish
();
ServerError
err
=
task_ptr
->
ErrorCode
();
ErrorCode
err
=
task_ptr
->
ErrorID
();
if
(
err
!=
SERVER_SUCCESS
)
{
grpc_status
->
set_reason
(
task_ptr
->
ErrorMsg
());
grpc_status
->
set_error_code
(
ErrorMap
().
at
(
err
));
...
...
@@ -153,12 +153,12 @@ void GrpcRequestScheduler::Stop() {
SERVER_LOG_INFO
<<
"Scheduler stopped"
;
}
ServerError
GrpcRequestScheduler
::
ExecuteTask
(
const
BaseTaskPtr
&
task_ptr
)
{
ErrorCode
GrpcRequestScheduler
::
ExecuteTask
(
const
BaseTaskPtr
&
task_ptr
)
{
if
(
task_ptr
==
nullptr
)
{
return
SERVER_NULL_POINTER
;
}
ServerError
err
=
PutTaskToQueue
(
task_ptr
);
ErrorCode
err
=
PutTaskToQueue
(
task_ptr
);
if
(
err
!=
SERVER_SUCCESS
)
{
SERVER_LOG_ERROR
<<
"Put task to queue failed with code: "
<<
err
;
return
err
;
...
...
@@ -185,7 +185,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
}
try
{
ServerError
err
=
task
->
Execute
();
ErrorCode
err
=
task
->
Execute
();
if
(
err
!=
SERVER_SUCCESS
)
{
SERVER_LOG_ERROR
<<
"Task failed with code: "
<<
err
;
}
...
...
@@ -195,7 +195,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
}
}
ServerError
GrpcRequestScheduler
::
PutTaskToQueue
(
const
BaseTaskPtr
&
task_ptr
)
{
ErrorCode
GrpcRequestScheduler
::
PutTaskToQueue
(
const
BaseTaskPtr
&
task_ptr
)
{
std
::
lock_guard
<
std
::
mutex
>
lock
(
queue_mtx_
);
std
::
string
group_name
=
task_ptr
->
TaskGroup
();
...
...
cpp/src/server/grpc_impl/GrpcRequestScheduler.h
浏览文件 @
52963119
...
...
@@ -25,24 +25,24 @@ protected:
virtual
~
GrpcBaseTask
();
public:
ServerError
Execute
();
ErrorCode
Execute
();
void
Done
();
ServerError
WaitToFinish
();
ErrorCode
WaitToFinish
();
std
::
string
TaskGroup
()
const
{
return
task_group_
;
}
ServerError
ErrorCode
()
const
{
return
error_code_
;
}
ErrorCode
ErrorID
()
const
{
return
error_code_
;
}
std
::
string
ErrorMsg
()
const
{
return
error_msg_
;
}
bool
IsAsync
()
const
{
return
async_
;
}
protected:
virtual
ServerError
OnExecute
()
=
0
;
virtual
ErrorCode
OnExecute
()
=
0
;
ServerError
SetError
(
ServerError
error_code
,
const
std
::
string
&
msg
);
ErrorCode
SetError
(
ErrorCode
error_code
,
const
std
::
string
&
msg
);
protected:
mutable
std
::
mutex
finish_mtx_
;
...
...
@@ -51,7 +51,7 @@ protected:
std
::
string
task_group_
;
bool
async_
;
bool
done_
;
ServerError
error_code_
;
ErrorCode
error_code_
;
std
::
string
error_msg_
;
};
...
...
@@ -71,7 +71,7 @@ public:
void
Stop
();
ServerError
ExecuteTask
(
const
BaseTaskPtr
&
task_ptr
);
ErrorCode
ExecuteTask
(
const
BaseTaskPtr
&
task_ptr
);
static
void
ExecTask
(
BaseTaskPtr
&
task_ptr
,
::
milvus
::
grpc
::
Status
*
grpc_status
);
...
...
@@ -82,7 +82,7 @@ protected:
void
TakeTaskToExecute
(
TaskQueuePtr
task_queue
);
ServerError
PutTaskToQueue
(
const
BaseTaskPtr
&
task_ptr
);
ErrorCode
PutTaskToQueue
(
const
BaseTaskPtr
&
task_ptr
);
private:
mutable
std
::
mutex
queue_mtx_
;
...
...
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @
52963119
...
...
@@ -68,7 +68,7 @@ namespace {
void
ConvertTimeRangeToDBDates
(
const
std
::
vector
<::
milvus
::
grpc
::
Range
>
&
range_array
,
std
::
vector
<
DB_DATE
>
&
dates
,
ServerError
&
error_code
,
ErrorCode
&
error_code
,
std
::
string
&
error_msg
)
{
dates
.
clear
();
for
(
auto
&
range
:
range_array
)
{
...
...
@@ -124,13 +124,13 @@ CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) {
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
CreateTableTask
(
schema
));
}
ServerError
ErrorCode
CreateTableTask
::
OnExecute
()
{
TimeRecorder
rc
(
"CreateTableTask"
);
try
{
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
schema_
->
table_name
().
table_name
());
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
schema_
->
table_name
().
table_name
());
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
schema_
->
table_name
().
table_name
());
}
...
...
@@ -161,7 +161,7 @@ CreateTableTask::OnExecute() {
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
CreateTable
(
table_info
);
if
(
!
stat
.
ok
())
{
//table could exist
if
(
stat
.
IsAlreadyExist
()
)
{
if
(
stat
.
code
()
==
DB_ALREADY_EXIST
)
{
return
SetError
(
SERVER_INVALID_TABLE_NAME
,
stat
.
ToString
());
}
return
SetError
(
DB_META_TRANSACTION_FAILED
,
stat
.
ToString
());
...
...
@@ -188,13 +188,13 @@ DescribeTableTask::Create(const std::string &table_name, ::milvus::grpc::TableSc
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
DescribeTableTask
(
table_name
,
schema
));
}
ServerError
ErrorCode
DescribeTableTask
::
OnExecute
()
{
TimeRecorder
rc
(
"DescribeTableTask"
);
try
{
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -236,14 +236,14 @@ CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) {
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
CreateIndexTask
(
index_param
));
}
ServerError
ErrorCode
CreateIndexTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"CreateIndexTask"
);
//step 1: check arguments
std
::
string
table_name_
=
index_param_
->
table_name
().
table_name
();
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -299,13 +299,13 @@ HasTableTask::Create(const std::string &table_name, bool &has_table) {
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
HasTableTask
(
table_name
,
has_table
));
}
ServerError
ErrorCode
HasTableTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"HasTableTask"
);
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -336,13 +336,13 @@ DropTableTask::Create(const std::string &table_name) {
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
DropTableTask
(
table_name
));
}
ServerError
ErrorCode
DropTableTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"DropTableTask"
);
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -352,7 +352,7 @@ DropTableTask::OnExecute() {
table_info
.
table_id_
=
table_name_
;
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
DescribeTable
(
table_info
);
if
(
!
stat
.
ok
())
{
if
(
stat
.
IsNotFound
()
)
{
if
(
stat
.
code
()
==
DB_NOT_FOUND
)
{
return
SetError
(
SERVER_TABLE_NOT_EXIST
,
"Table "
+
table_name_
+
" not exists"
);
}
else
{
return
SetError
(
DB_META_TRANSACTION_FAILED
,
stat
.
ToString
());
...
...
@@ -388,7 +388,7 @@ ShowTablesTask::Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer)
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
ShowTablesTask
(
writer
));
}
ServerError
ErrorCode
ShowTablesTask
::
OnExecute
()
{
std
::
vector
<
engine
::
meta
::
TableSchema
>
schema_array
;
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
AllTables
(
schema_array
);
...
...
@@ -425,13 +425,13 @@ InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param,
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
InsertTask
(
insert_param
,
record_ids
));
}
ServerError
ErrorCode
InsertTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"InsertVectorTask"
);
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
insert_param_
->
table_name
());
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
insert_param_
->
table_name
());
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
insert_param_
->
table_name
());
}
...
...
@@ -451,7 +451,7 @@ InsertTask::OnExecute() {
table_info
.
table_id_
=
insert_param_
->
table_name
();
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
DescribeTable
(
table_info
);
if
(
!
stat
.
ok
())
{
if
(
stat
.
IsNotFound
()
)
{
if
(
stat
.
code
()
==
DB_NOT_FOUND
)
{
return
SetError
(
SERVER_TABLE_NOT_EXIST
,
"Table "
+
insert_param_
->
table_name
()
+
" not exists"
);
}
else
{
...
...
@@ -489,7 +489,7 @@ InsertTask::OnExecute() {
}
uint64_t
vec_dim
=
insert_param_
->
row_record_array
(
i
).
vector_data
().
size
();
if
(
vec_dim
!=
table_info
.
dimension_
)
{
ServerError
error_code
=
SERVER_INVALID_VECTOR_DIMENSION
;
ErrorCode
error_code
=
SERVER_INVALID_VECTOR_DIMENSION
;
std
::
string
error_msg
=
"Invalid rowrecord dimension: "
+
std
::
to_string
(
vec_dim
)
+
" vs. table dimension:"
+
std
::
to_string
(
table_info
.
dimension_
);
...
...
@@ -569,14 +569,14 @@ SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos,
response
));
}
ServerError
ErrorCode
SearchTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"SearchTask"
);
//step 1: check table name
std
::
string
table_name_
=
search_param_
->
table_name
();
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -586,7 +586,7 @@ SearchTask::OnExecute() {
table_info
.
table_id_
=
table_name_
;
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
DescribeTable
(
table_info
);
if
(
!
stat
.
ok
())
{
if
(
stat
.
IsNotFound
()
)
{
if
(
stat
.
code
()
==
DB_NOT_FOUND
)
{
return
SetError
(
SERVER_TABLE_NOT_EXIST
,
"Table "
+
table_name_
+
" not exists"
);
}
else
{
return
SetError
(
DB_META_TRANSACTION_FAILED
,
stat
.
ToString
());
...
...
@@ -612,7 +612,7 @@ SearchTask::OnExecute() {
//step 4: check date range, and convert to db dates
std
::
vector
<
DB_DATE
>
dates
;
ServerError
error_code
=
SERVER_SUCCESS
;
ErrorCode
error_code
=
SERVER_SUCCESS
;
std
::
string
error_msg
;
std
::
vector
<::
milvus
::
grpc
::
Range
>
range_array
;
...
...
@@ -636,7 +636,7 @@ SearchTask::OnExecute() {
}
uint64_t
query_vec_dim
=
search_param_
->
query_record_array
(
i
).
vector_data
().
size
();
if
(
query_vec_dim
!=
table_info
.
dimension_
)
{
ServerError
error_code
=
SERVER_INVALID_VECTOR_DIMENSION
;
ErrorCode
error_code
=
SERVER_INVALID_VECTOR_DIMENSION
;
std
::
string
error_msg
=
"Invalid rowrecord dimension: "
+
std
::
to_string
(
query_vec_dim
)
+
" vs. table dimension:"
+
std
::
to_string
(
table_info
.
dimension_
);
return
SetError
(
error_code
,
error_msg
);
...
...
@@ -721,13 +721,13 @@ CountTableTask::Create(const std::string &table_name, int64_t &row_count) {
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
CountTableTask
(
table_name
,
row_count
));
}
ServerError
ErrorCode
CountTableTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"GetTableRowCountTask"
);
//step 1: check arguments
ServerError
res
=
SERVER_SUCCESS
;
ErrorCode
res
=
SERVER_SUCCESS
;
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
...
...
@@ -764,7 +764,7 @@ CmdTask::Create(const std::string &cmd, std::string &result) {
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
CmdTask
(
cmd
,
result
));
}
ServerError
ErrorCode
CmdTask
::
OnExecute
()
{
if
(
cmd_
==
"version"
)
{
result_
=
MILVUS_VERSION
;
...
...
@@ -793,14 +793,14 @@ DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_ra
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
DeleteByRangeTask
(
delete_by_range_param
));
}
ServerError
ErrorCode
DeleteByRangeTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"DeleteByRangeTask"
);
//step 1: check arguments
std
::
string
table_name
=
delete_by_range_param_
->
table_name
();
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name
);
}
...
...
@@ -810,7 +810,7 @@ DeleteByRangeTask::OnExecute() {
table_info
.
table_id_
=
table_name
;
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
DescribeTable
(
table_info
);
if
(
!
stat
.
ok
())
{
if
(
stat
.
IsNotFound
()
)
{
if
(
stat
.
code
(),
DB_NOT_FOUND
)
{
return
SetError
(
SERVER_TABLE_NOT_EXIST
,
"Table "
+
table_name
+
" not exists"
);
}
else
{
return
SetError
(
DB_META_TRANSACTION_FAILED
,
stat
.
ToString
());
...
...
@@ -821,7 +821,7 @@ DeleteByRangeTask::OnExecute() {
//step 3: check date range, and convert to db dates
std
::
vector
<
DB_DATE
>
dates
;
ServerError
error_code
=
SERVER_SUCCESS
;
ErrorCode
error_code
=
SERVER_SUCCESS
;
std
::
string
error_msg
;
std
::
vector
<::
milvus
::
grpc
::
Range
>
range_array
;
...
...
@@ -859,13 +859,13 @@ PreloadTableTask::Create(const std::string &table_name){
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
PreloadTableTask
(
table_name
));
}
ServerError
ErrorCode
PreloadTableTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"PreloadTableTask"
);
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -899,13 +899,13 @@ DescribeIndexTask::Create(const std::string &table_name,
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
DescribeIndexTask
(
table_name
,
index_param
));
}
ServerError
ErrorCode
DescribeIndexTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"DescribeIndexTask"
);
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
@@ -941,13 +941,13 @@ DropIndexTask::Create(const std::string &table_name){
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
DropIndexTask
(
table_name
));
}
ServerError
ErrorCode
DropIndexTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"DropIndexTask"
);
//step 1: check arguments
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
ErrorCode
res
=
ValidationUtil
::
ValidateTableName
(
table_name_
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name_
);
}
...
...
cpp/src/server/grpc_impl/GrpcRequestTask.h
浏览文件 @
52963119
...
...
@@ -29,7 +29,7 @@ protected:
explicit
CreateTableTask
(
const
::
milvus
::
grpc
::
TableSchema
*
request
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -45,7 +45,7 @@ public:
protected:
HasTableTask
(
const
std
::
string
&
request
,
bool
&
has_table
);
ServerError
ErrorCode
OnExecute
()
override
;
...
...
@@ -63,7 +63,7 @@ public:
protected:
DescribeTableTask
(
const
std
::
string
&
table_name
,
::
milvus
::
grpc
::
TableSchema
*
schema
);
ServerError
ErrorCode
OnExecute
()
override
;
...
...
@@ -82,7 +82,7 @@ protected:
explicit
DropTableTask
(
const
std
::
string
&
table_name
);
ServerError
ErrorCode
OnExecute
()
override
;
...
...
@@ -100,7 +100,7 @@ protected:
explicit
CreateIndexTask
(
const
::
milvus
::
grpc
::
IndexParam
*
index_Param
);
ServerError
ErrorCode
OnExecute
()
override
;
...
...
@@ -118,7 +118,7 @@ protected:
explicit
ShowTablesTask
(
::
grpc
::
ServerWriter
<::
milvus
::
grpc
::
TableName
>
*
writer
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -136,7 +136,7 @@ protected:
InsertTask
(
const
::
milvus
::
grpc
::
InsertParam
*
insert_Param
,
::
milvus
::
grpc
::
VectorIds
*
record_ids_
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -157,7 +157,7 @@ protected:
const
std
::
vector
<
std
::
string
>
&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResultList
*
response
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -175,7 +175,7 @@ public:
protected:
CountTableTask
(
const
std
::
string
&
table_name
,
int64_t
&
row_count
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -192,7 +192,7 @@ public:
protected:
CmdTask
(
const
std
::
string
&
cmd
,
std
::
string
&
result
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -209,7 +209,7 @@ public:
protected:
DeleteByRangeTask
(
const
::
milvus
::
grpc
::
DeleteByRangeParam
*
delete_by_range_param
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -225,7 +225,7 @@ public:
protected:
PreloadTableTask
(
const
std
::
string
&
table_name
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -243,7 +243,7 @@ protected:
DescribeIndexTask
(
const
std
::
string
&
table_name
,
::
milvus
::
grpc
::
IndexParam
*
index_param
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
@@ -260,7 +260,7 @@ public:
protected:
DropIndexTask
(
const
std
::
string
&
table_name
);
ServerError
ErrorCode
OnExecute
()
override
;
private:
...
...
cpp/src/utils/CommonUtil.cpp
浏览文件 @
52963119
...
...
@@ -61,7 +61,7 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) {
return
true
;
}
ServerError
CommonUtil
::
CreateDirectory
(
const
std
::
string
&
path
)
{
ErrorCode
CommonUtil
::
CreateDirectory
(
const
std
::
string
&
path
)
{
if
(
path
.
empty
())
{
return
SERVER_SUCCESS
;
}
...
...
@@ -74,7 +74,7 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) {
fs
::
path
fs_path
(
path
);
fs
::
path
parent_path
=
fs_path
.
parent_path
();
ServerError
err
=
CreateDirectory
(
parent_path
.
string
());
ErrorCode
err
=
CreateDirectory
(
parent_path
.
string
());
if
(
err
!=
SERVER_SUCCESS
){
return
err
;
}
...
...
@@ -122,7 +122,7 @@ namespace {
}
}
ServerError
CommonUtil
::
DeleteDirectory
(
const
std
::
string
&
path
)
{
ErrorCode
CommonUtil
::
DeleteDirectory
(
const
std
::
string
&
path
)
{
if
(
path
.
empty
())
{
return
SERVER_SUCCESS
;
}
...
...
cpp/src/utils/CommonUtil.h
浏览文件 @
52963119
...
...
@@ -22,8 +22,8 @@ class CommonUtil {
static
bool
IsFileExist
(
const
std
::
string
&
path
);
static
uint64_t
GetFileSize
(
const
std
::
string
&
path
);
static
bool
IsDirectoryExist
(
const
std
::
string
&
path
);
static
ServerError
CreateDirectory
(
const
std
::
string
&
path
);
static
ServerError
DeleteDirectory
(
const
std
::
string
&
path
);
static
ErrorCode
CreateDirectory
(
const
std
::
string
&
path
);
static
ErrorCode
DeleteDirectory
(
const
std
::
string
&
path
);
static
std
::
string
GetExePath
();
...
...
cpp/src/utils/Error.h
浏览文件 @
52963119
...
...
@@ -11,82 +11,95 @@
namespace
zilliz
{
namespace
milvus
{
namespace
server
{
using
ServerError
=
int32_t
;
constexpr
ServerError
SERVER_SUCCESS
=
0
;
constexpr
ServerError
SERVER_ERROR_CODE_BASE
=
0x30000
;
using
ErrorCode
=
int32_t
;
constexpr
ServerError
ToGlobalServerErrorCode
(
const
ServerError
error_code
)
{
constexpr
ErrorCode
SERVER_SUCCESS
=
0
;
constexpr
ErrorCode
SERVER_ERROR_CODE_BASE
=
0x30000
;
constexpr
ErrorCode
ToServerErrorCode
(
const
ErrorCode
error_code
)
{
return
SERVER_ERROR_CODE_BASE
+
error_code
;
}
constexpr
ServerError
SERVER_UNEXPECTED_ERROR
=
ToGlobalServerErrorCode
(
1
);
constexpr
ServerError
SERVER_UNSUPPORTED_ERROR
=
ToGlobalServerErrorCode
(
2
);
constexpr
ServerError
SERVER_NULL_POINTER
=
ToGlobalServerErrorCode
(
3
);
constexpr
ServerError
SERVER_INVALID_ARGUMENT
=
ToGlobalServerErrorCode
(
4
);
constexpr
ServerError
SERVER_FILE_NOT_FOUND
=
ToGlobalServerErrorCode
(
5
);
constexpr
ServerError
SERVER_NOT_IMPLEMENT
=
ToGlobalServerErrorCode
(
6
);
constexpr
ServerError
SERVER_BLOCKING_QUEUE_EMPTY
=
ToGlobalServerErrorCode
(
7
);
constexpr
ServerError
SERVER_CANNOT_CREATE_FOLDER
=
ToGlobalServerErrorCode
(
8
);
constexpr
ServerError
SERVER_CANNOT_CREATE_FILE
=
ToGlobalServerErrorCode
(
9
);
constexpr
ServerError
SERVER_CANNOT_DELETE_FOLDER
=
ToGlobalServerErrorCode
(
10
);
constexpr
ServerError
SERVER_CANNOT_DELETE_FILE
=
ToGlobalServerErrorCode
(
11
);
constexpr
ServerError
SERVER_BUILD_INDEX_ERROR
=
ToGlobalServerErrorCode
(
12
);
constexpr
ServerError
SERVER_TABLE_NOT_EXIST
=
ToGlobalServerErrorCode
(
100
);
constexpr
ServerError
SERVER_INVALID_TABLE_NAME
=
ToGlobalServerErrorCode
(
101
);
constexpr
ServerError
SERVER_INVALID_TABLE_DIMENSION
=
ToGlobalServerErrorCode
(
102
);
constexpr
ServerError
SERVER_INVALID_TIME_RANGE
=
ToGlobalServerErrorCode
(
103
);
constexpr
ServerError
SERVER_INVALID_VECTOR_DIMENSION
=
ToGlobalServerErrorCode
(
104
);
constexpr
ServerError
SERVER_INVALID_INDEX_TYPE
=
ToGlobalServerErrorCode
(
105
);
constexpr
ServerError
SERVER_INVALID_ROWRECORD
=
ToGlobalServerErrorCode
(
106
);
constexpr
ServerError
SERVER_INVALID_ROWRECORD_ARRAY
=
ToGlobalServerErrorCode
(
107
);
constexpr
ServerError
SERVER_INVALID_TOPK
=
ToGlobalServerErrorCode
(
108
);
constexpr
ServerError
SERVER_ILLEGAL_VECTOR_ID
=
ToGlobalServerErrorCode
(
109
);
constexpr
ServerError
SERVER_ILLEGAL_SEARCH_RESULT
=
ToGlobalServerErrorCode
(
110
);
constexpr
ServerError
SERVER_CACHE_ERROR
=
ToGlobalServerErrorCode
(
111
);
constexpr
ServerError
SERVER_WRITE_ERROR
=
ToGlobalServerErrorCode
(
112
);
constexpr
ServerError
SERVER_INVALID_NPROBE
=
ToGlobalServerErrorCode
(
113
);
constexpr
ServerError
SERVER_INVALID_INDEX_NLIST
=
ToGlobalServerErrorCode
(
114
);
constexpr
ServerError
SERVER_INVALID_INDEX_METRIC_TYPE
=
ToGlobalServerErrorCode
(
115
);
constexpr
ServerError
SERVER_INVALID_INDEX_FILE_SIZE
=
ToGlobalServerErrorCode
(
116
);
constexpr
ServerError
SERVER_LICENSE_FILE_NOT_EXIST
=
ToGlobalServerErrorCode
(
500
);
constexpr
ServerError
SERVER_LICENSE_VALIDATION_FAIL
=
ToGlobalServerErrorCode
(
501
);
constexpr
ErrorCode
DB_SUCCESS
=
0
;
constexpr
ErrorCode
DB_ERROR_CODE_BASE
=
0x40000
;
constexpr
ErrorCode
ToDbErrorCode
(
const
ErrorCode
error_code
)
{
return
DB_ERROR_CODE_BASE
+
error_code
;
}
constexpr
ServerError
DB_META_TRANSACTION_FAILED
=
ToGlobalServerErrorCode
(
1000
);
constexpr
ErrorCode
KNOWHERE_SUCCESS
=
0
;
constexpr
ErrorCode
KNOWHERE_ERROR_CODE_BASE
=
0x50000
;
constexpr
ErrorCode
ToKnowhereErrorCode
(
const
ErrorCode
error_code
)
{
return
KNOWHERE_ERROR_CODE_BASE
+
error_code
;
}
using
KnowhereError
=
int32_t
;
constexpr
KnowhereError
KNOWHERE_SUCCESS
=
0
;
constexpr
KnowhereError
KNOWHERE_ERROR
=
ToGlobalServerErrorCode
(
1
);
constexpr
KnowhereError
KNOWHERE_INVALID_ARGUMENT
=
ToGlobalServerErrorCode
(
2
);
constexpr
KnowhereError
KNOWHERE_UNEXPECTED_ERROR
=
ToGlobalServerErrorCode
(
3
);
//server error code
constexpr
ErrorCode
SERVER_UNEXPECTED_ERROR
=
ToServerErrorCode
(
1
);
constexpr
ErrorCode
SERVER_UNSUPPORTED_ERROR
=
ToServerErrorCode
(
2
);
constexpr
ErrorCode
SERVER_NULL_POINTER
=
ToServerErrorCode
(
3
);
constexpr
ErrorCode
SERVER_INVALID_ARGUMENT
=
ToServerErrorCode
(
4
);
constexpr
ErrorCode
SERVER_FILE_NOT_FOUND
=
ToServerErrorCode
(
5
);
constexpr
ErrorCode
SERVER_NOT_IMPLEMENT
=
ToServerErrorCode
(
6
);
constexpr
ErrorCode
SERVER_BLOCKING_QUEUE_EMPTY
=
ToServerErrorCode
(
7
);
constexpr
ErrorCode
SERVER_CANNOT_CREATE_FOLDER
=
ToServerErrorCode
(
8
);
constexpr
ErrorCode
SERVER_CANNOT_CREATE_FILE
=
ToServerErrorCode
(
9
);
constexpr
ErrorCode
SERVER_CANNOT_DELETE_FOLDER
=
ToServerErrorCode
(
10
);
constexpr
ErrorCode
SERVER_CANNOT_DELETE_FILE
=
ToServerErrorCode
(
11
);
constexpr
ErrorCode
SERVER_BUILD_INDEX_ERROR
=
ToServerErrorCode
(
12
);
constexpr
ErrorCode
SERVER_TABLE_NOT_EXIST
=
ToServerErrorCode
(
100
);
constexpr
ErrorCode
SERVER_INVALID_TABLE_NAME
=
ToServerErrorCode
(
101
);
constexpr
ErrorCode
SERVER_INVALID_TABLE_DIMENSION
=
ToServerErrorCode
(
102
);
constexpr
ErrorCode
SERVER_INVALID_TIME_RANGE
=
ToServerErrorCode
(
103
);
constexpr
ErrorCode
SERVER_INVALID_VECTOR_DIMENSION
=
ToServerErrorCode
(
104
);
constexpr
ErrorCode
SERVER_INVALID_INDEX_TYPE
=
ToServerErrorCode
(
105
);
constexpr
ErrorCode
SERVER_INVALID_ROWRECORD
=
ToServerErrorCode
(
106
);
constexpr
ErrorCode
SERVER_INVALID_ROWRECORD_ARRAY
=
ToServerErrorCode
(
107
);
constexpr
ErrorCode
SERVER_INVALID_TOPK
=
ToServerErrorCode
(
108
);
constexpr
ErrorCode
SERVER_ILLEGAL_VECTOR_ID
=
ToServerErrorCode
(
109
);
constexpr
ErrorCode
SERVER_ILLEGAL_SEARCH_RESULT
=
ToServerErrorCode
(
110
);
constexpr
ErrorCode
SERVER_CACHE_ERROR
=
ToServerErrorCode
(
111
);
constexpr
ErrorCode
SERVER_WRITE_ERROR
=
ToServerErrorCode
(
112
);
constexpr
ErrorCode
SERVER_INVALID_NPROBE
=
ToServerErrorCode
(
113
);
constexpr
ErrorCode
SERVER_INVALID_INDEX_NLIST
=
ToServerErrorCode
(
114
);
constexpr
ErrorCode
SERVER_INVALID_INDEX_METRIC_TYPE
=
ToServerErrorCode
(
115
);
constexpr
ErrorCode
SERVER_INVALID_INDEX_FILE_SIZE
=
ToServerErrorCode
(
116
);
//db error code
constexpr
ErrorCode
DB_META_TRANSACTION_FAILED
=
ToDbErrorCode
(
1
);
constexpr
ErrorCode
DB_ERROR
=
ToDbErrorCode
(
2
);
constexpr
ErrorCode
DB_NOT_FOUND
=
ToDbErrorCode
(
3
);
constexpr
ErrorCode
DB_ALREADY_EXIST
=
ToDbErrorCode
(
4
);
constexpr
ErrorCode
DB_INVALID_PATH
=
ToDbErrorCode
(
5
);
//knowhere error code
constexpr
ErrorCode
KNOWHERE_ERROR
=
ToKnowhereErrorCode
(
1
);
constexpr
ErrorCode
KNOWHERE_INVALID_ARGUMENT
=
ToKnowhereErrorCode
(
2
);
constexpr
ErrorCode
KNOWHERE_UNEXPECTED_ERROR
=
ToKnowhereErrorCode
(
3
);
class
ServerException
:
public
std
::
exception
{
public:
ServerException
(
ServerError
error_code
,
const
std
::
string
&
message
=
std
::
string
())
namespace
server
{
class
ServerException
:
public
std
::
exception
{
public:
ServerException
(
ErrorCode
error_code
,
const
std
::
string
&
message
=
std
::
string
())
:
error_code_
(
error_code
),
message_
(
message
)
{}
public:
ServerError
error_code
()
const
{
return
error_code_
;
}
public:
ErrorCode
error_code
()
const
{
return
error_code_
;
}
virtual
const
char
*
what
()
const
noexcept
{
return
message_
.
c_str
();
}
virtual
const
char
*
what
()
const
noexcept
{
return
message_
.
c_str
();
}
private:
ServerError
error_code_
;
std
::
string
message_
;
};
private:
ErrorCode
error_code_
;
std
::
string
message_
;
};
}
}
// namespace server
}
// namespace milvus
}
// namespace zilliz
cpp/src/utils/StringHelpFunctions.cpp
浏览文件 @
52963119
...
...
@@ -24,9 +24,9 @@ void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string
}
}
ServerError
StringHelpFunctions
::
SplitStringByDelimeter
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
std
::
vector
<
std
::
string
>
&
result
)
{
ErrorCode
StringHelpFunctions
::
SplitStringByDelimeter
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
std
::
vector
<
std
::
string
>
&
result
)
{
if
(
str
.
empty
())
{
return
SERVER_SUCCESS
;
}
...
...
@@ -46,10 +46,10 @@ ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str,
return
SERVER_SUCCESS
;
}
ServerError
StringHelpFunctions
::
SplitStringByQuote
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
const
std
::
string
&
quote
,
std
::
vector
<
std
::
string
>
&
result
)
{
ErrorCode
StringHelpFunctions
::
SplitStringByQuote
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
const
std
::
string
&
quote
,
std
::
vector
<
std
::
string
>
&
result
)
{
if
(
quote
.
empty
())
{
return
SplitStringByDelimeter
(
str
,
delimeter
,
result
);
}
...
...
cpp/src/utils/StringHelpFunctions.h
浏览文件 @
52963119
...
...
@@ -29,9 +29,9 @@ public:
// ,b, | b |
// ,, | |
// a a
static
ServerError
SplitStringByDelimeter
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
std
::
vector
<
std
::
string
>
&
result
);
static
ErrorCode
SplitStringByDelimeter
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
std
::
vector
<
std
::
string
>
&
result
);
//assume the table has two columns, quote='\"', delimeter=','
// a,b a | b
...
...
@@ -40,10 +40,10 @@ public:
// "aa,bb" aa,bb
// 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk
// "55,1122"aa,bb",yyy,"kkk" illegal
static
ServerError
SplitStringByQuote
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
const
std
::
string
&
quote
,
std
::
vector
<
std
::
string
>
&
result
);
static
ErrorCode
SplitStringByQuote
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
const
std
::
string
&
quote
,
std
::
vector
<
std
::
string
>
&
result
);
};
...
...
cpp/src/utils/ValidationUtil.cpp
浏览文件 @
52963119
...
...
@@ -12,7 +12,7 @@ constexpr size_t table_name_size_limit = 255;
constexpr
int64_t
table_dimension_limit
=
16384
;
constexpr
int32_t
index_file_size_limit
=
4096
;
//index trigger size max = 4096 MB
ServerError
ErrorCode
ValidationUtil
::
ValidateTableName
(
const
std
::
string
&
table_name
)
{
// Table name shouldn't be empty.
...
...
@@ -46,7 +46,7 @@ ValidationUtil::ValidateTableName(const std::string &table_name) {
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateTableDimension
(
int64_t
dimension
)
{
if
(
dimension
<=
0
||
dimension
>
table_dimension_limit
)
{
SERVER_LOG_ERROR
<<
"Table dimension excceed the limitation: "
<<
table_dimension_limit
;
...
...
@@ -56,7 +56,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) {
}
}
ServerError
ErrorCode
ValidationUtil
::
ValidateTableIndexType
(
int32_t
index_type
)
{
int
engine_type
=
(
int
)
engine
::
EngineType
(
index_type
);
if
(
engine_type
<=
0
||
engine_type
>
(
int
)
engine
::
EngineType
::
MAX_VALUE
)
{
...
...
@@ -66,7 +66,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateTableIndexNlist
(
int32_t
nlist
)
{
if
(
nlist
<=
0
)
{
return
SERVER_INVALID_INDEX_NLIST
;
...
...
@@ -75,7 +75,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateTableIndexFileSize
(
int64_t
index_file_size
)
{
if
(
index_file_size
<=
0
||
index_file_size
>
index_file_size_limit
)
{
return
SERVER_INVALID_INDEX_FILE_SIZE
;
...
...
@@ -84,7 +84,7 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateTableIndexMetricType
(
int32_t
metric_type
)
{
if
(
metric_type
!=
(
int32_t
)
engine
::
MetricType
::
L2
&&
metric_type
!=
(
int32_t
)
engine
::
MetricType
::
IP
)
{
return
SERVER_INVALID_INDEX_METRIC_TYPE
;
...
...
@@ -92,7 +92,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateSearchTopk
(
int64_t
top_k
,
const
engine
::
meta
::
TableSchema
&
table_schema
)
{
if
(
top_k
<=
0
||
top_k
>
1024
)
{
return
SERVER_INVALID_TOPK
;
...
...
@@ -101,7 +101,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateSearchNprobe
(
int64_t
nprobe
,
const
engine
::
meta
::
TableSchema
&
table_schema
)
{
if
(
nprobe
<=
0
||
nprobe
>
table_schema
.
nlist_
)
{
return
SERVER_INVALID_NPROBE
;
...
...
@@ -110,7 +110,7 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
ValidateGpuIndex
(
uint32_t
gpu_index
)
{
int
num_devices
=
0
;
auto
cuda_err
=
cudaGetDeviceCount
(
&
num_devices
);
...
...
@@ -126,7 +126,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
return
SERVER_SUCCESS
;
}
ServerError
ErrorCode
ValidationUtil
::
GetGpuMemory
(
uint32_t
gpu_index
,
size_t
&
memory
)
{
cudaDeviceProp
deviceProp
;
auto
cuda_err
=
cudaGetDeviceProperties
(
&
deviceProp
,
gpu_index
);
...
...
cpp/src/utils/ValidationUtil.h
浏览文件 @
52963119
...
...
@@ -9,34 +9,34 @@ namespace server {
class
ValidationUtil
{
public:
static
ServerError
static
ErrorCode
ValidateTableName
(
const
std
::
string
&
table_name
);
static
ServerError
static
ErrorCode
ValidateTableDimension
(
int64_t
dimension
);
static
ServerError
static
ErrorCode
ValidateTableIndexType
(
int32_t
index_type
);
static
ServerError
static
ErrorCode
ValidateTableIndexNlist
(
int32_t
nlist
);
static
ServerError
static
ErrorCode
ValidateTableIndexFileSize
(
int64_t
index_file_size
);
static
ServerError
static
ErrorCode
ValidateTableIndexMetricType
(
int32_t
metric_type
);
static
ServerError
static
ErrorCode
ValidateSearchTopk
(
int64_t
top_k
,
const
engine
::
meta
::
TableSchema
&
table_schema
);
static
ServerError
static
ErrorCode
ValidateSearchNprobe
(
int64_t
nprobe
,
const
engine
::
meta
::
TableSchema
&
table_schema
);
static
ServerError
static
ErrorCode
ValidateGpuIndex
(
uint32_t
gpu_index
);
static
ServerError
static
ErrorCode
GetGpuMemory
(
uint32_t
gpu_index
,
size_t
&
memory
);
};
...
...
cpp/src/wrapper/knowhere/vec_impl.cpp
浏览文件 @
52963119
...
...
@@ -21,7 +21,7 @@ namespace engine {
using
namespace
zilliz
::
knowhere
;
server
::
KnowhereError
VecIndexImpl
::
BuildAll
(
const
long
&
nb
,
ErrorCode
VecIndexImpl
::
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
...
...
@@ -38,36 +38,36 @@ server::KnowhereError VecIndexImpl::BuildAll(const long &nb,
index_
->
Add
(
dataset
,
cfg
);
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
jsoncons
::
json_exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_INVALID_ARGUMENT
;
return
KNOWHERE_INVALID_ARGUMENT
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
server
::
KnowhereError
VecIndexImpl
::
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
{
ErrorCode
VecIndexImpl
::
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
{
try
{
auto
dataset
=
GenDatasetWithIds
(
nb
,
dim
,
xb
,
ids
);
index_
->
Add
(
dataset
,
cfg
);
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
jsoncons
::
json_exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_INVALID_ARGUMENT
;
return
KNOWHERE_INVALID_ARGUMENT
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
server
::
KnowhereError
VecIndexImpl
::
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
{
ErrorCode
VecIndexImpl
::
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
{
try
{
auto
k
=
cfg
[
"k"
].
as
<
int
>
();
auto
dataset
=
GenDataset
(
nq
,
dim
,
xq
);
...
...
@@ -105,15 +105,15 @@ server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, floa
memcpy
(
dist
,
p_dist
,
sizeof
(
float
)
*
nq
*
k
);
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
jsoncons
::
json_exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_INVALID_ARGUMENT
;
return
KNOWHERE_INVALID_ARGUMENT
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
zilliz
::
knowhere
::
BinarySet
VecIndexImpl
::
Serialize
()
{
...
...
@@ -121,10 +121,10 @@ zilliz::knowhere::BinarySet VecIndexImpl::Serialize() {
return
index_
->
Serialize
();
}
server
::
KnowhereError
VecIndexImpl
::
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
ErrorCode
VecIndexImpl
::
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
index_
->
Load
(
index_binary
);
dim
=
Dimension
();
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
int64_t
VecIndexImpl
::
Dimension
()
{
...
...
@@ -180,24 +180,24 @@ int64_t *BFIndex::GetRawIds() {
return
std
::
static_pointer_cast
<
IDMAP
>
(
index_
)
->
GetRawIds
();
}
server
::
KnowhereError
BFIndex
::
Build
(
const
Config
&
cfg
)
{
ErrorCode
BFIndex
::
Build
(
const
Config
&
cfg
)
{
try
{
dim
=
cfg
[
"dim"
].
as
<
int
>
();
std
::
static_pointer_cast
<
IDMAP
>
(
index_
)
->
Train
(
cfg
);
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
jsoncons
::
json_exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_INVALID_ARGUMENT
;
return
KNOWHERE_INVALID_ARGUMENT
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
server
::
KnowhereError
BFIndex
::
BuildAll
(
const
long
&
nb
,
ErrorCode
BFIndex
::
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
...
...
@@ -211,19 +211,19 @@ server::KnowhereError BFIndex::BuildAll(const long &nb,
index_
->
Add
(
dataset
,
cfg
);
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
jsoncons
::
json_exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_INVALID_ARGUMENT
;
return
KNOWHERE_INVALID_ARGUMENT
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
// TODO(linxj): add lock here.
server
::
KnowhereError
IVFMixIndex
::
BuildAll
(
const
long
&
nb
,
ErrorCode
IVFMixIndex
::
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
...
...
@@ -245,26 +245,26 @@ server::KnowhereError IVFMixIndex::BuildAll(const long &nb,
type
=
ConvertToCpuIndexType
(
type
);
}
else
{
WRAPPER_LOG_ERROR
<<
"Build IVFMIXIndex Failed"
;
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
jsoncons
::
json_exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_INVALID_ARGUMENT
;
return
KNOWHERE_INVALID_ARGUMENT
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
server
::
KnowhereError
IVFMixIndex
::
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
ErrorCode
IVFMixIndex
::
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
//index_ = std::make_shared<IVF>();
index_
->
Load
(
index_binary
);
dim
=
Dimension
();
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
}
...
...
cpp/src/wrapper/knowhere/vec_impl.h
浏览文件 @
52963119
...
...
@@ -19,7 +19,7 @@ class VecIndexImpl : public VecIndex {
public:
explicit
VecIndexImpl
(
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
,
const
IndexType
&
type
)
:
index_
(
std
::
move
(
index
)),
type
(
type
)
{};
server
::
KnowhereError
BuildAll
(
const
long
&
nb
,
ErrorCode
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
...
...
@@ -30,12 +30,12 @@ class VecIndexImpl : public VecIndex {
IndexType
GetType
()
override
;
int64_t
Dimension
()
override
;
int64_t
Count
()
override
;
server
::
KnowhereError
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
override
;
ErrorCode
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
override
;
zilliz
::
knowhere
::
BinarySet
Serialize
()
override
;
server
::
KnowhereError
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
override
;
ErrorCode
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
override
;
VecIndexPtr
Clone
()
override
;
int64_t
GetDeviceId
()
override
;
server
::
KnowhereError
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
override
;
ErrorCode
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
override
;
protected:
int64_t
dim
=
0
;
...
...
@@ -48,22 +48,22 @@ class IVFMixIndex : public VecIndexImpl {
explicit
IVFMixIndex
(
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
,
const
IndexType
&
type
)
:
VecIndexImpl
(
std
::
move
(
index
),
type
)
{};
server
::
KnowhereError
BuildAll
(
const
long
&
nb
,
ErrorCode
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
const
long
&
nt
,
const
float
*
xt
)
override
;
server
::
KnowhereError
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
override
;
ErrorCode
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
override
;
};
class
BFIndex
:
public
VecIndexImpl
{
public:
explicit
BFIndex
(
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
)
:
VecIndexImpl
(
std
::
move
(
index
),
IndexType
::
FAISS_IDMAP
)
{};
server
::
KnowhereError
Build
(
const
Config
&
cfg
);
ErrorCode
Build
(
const
Config
&
cfg
);
float
*
GetRawVectors
();
server
::
KnowhereError
BuildAll
(
const
long
&
nb
,
ErrorCode
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
...
...
cpp/src/wrapper/knowhere/vec_index.cpp
浏览文件 @
52963119
...
...
@@ -172,7 +172,7 @@ VecIndexPtr read_index(const std::string &location) {
return
LoadVecIndex
(
current_type
,
load_data_list
);
}
server
::
KnowhereError
write_index
(
VecIndexPtr
index
,
const
std
::
string
&
location
)
{
ErrorCode
write_index
(
VecIndexPtr
index
,
const
std
::
string
&
location
)
{
try
{
auto
binaryset
=
index
->
Serialize
();
auto
index_type
=
index
->
GetType
();
...
...
@@ -192,12 +192,12 @@ server::KnowhereError write_index(VecIndexPtr index, const std::string &location
}
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_UNEXPECTED_ERROR
;
return
KNOWHERE_UNEXPECTED_ERROR
;
}
catch
(
std
::
exception
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
server
::
KNOWHERE_ERROR
;
return
KNOWHERE_ERROR
;
}
return
server
::
KNOWHERE_SUCCESS
;
return
KNOWHERE_SUCCESS
;
}
...
...
cpp/src/wrapper/knowhere/vec_index.h
浏览文件 @
52963119
...
...
@@ -44,23 +44,23 @@ using VecIndexPtr = std::shared_ptr<VecIndex>;
class
VecIndex
{
public:
virtual
server
::
KnowhereError
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
const
long
&
nt
=
0
,
const
float
*
xt
=
nullptr
)
=
0
;
virtual
server
::
KnowhereError
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
=
Config
())
=
0
;
virtual
server
::
KnowhereError
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
=
Config
())
=
0
;
virtual
ErrorCode
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
,
const
long
&
nt
=
0
,
const
float
*
xt
=
nullptr
)
=
0
;
virtual
ErrorCode
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
=
Config
())
=
0
;
virtual
ErrorCode
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
=
Config
())
=
0
;
virtual
VecIndexPtr
CopyToGpu
(
const
int64_t
&
device_id
,
const
Config
&
cfg
=
Config
())
=
0
;
...
...
@@ -79,10 +79,10 @@ class VecIndex {
virtual
zilliz
::
knowhere
::
BinarySet
Serialize
()
=
0
;
virtual
server
::
KnowhereError
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
=
0
;
virtual
ErrorCode
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
=
0
;
};
extern
server
::
KnowhereError
write_index
(
VecIndexPtr
index
,
const
std
::
string
&
location
);
extern
ErrorCode
write_index
(
VecIndexPtr
index
,
const
std
::
string
&
location
);
extern
VecIndexPtr
read_index
(
const
std
::
string
&
location
);
...
...
cpp/unittest/db/meta_tests.cpp
浏览文件 @
52963119
...
...
@@ -15,6 +15,7 @@
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
using
namespace
zilliz
::
milvus
;
using
namespace
zilliz
::
milvus
::
engine
;
TEST_F
(
MetaTest
,
TABLE_TEST
)
{
...
...
@@ -38,7 +39,7 @@ TEST_F(MetaTest, TABLE_TEST) {
table
.
table_id_
=
table_id
;
status
=
impl_
->
CreateTable
(
table
);
ASSERT_
TRUE
(
status
.
IsAlreadyExist
()
);
ASSERT_
EQ
(
status
.
code
(),
DB_ALREADY_EXIST
);
table
.
table_id_
=
""
;
status
=
impl_
->
CreateTable
(
table
);
...
...
cpp/unittest/db/misc_test.cpp
浏览文件 @
52963119
...
...
@@ -41,24 +41,29 @@ TEST(DBMiscTest, STATUS_TEST) {
std
::
string
str
=
status
.
ToString
();
ASSERT_FALSE
(
str
.
empty
());
status
=
engine
::
Status
::
Error
(
"wrong"
,
"mistake"
);
ASSERT_
TRUE
(
status
.
IsError
()
);
status
=
engine
::
Status
(
DB_ERROR
,
"mistake"
);
ASSERT_
EQ
(
status
.
code
(),
DB_ERROR
);
str
=
status
.
ToString
();
ASSERT_FALSE
(
str
.
empty
());
status
=
engine
::
Status
::
NotFound
(
"wrong"
,
"mistake"
);
ASSERT_
TRUE
(
status
.
IsNotFound
()
);
status
=
engine
::
Status
(
DB_NOT_FOUND
,
"mistake"
);
ASSERT_
EQ
(
status
.
code
(),
DB_NOT_FOUND
);
str
=
status
.
ToString
();
ASSERT_FALSE
(
str
.
empty
());
status
=
engine
::
Status
::
DBTransactionError
(
"wrong"
,
"mistake"
);
ASSERT_TRUE
(
status
.
IsDBTransactionError
());
status
=
engine
::
Status
(
DB_ALREADY_EXIST
,
"mistake"
);
ASSERT_EQ
(
status
.
code
(),
DB_ALREADY_EXIST
);
str
=
status
.
ToString
();
ASSERT_FALSE
(
str
.
empty
());
status
=
engine
::
Status
(
DB_META_TRANSACTION_FAILED
,
"mistake"
);
ASSERT_EQ
(
status
.
code
(),
DB_META_TRANSACTION_FAILED
);
str
=
status
.
ToString
();
ASSERT_FALSE
(
str
.
empty
());
engine
::
Status
status_copy
=
engine
::
Status
::
OK
();
CopyStatus
(
status_copy
,
status
);
ASSERT_
TRUE
(
status
.
IsDBTransactionError
()
);
ASSERT_
EQ
(
status
.
code
(),
DB_META_TRANSACTION_FAILED
);
}
TEST
(
DBMiscTest
,
OPTIONS_TEST
)
{
...
...
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @
52963119
...
...
@@ -19,6 +19,7 @@
#include <iostream>
using
namespace
zilliz
::
milvus
;
using
namespace
zilliz
::
milvus
::
engine
;
TEST_F
(
MySqlMetaTest
,
TABLE_TEST
)
{
...
...
@@ -42,7 +43,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table
.
table_id_
=
table_id
;
status
=
impl_
->
CreateTable
(
table
);
ASSERT_
TRUE
(
status
.
IsAlreadyExist
()
);
ASSERT_
EQ
(
status
.
code
(),
DB_ALREADY_EXIST
);
table
.
table_id_
=
""
;
status
=
impl_
->
CreateTable
(
table
);
...
...
cpp/unittest/metrics/metrics_test.cpp
浏览文件 @
52963119
...
...
@@ -127,7 +127,7 @@ TEST_F(MetricTest, Metric_Tes) {
TEST_F
(
MetricTest
,
Collector_Metrics_Test
){
engine
::
Status
status
=
engine
::
Status
::
OK
();
server
::
CollectInsertMetrics
insert_metrics0
(
0
,
status
);
status
=
engine
::
Status
::
Error
(
"error"
);
status
=
engine
::
Status
(
DB_ERROR
,
"error"
);
server
::
CollectInsertMetrics
insert_metrics1
(
0
,
status
);
server
::
CollectQueryMetrics
query_metrics
(
10
);
...
...
cpp/unittest/server/cache_test.cpp
浏览文件 @
52963119
...
...
@@ -30,12 +30,12 @@ public:
class
MockVecIndex
:
public
engine
::
VecIndex
{
public:
virtual
server
::
KnowhereError
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
engine
::
Config
&
cfg
,
const
long
&
nt
=
0
,
const
float
*
xt
=
nullptr
)
{
virtual
ErrorCode
BuildAll
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
engine
::
Config
&
cfg
,
const
long
&
nt
=
0
,
const
float
*
xt
=
nullptr
)
{
}
...
...
@@ -51,18 +51,18 @@ public:
return
engine
::
IndexType
::
INVALID
;
}
virtual
server
::
KnowhereError
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
engine
::
Config
&
cfg
=
engine
::
Config
())
{
virtual
ErrorCode
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
engine
::
Config
&
cfg
=
engine
::
Config
())
{
}
virtual
server
::
KnowhereError
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
engine
::
Config
&
cfg
=
engine
::
Config
())
{
virtual
ErrorCode
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
engine
::
Config
&
cfg
=
engine
::
Config
())
{
}
...
...
@@ -87,7 +87,7 @@ public:
return
binset
;
}
virtual
server
::
KnowhereError
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
virtual
ErrorCode
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
}
...
...
cpp/unittest/server/config_test.cpp
浏览文件 @
52963119
...
...
@@ -27,14 +27,14 @@ static constexpr uint64_t GB = MB*1024;
TEST
(
ConfigTest
,
CONFIG_TEST
)
{
server
::
ConfigMgr
*
config_mgr
=
server
::
ConfigMgr
::
GetInstance
();
server
::
ServerError
err
=
config_mgr
->
LoadConfigFile
(
""
);
ASSERT_EQ
(
err
,
server
::
SERVER_UNEXPECTED_ERROR
);
ErrorCode
err
=
config_mgr
->
LoadConfigFile
(
""
);
ASSERT_EQ
(
err
,
SERVER_UNEXPECTED_ERROR
);
err
=
config_mgr
->
LoadConfigFile
(
LOG_FILE_PATH
);
ASSERT_EQ
(
err
,
server
::
SERVER_UNEXPECTED_ERROR
);
ASSERT_EQ
(
err
,
SERVER_UNEXPECTED_ERROR
);
err
=
config_mgr
->
LoadConfigFile
(
CONFIG_FILE_PATH
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
config_mgr
->
Print
();
...
...
@@ -95,11 +95,11 @@ TEST(ConfigTest, CONFIG_TEST) {
TEST
(
ConfigTest
,
SERVER_CONFIG_TEST
)
{
server
::
ServerConfig
&
config
=
server
::
ServerConfig
::
GetInstance
();
server
::
ServerError
err
=
config
.
LoadConfigFile
(
CONFIG_FILE_PATH
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ErrorCode
err
=
config
.
LoadConfigFile
(
CONFIG_FILE_PATH
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
err
=
server
::
ServerConfig
::
GetInstance
().
ValidateConfig
();
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
server
::
ConfigNode
node1
=
config
.
GetConfig
(
"server_config"
);
server
::
ConfigNode
&
node2
=
config
.
GetConfig
(
"cache_config"
);
...
...
@@ -125,26 +125,26 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
server
::
ConfigNode
&
cache_config
=
config
.
GetConfig
(
server
::
CONFIG_CACHE
);
cache_config
.
SetValue
(
server
::
CACHE_FREE_PERCENT
,
"2.0"
);
err
=
config
.
ValidateConfig
();
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
err
,
SERVER_SUCCESS
);
size_t
cache_cap
=
16
;
size_t
insert_buffer_size
=
(
total_mem
-
cache_cap
*
GB
+
1
*
GB
)
/
GB
;
db_config
.
SetValue
(
server
::
CONFIG_DB_INSERT_BUFFER_SIZE
,
std
::
to_string
(
insert_buffer_size
));
cache_config
.
SetValue
(
server
::
CONFIG_CPU_CACHE_CAPACITY
,
std
::
to_string
(
cache_cap
));
err
=
config
.
ValidateConfig
();
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
err
,
SERVER_SUCCESS
);
cache_cap
=
total_mem
/
GB
+
2
;
cache_config
.
SetValue
(
server
::
CONFIG_CPU_CACHE_CAPACITY
,
std
::
to_string
(
cache_cap
));
err
=
config
.
ValidateConfig
();
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
err
,
SERVER_SUCCESS
);
insert_buffer_size
=
total_mem
/
GB
+
2
;
db_config
.
SetValue
(
server
::
CONFIG_DB_INSERT_BUFFER_SIZE
,
std
::
to_string
(
insert_buffer_size
));
err
=
config
.
ValidateConfig
();
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
err
,
SERVER_SUCCESS
);
server_config
.
SetValue
(
server
::
CONFIG_GPU_INDEX
,
"9999"
);
err
=
config
.
ValidateConfig
();
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
err
,
SERVER_SUCCESS
);
}
\ No newline at end of file
cpp/unittest/server/util_test.cpp
浏览文件 @
52963119
...
...
@@ -25,8 +25,8 @@ static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
TEST
(
UtilTest
,
EXCEPTION_TEST
)
{
std
::
string
err_msg
=
"failed"
;
server
::
ServerException
ex
(
server
::
SERVER_UNEXPECTED_ERROR
,
err_msg
);
ASSERT_EQ
(
ex
.
error_code
(),
server
::
SERVER_UNEXPECTED_ERROR
);
server
::
ServerException
ex
(
SERVER_UNEXPECTED_ERROR
,
err_msg
);
ASSERT_EQ
(
ex
.
error_code
(),
SERVER_UNEXPECTED_ERROR
);
std
::
string
msg
=
ex
.
what
();
ASSERT_EQ
(
msg
,
err_msg
);
}
...
...
@@ -44,19 +44,19 @@ TEST(UtilTest, COMMON_TEST) {
std
::
string
path1
=
"/tmp/milvus_test/"
;
std
::
string
path2
=
path1
+
"common_test_12345/"
;
std
::
string
path3
=
path2
+
"abcdef"
;
server
::
ServerError
err
=
server
::
CommonUtil
::
CreateDirectory
(
path3
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ErrorCode
err
=
server
::
CommonUtil
::
CreateDirectory
(
path3
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
//test again
err
=
server
::
CommonUtil
::
CreateDirectory
(
path3
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
ASSERT_TRUE
(
server
::
CommonUtil
::
IsDirectoryExist
(
path3
));
err
=
server
::
CommonUtil
::
DeleteDirectory
(
path1
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
//test again
err
=
server
::
CommonUtil
::
DeleteDirectory
(
path1
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
ASSERT_FALSE
(
server
::
CommonUtil
::
IsDirectoryExist
(
path1
));
ASSERT_FALSE
(
server
::
CommonUtil
::
IsFileExist
(
path1
));
...
...
@@ -94,24 +94,24 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) {
str
=
"a,b,c"
;
std
::
vector
<
std
::
string
>
result
;
server
::
ServerError
err
=
server
::
StringHelpFunctions
::
SplitStringByDelimeter
(
str
,
","
,
result
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ErrorCode
err
=
server
::
StringHelpFunctions
::
SplitStringByDelimeter
(
str
,
","
,
result
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
ASSERT_EQ
(
result
.
size
(),
3UL
);
result
.
clear
();
err
=
server
::
StringHelpFunctions
::
SplitStringByQuote
(
str
,
","
,
"
\"
"
,
result
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
ASSERT_EQ
(
result
.
size
(),
3UL
);
result
.
clear
();
err
=
server
::
StringHelpFunctions
::
SplitStringByQuote
(
str
,
","
,
""
,
result
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
ASSERT_EQ
(
result
.
size
(),
3UL
);
str
=
"55,
\"
aa,gg,yy
\"
,b"
;
result
.
clear
();
err
=
server
::
StringHelpFunctions
::
SplitStringByQuote
(
str
,
","
,
"
\"
"
,
result
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
err
,
SERVER_SUCCESS
);
ASSERT_EQ
(
result
.
size
(),
3UL
);
...
...
@@ -159,86 +159,86 @@ TEST(UtilTest, LOG_TEST) {
TEST
(
UtilTest
,
VALIDATE_TABLENAME_TEST
)
{
std
::
string
table_name
=
"Normal123_"
;
server
::
ServerError
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_SUCCESS
);
ErrorCode
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
SERVER_SUCCESS
);
table_name
=
"12sds"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
ASSERT_EQ
(
res
,
SERVER_INVALID_TABLE_NAME
);
table_name
=
""
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
ASSERT_EQ
(
res
,
SERVER_INVALID_TABLE_NAME
);
table_name
=
"_asdasd"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
res
,
SERVER_SUCCESS
);
table_name
=
"!@#!@"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
ASSERT_EQ
(
res
,
SERVER_INVALID_TABLE_NAME
);
table_name
=
"_!@#!@"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
ASSERT_EQ
(
res
,
SERVER_INVALID_TABLE_NAME
);
table_name
=
"中文"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
ASSERT_EQ
(
res
,
SERVER_INVALID_TABLE_NAME
);
table_name
=
std
::
string
(
10000
,
'a'
);
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
ASSERT_EQ
(
res
,
SERVER_INVALID_TABLE_NAME
);
}
TEST
(
UtilTest
,
VALIDATE_DIMENSIONTEST
)
{
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
-
1
),
server
::
SERVER_INVALID_VECTOR_DIMENSION
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
0
),
server
::
SERVER_INVALID_VECTOR_DIMENSION
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
16385
),
server
::
SERVER_INVALID_VECTOR_DIMENSION
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
16384
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
1
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
-
1
),
SERVER_INVALID_VECTOR_DIMENSION
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
0
),
SERVER_INVALID_VECTOR_DIMENSION
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
16385
),
SERVER_INVALID_VECTOR_DIMENSION
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
16384
),
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableDimension
(
1
),
SERVER_SUCCESS
);
}
TEST
(
UtilTest
,
VALIDATE_INDEX_TEST
)
{
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
((
int
)
engine
::
EngineType
::
INVALID
),
server
::
SERVER_INVALID_INDEX_TYPE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
((
int
)
engine
::
EngineType
::
INVALID
),
SERVER_INVALID_INDEX_TYPE
);
for
(
int
i
=
1
;
i
<=
(
int
)
engine
::
EngineType
::
MAX_VALUE
;
i
++
)
{
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
(
i
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
(
i
),
SERVER_SUCCESS
);
}
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
((
int
)
engine
::
EngineType
::
MAX_VALUE
+
1
),
server
::
SERVER_INVALID_INDEX_TYPE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
((
int
)
engine
::
EngineType
::
MAX_VALUE
+
1
),
SERVER_INVALID_INDEX_TYPE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexNlist
(
0
),
server
::
SERVER_INVALID_INDEX_NLIST
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexNlist
(
100
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexNlist
(
0
),
SERVER_INVALID_INDEX_NLIST
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexNlist
(
100
),
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexFileSize
(
0
),
server
::
SERVER_INVALID_INDEX_FILE_SIZE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexFileSize
(
100
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexFileSize
(
0
),
SERVER_INVALID_INDEX_FILE_SIZE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexFileSize
(
100
),
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexMetricType
(
0
),
server
::
SERVER_INVALID_INDEX_METRIC_TYPE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexMetricType
(
1
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexMetricType
(
2
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexMetricType
(
0
),
SERVER_INVALID_INDEX_METRIC_TYPE
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexMetricType
(
1
),
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexMetricType
(
2
),
SERVER_SUCCESS
);
}
TEST
(
ValidationUtilTest
,
ValidateTopkTest
)
{
engine
::
meta
::
TableSchema
schema
;
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateSearchTopk
(
10
,
schema
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchTopk
(
65536
,
schema
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchTopk
(
0
,
schema
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateSearchTopk
(
10
,
schema
),
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchTopk
(
65536
,
schema
),
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchTopk
(
0
,
schema
),
SERVER_SUCCESS
);
}
TEST
(
ValidationUtilTest
,
ValidateNprobeTest
)
{
engine
::
meta
::
TableSchema
schema
;
schema
.
nlist_
=
100
;
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateSearchNprobe
(
10
,
schema
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchNprobe
(
0
,
schema
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchNprobe
(
101
,
schema
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateSearchNprobe
(
10
,
schema
),
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchNprobe
(
0
,
schema
),
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateSearchNprobe
(
101
,
schema
),
SERVER_SUCCESS
);
}
TEST
(
ValidationUtilTest
,
ValidateGpuTest
)
{
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateGpuIndex
(
0
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateGpuIndex
(
100
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateGpuIndex
(
0
),
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateGpuIndex
(
100
),
SERVER_SUCCESS
);
size_t
memory
=
0
;
ASSERT_EQ
(
server
::
ValidationUtil
::
GetGpuMemory
(
0
,
memory
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
GetGpuMemory
(
100
,
memory
),
server
::
SERVER_SUCCESS
);
ASSERT_EQ
(
server
::
ValidationUtil
::
GetGpuMemory
(
0
,
memory
),
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
GetGpuMemory
(
100
,
memory
),
SERVER_SUCCESS
);
}
TEST
(
UtilTest
,
TIMERECORDER_TEST
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录