Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
f97ba056
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f97ba056
编写于
10月 09, 2019
作者:
S
starlord
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-614 Preload table at startup
Former-commit-id: 15ea4eb6d6c6f3437979a8b741a07bee5eaada5b
上级
50496541
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
77 addition
and
6 deletion
+77
-6
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/conf/server_config.template
cpp/conf/server_config.template
+3
-0
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+3
-2
cpp/src/server/Config.cpp
cpp/src/server/Config.cpp
+16
-0
cpp/src/server/Config.h
cpp/src/server/Config.h
+5
-0
cpp/src/server/DBWrapper.cpp
cpp/src/server/DBWrapper.cpp
+43
-0
cpp/src/server/DBWrapper.h
cpp/src/server/DBWrapper.h
+4
-0
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
+1
-2
cpp/src/utils/Error.h
cpp/src/utils/Error.h
+1
-2
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
f97ba056
...
...
@@ -28,6 +28,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-609 - Update task construct function
-
MS-611 - Add resources validity check in ResourceMgr
-
MS-619 - Add optimizer class in scheduler
-
MS-614 - Preload table at startup
## New Feature
...
...
cpp/conf/server_config.template
浏览文件 @
f97ba056
...
...
@@ -18,6 +18,9 @@ db_config:
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
build_index_gpu: 0 # gpu id used for building index
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config:
enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus
...
...
cpp/src/db/DBImpl.cpp
浏览文件 @
f97ba056
...
...
@@ -206,7 +206,7 @@ DBImpl::PreloadTable(const std::string& table_id) {
size
+=
engine
->
PhysicalSize
();
if
(
size
>
available_size
)
{
break
;
return
Status
(
SERVER_CACHE_FULL
,
"Cache is full"
)
;
}
else
{
try
{
// step 1: load index
...
...
@@ -639,8 +639,9 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m
ENGINE_LOG_DEBUG
<<
"Merging file "
<<
file_schema
.
file_id_
;
index_size
=
index
->
Size
();
if
(
index_size
>=
file_schema
.
index_file_size_
)
if
(
index_size
>=
file_schema
.
index_file_size_
)
{
break
;
}
}
// step 3: serialize to disk
...
...
cpp/src/server/Config.cpp
浏览文件 @
f97ba056
...
...
@@ -770,6 +770,16 @@ Config::GetDBConfigStrBuildIndexGPU() {
return
value
;
}
std
::
string
Config
::
GetDBConfigStrPreloadTable
()
{
std
::
string
value
;
if
(
!
GetConfigValueInMem
(
CONFIG_DB
,
CONFIG_DB_PRELOAD_TABLE
,
value
).
ok
())
{
value
=
GetConfigNode
(
CONFIG_DB
).
GetValue
(
CONFIG_DB_PRELOAD_TABLE
);
SetConfigValueInMem
(
CONFIG_DB
,
CONFIG_DB_PRELOAD_TABLE
,
value
);
}
return
value
;
}
////////////////////////////////////////////////////////////////////////////////
/* metric config */
std
::
string
...
...
@@ -988,6 +998,12 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) {
return
Status
::
OK
();
}
Status
Config
::
GetDBConfigPreloadTable
(
std
::
string
&
value
)
{
value
=
GetDBConfigStrPreloadTable
();
return
Status
::
OK
();
}
Status
Config
::
GetMetricConfigEnableMonitor
(
bool
&
value
)
{
std
::
string
str
=
GetMetricConfigStrEnableMonitor
();
...
...
cpp/src/server/Config.h
浏览文件 @
f97ba056
...
...
@@ -56,6 +56,7 @@ static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static
const
char
*
CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT
=
"4"
;
static
const
char
*
CONFIG_DB_BUILD_INDEX_GPU
=
"build_index_gpu"
;
static
const
char
*
CONFIG_DB_BUILD_INDEX_GPU_DEFAULT
=
"0"
;
static
const
char
*
CONFIG_DB_PRELOAD_TABLE
=
"preload_table"
;
/* cache config */
static
const
char
*
CONFIG_CACHE
=
"cache_config"
;
...
...
@@ -204,6 +205,8 @@ class Config {
GetDBConfigStrInsertBufferSize
();
std
::
string
GetDBConfigStrBuildIndexGPU
();
std
::
string
GetDBConfigStrPreloadTable
();
/* metric config */
std
::
string
...
...
@@ -261,6 +264,8 @@ class Config {
GetDBConfigInsertBufferSize
(
int32_t
&
value
);
Status
GetDBConfigBuildIndexGPU
(
int32_t
&
value
);
Status
GetDBConfigPreloadTable
(
std
::
string
&
value
);
/* metric config */
Status
...
...
cpp/src/server/DBWrapper.cpp
浏览文件 @
f97ba056
...
...
@@ -155,6 +155,20 @@ DBWrapper::StartService() {
db_
->
Start
();
// preload table
std
::
string
preload_tables
;
s
=
config
.
GetDBConfigPreloadTable
(
preload_tables
);
if
(
!
s
.
ok
())
{
return
s
;
}
s
=
PreloadTables
(
preload_tables
);
if
(
!
s
.
ok
())
{
std
::
cerr
<<
"ERROR! Failed to preload tables: "
<<
preload_tables
<<
std
::
endl
;
std
::
cerr
<<
s
.
ToString
()
<<
std
::
endl
;
kill
(
0
,
SIGUSR1
);
}
return
Status
::
OK
();
}
...
...
@@ -167,5 +181,34 @@ DBWrapper::StopService() {
return
Status
::
OK
();
}
Status
DBWrapper
::
PreloadTables
(
const
std
::
string
&
preload_tables
)
{
if
(
preload_tables
.
empty
())
{
//do nothing
}
else
if
(
preload_tables
==
"*"
)
{
//load all tables
std
::
vector
<
engine
::
meta
::
TableSchema
>
table_schema_array
;
db_
->
AllTables
(
table_schema_array
);
for
(
auto
&
schema
:
table_schema_array
)
{
auto
status
=
db_
->
PreloadTable
(
schema
.
table_id_
);
if
(
!
status
.
ok
())
{
return
status
;
}
}
}
else
{
std
::
vector
<
std
::
string
>
table_names
;
StringHelpFunctions
::
SplitStringByDelimeter
(
preload_tables
,
","
,
table_names
);
for
(
auto
&
name
:
table_names
)
{
auto
status
=
db_
->
PreloadTable
(
name
);
if
(
!
status
.
ok
())
{
return
status
;
}
}
}
return
Status
::
OK
();
}
}
// namespace server
}
// namespace milvus
cpp/src/server/DBWrapper.h
浏览文件 @
f97ba056
...
...
@@ -52,6 +52,10 @@ class DBWrapper {
return
db_
;
}
private:
Status
PreloadTables
(
const
std
::
string
&
preload_tables
);
private:
engine
::
DBPtr
db_
;
};
...
...
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
浏览文件 @
f97ba056
...
...
@@ -36,7 +36,6 @@ ErrorMap(ErrorCode code) {
{
SERVER_INVALID_ARGUMENT
,
::
milvus
::
grpc
::
ErrorCode
::
ILLEGAL_ARGUMENT
},
{
SERVER_FILE_NOT_FOUND
,
::
milvus
::
grpc
::
ErrorCode
::
FILE_NOT_FOUND
},
{
SERVER_NOT_IMPLEMENT
,
::
milvus
::
grpc
::
ErrorCode
::
UNEXPECTED_ERROR
},
{
SERVER_BLOCKING_QUEUE_EMPTY
,
::
milvus
::
grpc
::
ErrorCode
::
UNEXPECTED_ERROR
},
{
SERVER_CANNOT_CREATE_FOLDER
,
::
milvus
::
grpc
::
ErrorCode
::
CANNOT_CREATE_FOLDER
},
{
SERVER_CANNOT_CREATE_FILE
,
::
milvus
::
grpc
::
ErrorCode
::
CANNOT_CREATE_FILE
},
{
SERVER_CANNOT_DELETE_FOLDER
,
::
milvus
::
grpc
::
ErrorCode
::
CANNOT_DELETE_FOLDER
},
...
...
@@ -57,7 +56,7 @@ ErrorMap(ErrorCode code) {
{
SERVER_INVALID_INDEX_FILE_SIZE
,
::
milvus
::
grpc
::
ErrorCode
::
ILLEGAL_ARGUMENT
},
{
SERVER_ILLEGAL_VECTOR_ID
,
::
milvus
::
grpc
::
ErrorCode
::
ILLEGAL_VECTOR_ID
},
{
SERVER_ILLEGAL_SEARCH_RESULT
,
::
milvus
::
grpc
::
ErrorCode
::
ILLEGAL_SEARCH_RESULT
},
{
SERVER_CACHE_
ERROR
,
::
milvus
::
grpc
::
ErrorCode
::
CACHE_FAILED
},
{
SERVER_CACHE_
FULL
,
::
milvus
::
grpc
::
ErrorCode
::
CACHE_FAILED
},
{
DB_META_TRANSACTION_FAILED
,
::
milvus
::
grpc
::
ErrorCode
::
META_FAILED
},
{
SERVER_BUILD_INDEX_ERROR
,
::
milvus
::
grpc
::
ErrorCode
::
BUILD_INDEX_ERROR
},
{
SERVER_OUT_OF_MEMORY
,
::
milvus
::
grpc
::
ErrorCode
::
OUT_OF_MEMORY
},
...
...
cpp/src/utils/Error.h
浏览文件 @
f97ba056
...
...
@@ -56,7 +56,6 @@ constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3);
constexpr
ErrorCode
SERVER_INVALID_ARGUMENT
=
ToServerErrorCode
(
4
);
constexpr
ErrorCode
SERVER_FILE_NOT_FOUND
=
ToServerErrorCode
(
5
);
constexpr
ErrorCode
SERVER_NOT_IMPLEMENT
=
ToServerErrorCode
(
6
);
constexpr
ErrorCode
SERVER_BLOCKING_QUEUE_EMPTY
=
ToServerErrorCode
(
7
);
constexpr
ErrorCode
SERVER_CANNOT_CREATE_FOLDER
=
ToServerErrorCode
(
8
);
constexpr
ErrorCode
SERVER_CANNOT_CREATE_FILE
=
ToServerErrorCode
(
9
);
constexpr
ErrorCode
SERVER_CANNOT_DELETE_FOLDER
=
ToServerErrorCode
(
10
);
...
...
@@ -74,7 +73,7 @@ constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107);
constexpr
ErrorCode
SERVER_INVALID_TOPK
=
ToServerErrorCode
(
108
);
constexpr
ErrorCode
SERVER_ILLEGAL_VECTOR_ID
=
ToServerErrorCode
(
109
);
constexpr
ErrorCode
SERVER_ILLEGAL_SEARCH_RESULT
=
ToServerErrorCode
(
110
);
constexpr
ErrorCode
SERVER_CACHE_
ERROR
=
ToServerErrorCode
(
111
);
constexpr
ErrorCode
SERVER_CACHE_
FULL
=
ToServerErrorCode
(
111
);
constexpr
ErrorCode
SERVER_WRITE_ERROR
=
ToServerErrorCode
(
112
);
constexpr
ErrorCode
SERVER_INVALID_NPROBE
=
ToServerErrorCode
(
113
);
constexpr
ErrorCode
SERVER_INVALID_INDEX_NLIST
=
ToServerErrorCode
(
114
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录