Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
45a777b4
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
45a777b4
编写于
8月 22, 2019
作者:
S
starlord
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
merge 0.3.1
Former-commit-id: 6fa5ce5365752369865360d715659c56fc367f89
上级
223cf1f5
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
159 addition
and
22 deletion
+159
-22
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/db/Utils.cpp
cpp/src/db/Utils.cpp
+14
-10
cpp/src/db/Utils.h
cpp/src/db/Utils.h
+1
-1
cpp/src/db/meta/MySQLMetaImpl.cpp
cpp/src/db/meta/MySQLMetaImpl.cpp
+40
-7
cpp/src/db/meta/SqliteMetaImpl.cpp
cpp/src/db/meta/SqliteMetaImpl.cpp
+23
-1
cpp/unittest/db/db_tests.cpp
cpp/unittest/db/db_tests.cpp
+7
-0
cpp/unittest/db/mem_test.cpp
cpp/unittest/db/mem_test.cpp
+38
-1
cpp/unittest/db/mysql_meta_test.cpp
cpp/unittest/db/mysql_meta_test.cpp
+7
-0
cpp/unittest/db/scheduler_test.cpp
cpp/unittest/db/scheduler_test.cpp
+1
-0
cpp/unittest/server/config_test.cpp
cpp/unittest/server/config_test.cpp
+3
-0
cpp/unittest/server/util_test.cpp
cpp/unittest/server/util_test.cpp
+24
-2
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
45a777b4
...
...
@@ -61,6 +61,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-257 - Update bzip2 download url
-
MS-288 - Update compile scripts
-
MS-330 - Stability test failed caused by server core dumped
-
MS-347 - Build index hangs again
## Improvement
-
MS-156 - Add unittest for merge result functions
...
...
cpp/src/db/Utils.cpp
浏览文件 @
45a777b4
...
...
@@ -85,16 +85,20 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return
Status
::
OK
();
}
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
)
{
std
::
string
db_path
=
options
.
path
;
std
::
string
table_path
=
db_path
+
TABLES_FOLDER
+
table_id
;
boost
::
filesystem
::
remove_all
(
table_path
);
ENGINE_LOG_DEBUG
<<
"Remove table folder: "
<<
table_path
;
for
(
auto
&
path
:
options
.
slave_paths
)
{
table_path
=
path
+
TABLES_FOLDER
+
table_id
;
boost
::
filesystem
::
remove_all
(
table_path
);
ENGINE_LOG_DEBUG
<<
"Remove table folder: "
<<
table_path
;
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
,
bool
force
)
{
std
::
vector
<
std
::
string
>
paths
=
options
.
slave_paths
;
paths
.
push_back
(
options
.
path
);
for
(
auto
&
path
:
paths
)
{
std
::
string
table_path
=
path
+
TABLES_FOLDER
+
table_id
;
if
(
force
)
{
boost
::
filesystem
::
remove_all
(
table_path
);
ENGINE_LOG_DEBUG
<<
"Remove table folder: "
<<
table_path
;
}
else
if
(
boost
::
filesystem
::
exists
(
table_path
)
&&
boost
::
filesystem
::
is_empty
(
table_path
))
{
boost
::
filesystem
::
remove_all
(
table_path
);
ENGINE_LOG_DEBUG
<<
"Remove table folder: "
<<
table_path
;
}
}
return
Status
::
OK
();
...
...
cpp/src/db/Utils.h
浏览文件 @
45a777b4
...
...
@@ -19,7 +19,7 @@ namespace utils {
long
GetMicroSecTimeStamp
();
Status
CreateTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
);
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
);
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
,
bool
force
=
true
);
Status
CreateTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
Status
GetTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
...
...
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
45a777b4
...
...
@@ -1652,15 +1652,14 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
}
Status
MySQLMetaImpl
::
CleanUpFilesWithTTL
(
uint16_t
seconds
)
{
auto
now
=
utils
::
GetMicroSecTimeStamp
();
std
::
set
<
std
::
string
>
table_ids
;
//remove to_delete files
try
{
MetricCollector
metric
;
{
ScopedConnection
connectionPtr
(
*
mysql_connection_pool_
,
safe_grab
);
if
(
connectionPtr
==
nullptr
)
{
...
...
@@ -1700,6 +1699,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
ENGINE_LOG_DEBUG
<<
"Removing file id:"
<<
table_file
.
id_
<<
" location:"
<<
table_file
.
location_
;
idsToDelete
.
emplace_back
(
std
::
to_string
(
table_file
.
id_
));
table_ids
.
insert
(
table_file
.
table_id_
);
}
if
(
!
idsToDelete
.
empty
())
{
...
...
@@ -1734,12 +1735,11 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return
Status
::
DBTransactionError
(
"GENERAL ERROR WHEN CLEANING UP FILES WITH TTL"
,
er
.
what
());
}
//remove to_delete tables
try
{
MetricCollector
metric
;
{
ScopedConnection
connectionPtr
(
*
mysql_connection_pool_
,
safe_grab
);
if
(
connectionPtr
==
nullptr
)
{
...
...
@@ -1765,7 +1765,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
std
::
string
table_id
;
resRow
[
"table_id"
].
to_string
(
table_id
);
utils
::
DeleteTablePath
(
options_
,
table_id
);
utils
::
DeleteTablePath
(
options_
,
table_id
,
false
);
//only delete empty folder
idsToDeleteSS
<<
"id = "
<<
std
::
to_string
(
id
)
<<
" OR "
;
}
...
...
@@ -1794,6 +1794,39 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return
Status
::
DBTransactionError
(
"GENERAL ERROR WHEN CLEANING UP FILES WITH TTL"
,
er
.
what
());
}
//remove deleted table folder
//don't remove table folder until all its files has been deleted
try
{
MetricCollector
metric
;
{
ScopedConnection
connectionPtr
(
*
mysql_connection_pool_
,
safe_grab
);
if
(
connectionPtr
==
nullptr
)
{
return
Status
::
Error
(
"Failed to connect to database server"
);
}
for
(
auto
&
table_id
:
table_ids
)
{
Query
cleanUpFilesWithTTLQuery
=
connectionPtr
->
query
();
cleanUpFilesWithTTLQuery
<<
"SELECT file_id "
<<
"FROM TableFiles "
<<
"WHERE table_id = "
<<
table_id
<<
";"
;
ENGINE_LOG_DEBUG
<<
"MySQLMetaImpl::CleanUpFilesWithTTL: "
<<
cleanUpFilesWithTTLQuery
.
str
();
StoreQueryResult
res
=
cleanUpFilesWithTTLQuery
.
store
();
if
(
res
.
empty
())
{
utils
::
DeleteTablePath
(
options_
,
table_id
);
}
}
}
}
catch
(
const
Exception
&
er
)
{
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR
<<
"GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL"
<<
": "
<<
er
.
what
();
return
Status
::
DBTransactionError
(
"GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL"
,
er
.
what
());
}
return
Status
::
OK
();
}
...
...
cpp/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
45a777b4
...
...
@@ -1069,6 +1069,9 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
Status
SqliteMetaImpl
::
CleanUpFilesWithTTL
(
uint16_t
seconds
)
{
auto
now
=
utils
::
GetMicroSecTimeStamp
();
std
::
set
<
std
::
string
>
table_ids
;
//remove to_delete files
try
{
MetricCollector
metric
;
...
...
@@ -1098,6 +1101,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
ENGINE_LOG_DEBUG
<<
"Removing file id:"
<<
table_file
.
file_id_
<<
" location:"
<<
table_file
.
location_
;
ConnectorPtr
->
remove
<
TableFileSchema
>
(
table_file
.
id_
);
table_ids
.
insert
(
table_file
.
table_id_
);
}
return
true
;
});
...
...
@@ -1111,6 +1115,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return
HandleException
(
"Encounter exception when clean table files"
,
e
);
}
//remove to_delete tables
try
{
MetricCollector
metric
;
...
...
@@ -1123,7 +1128,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto
commited
=
ConnectorPtr
->
transaction
([
&
]()
mutable
{
for
(
auto
&
table
:
tables
)
{
utils
::
DeleteTablePath
(
options_
,
std
::
get
<
1
>
(
table
)
);
utils
::
DeleteTablePath
(
options_
,
std
::
get
<
1
>
(
table
)
,
false
);
//only delete empty folder
ConnectorPtr
->
remove
<
TableSchema
>
(
std
::
get
<
0
>
(
table
));
}
...
...
@@ -1139,6 +1144,23 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return
HandleException
(
"Encounter exception when clean table files"
,
e
);
}
//remove deleted table folder
//don't remove table folder until all its files has been deleted
try
{
MetricCollector
metric
;
for
(
auto
&
table_id
:
table_ids
)
{
auto
selected
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
file_id_
),
where
(
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
));
if
(
selected
.
size
()
==
0
)
{
utils
::
DeleteTablePath
(
options_
,
table_id
);
}
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table folder"
,
e
);
}
return
Status
::
OK
();
}
...
...
cpp/unittest/db/db_tests.cpp
浏览文件 @
45a777b4
...
...
@@ -93,6 +93,8 @@ TEST_F(DBTest, CONFIG_TEST) {
TEST_F
(
DBTest
,
DB_TEST
)
{
db_
->
Open
(
GetOptions
(),
&
db_
);
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
...
...
@@ -161,6 +163,11 @@ TEST_F(DBTest, DB_TEST) {
}
search
.
join
();
uint64_t
count
;
stat
=
db_
->
GetTableRowCount
(
TABLE_NAME
,
count
);
ASSERT_STATS
(
stat
);
ASSERT_TRUE
(
count
>
0
);
};
TEST_F
(
DBTest
,
SEARCH_TEST
)
{
...
...
cpp/unittest/db/mem_test.cpp
浏览文件 @
45a777b4
...
...
@@ -445,4 +445,41 @@ TEST_F(DBTest, VECTOR_IDS_TEST)
for
(
auto
i
=
0
;
i
<
nb
;
i
++
)
{
ASSERT_EQ
(
vector_ids
[
i
],
i
+
nb
);
}
}
\ No newline at end of file
}
TEST_F
(
NewMemManagerTest
,
MEMMANAGER_TEST
)
{
int
setenv_res
=
setenv
(
"MILVUS_USE_OLD_MEM_MANAGER"
,
"ON"
,
1
);
ASSERT_TRUE
(
setenv_res
==
0
);
auto
options
=
engine
::
OptionsFactory
::
Build
();
options
.
meta
.
path
=
"/tmp/milvus_test"
;
options
.
meta
.
backend_uri
=
"sqlite://:@:/"
;
auto
db_
=
engine
::
DBFactory
::
Build
(
options
);
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
engine
::
meta
::
TableSchema
table_info_get
;
table_info_get
.
table_id_
=
TABLE_NAME
;
stat
=
db_
->
DescribeTable
(
table_info_get
);
ASSERT_STATS
(
stat
);
ASSERT_EQ
(
table_info_get
.
dimension_
,
TABLE_DIM
);
auto
start_time
=
METRICS_NOW_TIME
;
int
insert_loop
=
20
;
for
(
int
i
=
0
;
i
<
insert_loop
;
++
i
)
{
int64_t
nb
=
40960
;
std
::
vector
<
float
>
xb
;
BuildVectors
(
nb
,
xb
);
engine
::
IDNumbers
vector_ids
;
engine
::
Status
status
=
db_
->
InsertVectors
(
TABLE_NAME
,
nb
,
xb
.
data
(),
vector_ids
);
ASSERT_TRUE
(
status
.
ok
());
}
auto
end_time
=
METRICS_NOW_TIME
;
auto
total_time
=
METRICS_MICROSECONDS
(
start_time
,
end_time
);
LOG
(
DEBUG
)
<<
"total_time spent in INSERT_TEST (ms) : "
<<
total_time
;
delete
db_
;
boost
::
filesystem
::
remove_all
(
options
.
meta
.
path
);
}
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @
45a777b4
...
...
@@ -194,6 +194,13 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) {
i
++
;
}
bool
has
;
status
=
impl
.
HasNonIndexFiles
(
table_id
,
has
);
ASSERT_TRUE
(
status
.
ok
());
status
=
impl
.
UpdateTableFilesToIndex
(
table_id
);
ASSERT_TRUE
(
status
.
ok
());
status
=
impl
.
DropAll
();
ASSERT_TRUE
(
status
.
ok
());
}
...
...
cpp/unittest/db/scheduler_test.cpp
浏览文件 @
45a777b4
...
...
@@ -56,6 +56,7 @@ TEST(DBSchedulerTest, TASK_QUEUE_TEST) {
ptr
=
queue
.
Back
();
ASSERT_EQ
(
ptr
->
type
(),
engine
::
ScheduleTaskType
::
kIndexLoad
);
load_task
->
Execute
();
}
TEST
(
DBSchedulerTest
,
SEARCH_SCHEDULER_TEST
)
{
...
...
cpp/unittest/server/config_test.cpp
浏览文件 @
45a777b4
...
...
@@ -87,6 +87,9 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
server
::
ServerError
err
=
config
.
LoadConfigFile
(
CONFIG_FILE_PATH
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
err
=
server
::
ServerConfig
::
GetInstance
().
ValidateConfig
();
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
server
::
ConfigNode
node1
=
config
.
GetConfig
(
"server_config"
);
server
::
ConfigNode
&
node2
=
config
.
GetConfig
(
"cache_config"
);
node1
.
Combine
(
node2
);
...
...
cpp/unittest/server/util_test.cpp
浏览文件 @
45a777b4
...
...
@@ -178,12 +178,15 @@ TEST(UtilTest, VALIDATE_TABLENAME_TEST) {
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
table_name
=
"
中文
"
;
table_name
=
"
_!@#!@
"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
table_name
=
"中文"
;
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
table_name
=
std
::
string
(
'a'
,
32768
);
table_name
=
std
::
string
(
10000
,
'a'
);
res
=
server
::
ValidationUtil
::
ValidateTableName
(
table_name
);
ASSERT_EQ
(
res
,
server
::
SERVER_INVALID_TABLE_NAME
);
}
...
...
@@ -204,3 +207,22 @@ TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) {
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateTableIndexType
((
int
)
engine
::
EngineType
::
MAX_VALUE
+
1
),
server
::
SERVER_INVALID_INDEX_TYPE
);
}
TEST
(
ValidationUtilTest
,
ValidateGpuTest
)
{
ASSERT_EQ
(
server
::
ValidationUtil
::
ValidateGpuIndex
(
0
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
ValidateGpuIndex
(
100
),
server
::
SERVER_SUCCESS
);
size_t
memory
=
0
;
ASSERT_EQ
(
server
::
ValidationUtil
::
GetGpuMemory
(
0
,
memory
),
server
::
SERVER_SUCCESS
);
ASSERT_NE
(
server
::
ValidationUtil
::
GetGpuMemory
(
100
,
memory
),
server
::
SERVER_SUCCESS
);
}
TEST
(
UtilTest
,
TIMERECORDER_TEST
)
{
for
(
int64_t
log_level
=
0
;
log_level
<=
6
;
log_level
++
)
{
if
(
log_level
==
5
)
{
continue
;
//skip fatal
}
server
::
TimeRecorder
rc
(
"time"
,
log_level
);
rc
.
RecordSection
(
"end"
);
}
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录