Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
a2a9e16d
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a2a9e16d
编写于
9月 02, 2019
作者:
Y
Yu Kun
浏览文件
操作
浏览文件
下载
差异文件
merge conflict
Former-commit-id: a872834e19500241b2f7465959c71f03cd61ab36
上级
2ae5ed07
1584b048
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
47 addition
and
69 deletion
+47
-69
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+14
-2
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+3
-1
cpp/src/db/meta/MySQLMetaImpl.cpp
cpp/src/db/meta/MySQLMetaImpl.cpp
+11
-5
cpp/unittest/db/db_tests.cpp
cpp/unittest/db/db_tests.cpp
+0
-1
cpp/unittest/db/mysql_db_test.cpp
cpp/unittest/db/mysql_db_test.cpp
+6
-47
cpp/unittest/db/mysql_meta_test.cpp
cpp/unittest/db/mysql_meta_test.cpp
+7
-7
cpp/unittest/db/utils.cpp
cpp/unittest/db/utils.cpp
+3
-4
cpp/unittest/db/utils.h
cpp/unittest/db/utils.h
+2
-2
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
a2a9e16d
...
...
@@ -19,6 +19,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-436 - Delete vectors failed if index created with index_type: IVF_FLAT/IVF_SQ8
-
MS-450 - server hang after run stop_server.sh
-
MS-449 - Add vectors twice success, once with ids, the other no ids
-
MS-461 - Mysql meta unittest failed
-
MS-462 - Run milvus server twices, should display error
## Improvement
...
...
cpp/src/db/DBImpl.cpp
浏览文件 @
a2a9e16d
...
...
@@ -645,14 +645,18 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
try
{
//step 1: load index
to_index
->
Load
(
options_
.
insert_cache_immediately_
);
Status
status
=
to_index
->
Load
(
options_
.
insert_cache_immediately_
);
if
(
!
status
.
ok
())
{
ENGINE_LOG_ERROR
<<
"Failed to load index file: "
<<
status
.
ToString
();
return
status
;
}
//step 2: create table file
meta
::
TableFileSchema
table_file
;
table_file
.
table_id_
=
file
.
table_id_
;
table_file
.
date_
=
file
.
date_
;
table_file
.
file_type_
=
meta
::
TableFileSchema
::
NEW_INDEX
;
//for multi-db-path, distribute index file averagely to each path
Status
status
=
meta_ptr_
->
CreateTableFile
(
table_file
);
status
=
meta_ptr_
->
CreateTableFile
(
table_file
);
if
(
!
status
.
ok
())
{
ENGINE_LOG_ERROR
<<
"Failed to create table file: "
<<
status
.
ToString
();
return
status
;
...
...
@@ -664,6 +668,14 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
try
{
server
::
CollectBuildIndexMetrics
metrics
;
index
=
to_index
->
BuildIndex
(
table_file
.
location_
,
(
EngineType
)
table_file
.
engine_type_
);
if
(
index
==
nullptr
)
{
table_file
.
file_type_
=
meta
::
TableFileSchema
::
TO_DELETE
;
status
=
meta_ptr_
->
UpdateTableFile
(
table_file
);
ENGINE_LOG_DEBUG
<<
"Failed to update file to index, mark file: "
<<
table_file
.
file_id_
<<
" to to_delete"
;
return
status
;
}
}
catch
(
std
::
exception
&
ex
)
{
//typical error: out of gpu memory
std
::
string
msg
=
"BuildIndex encounter exception: "
+
std
::
string
(
ex
.
what
());
...
...
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
a2a9e16d
...
...
@@ -132,7 +132,9 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
server
::
CollectExecutionEngineMetrics
metrics
(
physical_size
);
index_
=
read_index
(
location_
);
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Failed to load index from "
<<
location_
;
std
::
string
msg
=
"Failed to load index from "
+
location_
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
::
Error
(
msg
);
}
else
{
ENGINE_LOG_DEBUG
<<
"Disk io from: "
<<
location_
;
}
...
...
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
a2a9e16d
...
...
@@ -144,7 +144,7 @@ Status MySQLMetaImpl::Initialize() {
"dimension SMALLINT NOT NULL, "
<<
"created_on BIGINT NOT NULL, "
<<
"flag BIGINT DEFAULT 0 NOT NULL, "
<<
"index_file_size INT DEFAULT 1024 NOT NULL, "
<<
"index_file_size
BIG
INT DEFAULT 1024 NOT NULL, "
<<
"engine_type INT DEFAULT 1 NOT NULL, "
<<
"nlist INT DEFAULT 16384 NOT NULL, "
<<
"metric_type INT DEFAULT 1 NOT NULL);"
;
...
...
@@ -291,11 +291,16 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) {
std
::
string
state
=
std
::
to_string
(
table_schema
.
state_
);
std
::
string
dimension
=
std
::
to_string
(
table_schema
.
dimension_
);
std
::
string
created_on
=
std
::
to_string
(
table_schema
.
created_on_
);
std
::
string
flag
=
std
::
to_string
(
table_schema
.
flag_
);
std
::
string
index_file_size
=
std
::
to_string
(
table_schema
.
index_file_size_
);
std
::
string
engine_type
=
std
::
to_string
(
table_schema
.
engine_type_
);
std
::
string
nlist
=
std
::
to_string
(
table_schema
.
nlist_
);
std
::
string
metric_type
=
std
::
to_string
(
table_schema
.
metric_type_
);
createTableQuery
<<
"INSERT INTO Tables VALUES"
<<
"("
<<
id
<<
", "
<<
quote
<<
table_id
<<
", "
<<
state
<<
", "
<<
dimension
<<
", "
<<
created_on
<<
", "
<<
engine_type
<<
");"
;
created_on
<<
", "
<<
flag
<<
", "
<<
index_file_size
<<
", "
<<
engine_type
<<
", "
<<
nlist
<<
", "
<<
metric_type
<<
");"
;
ENGINE_LOG_DEBUG
<<
"MySQLMetaImpl::CreateTable: "
<<
createTableQuery
.
str
();
...
...
@@ -904,6 +909,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
std
::
string
engine_type
=
std
::
to_string
(
file_schema
.
engine_type_
);
std
::
string
file_id
=
file_schema
.
file_id_
;
std
::
string
file_type
=
std
::
to_string
(
file_schema
.
file_type_
);
std
::
string
file_size
=
std
::
to_string
(
file_schema
.
file_size_
);
std
::
string
row_count
=
std
::
to_string
(
file_schema
.
row_count_
);
std
::
string
updated_time
=
std
::
to_string
(
file_schema
.
updated_time_
);
std
::
string
created_on
=
std
::
to_string
(
file_schema
.
created_on_
);
...
...
@@ -920,8 +926,8 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
createTableFileQuery
<<
"INSERT INTO TableFiles VALUES"
<<
"("
<<
id
<<
", "
<<
quote
<<
table_id
<<
", "
<<
engine_type
<<
", "
<<
quote
<<
file_id
<<
", "
<<
file_type
<<
", "
<<
row_count
<<
", "
<<
updated_time
<<
", "
<<
created_on
<<
", "
<<
date
<<
");"
;
quote
<<
file_id
<<
", "
<<
file_type
<<
", "
<<
file_size
<<
", "
<<
row_count
<<
", "
<<
updated_time
<<
", "
<<
created_on
<<
", "
<<
date
<<
");"
;
ENGINE_LOG_DEBUG
<<
"MySQLMetaImpl::CreateTableFile: "
<<
createTableFileQuery
.
str
();
...
...
@@ -1170,7 +1176,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
}
Query
filesToMergeQuery
=
connectionPtr
->
query
();
filesToMergeQuery
<<
"SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, create_on "
<<
filesToMergeQuery
<<
"SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, create
d
_on "
<<
"FROM TableFiles "
<<
"WHERE table_id = "
<<
quote
<<
table_id
<<
" AND "
<<
"file_type = "
<<
std
::
to_string
(
TableFileSchema
::
RAW
)
<<
" "
<<
...
...
cpp/unittest/db/db_tests.cpp
浏览文件 @
a2a9e16d
...
...
@@ -141,7 +141,6 @@ TEST_F(DBTest, CONFIG_TEST) {
TEST_F
(
DBTest
,
DB_TEST
)
{
db_
->
Open
(
GetOptions
(),
&
db_
);
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
...
...
cpp/unittest/db/mysql_db_test.cpp
浏览文件 @
a2a9e16d
...
...
@@ -46,11 +46,7 @@ namespace {
}
TEST_F
(
DISABLED_MySQLDBTest
,
DB_TEST
)
{
auto
options
=
GetOptions
();
auto
db_
=
engine
::
DBFactory
::
Build
(
options
);
TEST_F
(
MySQLDBTest
,
DB_TEST
)
{
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
...
...
@@ -115,6 +111,8 @@ TEST_F(DISABLED_MySQLDBTest, DB_TEST) {
ASSERT_TRUE
(
count
>=
prev_count
);
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
seconds
(
3
));
}
std
::
cout
<<
"Search AAA done"
<<
std
::
endl
;
});
int
loop
=
INSERT_LOOP
;
...
...
@@ -131,18 +129,9 @@ TEST_F(DISABLED_MySQLDBTest, DB_TEST) {
}
search
.
join
();
delete
db_
;
auto
dummyDB
=
engine
::
DBFactory
::
Build
(
options
);
dummyDB
->
DropAll
();
delete
dummyDB
;
};
TEST_F
(
DISABLED_MySQLDBTest
,
SEARCH_TEST
)
{
auto
options
=
GetOptions
();
auto
db_
=
engine
::
DBFactory
::
Build
(
options
);
TEST_F
(
MySQLDBTest
,
SEARCH_TEST
)
{
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
...
...
@@ -192,22 +181,9 @@ TEST_F(DISABLED_MySQLDBTest, SEARCH_TEST) {
engine
::
QueryResults
results
;
stat
=
db_
->
Query
(
TABLE_NAME
,
k
,
nq
,
10
,
xq
.
data
(),
results
);
ASSERT_STATS
(
stat
);
delete
db_
;
auto
dummyDB
=
engine
::
DBFactory
::
Build
(
options
);
dummyDB
->
DropAll
();
delete
dummyDB
;
// TODO(linxj): add groundTruth assert
};
TEST_F
(
DISABLED_MySQLDBTest
,
ARHIVE_DISK_CHECK
)
{
auto
options
=
GetOptions
();
options
.
meta
.
archive_conf
=
engine
::
ArchiveConf
(
"delete"
,
"disk:1"
);
auto
db_
=
engine
::
DBFactory
::
Build
(
options
);
TEST_F
(
MySQLDBTest
,
ARHIVE_DISK_CHECK
)
{
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
...
...
@@ -250,20 +226,9 @@ TEST_F(DISABLED_MySQLDBTest, ARHIVE_DISK_CHECK) {
db_
->
Size
(
size
);
LOG
(
DEBUG
)
<<
"size="
<<
size
;
ASSERT_LE
(
size
,
1
*
engine
::
meta
::
G
);
delete
db_
;
auto
dummyDB
=
engine
::
DBFactory
::
Build
(
options
);
dummyDB
->
DropAll
();
delete
dummyDB
;
};
TEST_F
(
DISABLED_MySQLDBTest
,
DELETE_TEST
)
{
auto
options
=
GetOptions
();
options
.
meta
.
archive_conf
=
engine
::
ArchiveConf
(
"delete"
,
"disk:1"
);
auto
db_
=
engine
::
DBFactory
::
Build
(
options
);
TEST_F
(
MySQLDBTest
,
DELETE_TEST
)
{
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
// std::cout << stat.ToString() << std::endl;
...
...
@@ -301,10 +266,4 @@ TEST_F(DISABLED_MySQLDBTest, DELETE_TEST) {
db_
->
HasTable
(
TABLE_NAME
,
has_table
);
ASSERT_FALSE
(
has_table
);
delete
db_
;
auto
dummyDB
=
engine
::
DBFactory
::
Build
(
options
);
dummyDB
->
DropAll
();
delete
dummyDB
;
};
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @
a2a9e16d
...
...
@@ -21,7 +21,7 @@
using
namespace
zilliz
::
milvus
::
engine
;
TEST_F
(
DISABLED_
MySQLTest
,
TABLE_TEST
)
{
TEST_F
(
MySQLTest
,
TABLE_TEST
)
{
DBMetaOptions
options
;
try
{
options
=
getDBMetaOptions
();
...
...
@@ -53,7 +53,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_TEST) {
table
.
table_id_
=
table_id
;
status
=
impl
.
CreateTable
(
table
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_TRUE
(
status
.
IsAlreadyExist
());
table
.
table_id_
=
""
;
status
=
impl
.
CreateTable
(
table
);
...
...
@@ -63,7 +63,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_TEST) {
ASSERT_TRUE
(
status
.
ok
());
}
TEST_F
(
DISABLED_
MySQLTest
,
TABLE_FILE_TEST
)
{
TEST_F
(
MySQLTest
,
TABLE_FILE_TEST
)
{
DBMetaOptions
options
;
try
{
options
=
getDBMetaOptions
();
...
...
@@ -92,7 +92,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) {
meta
::
DatesT
dates
;
dates
.
push_back
(
utils
::
GetDate
());
status
=
impl
.
DropPartitionsByDates
(
table_file
.
table_id_
,
dates
);
ASSERT_
FALS
E
(
status
.
ok
());
ASSERT_
TRU
E
(
status
.
ok
());
uint64_t
cnt
=
0
;
status
=
impl
.
Count
(
table_id
,
cnt
);
...
...
@@ -139,7 +139,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) {
ASSERT_TRUE
(
status
.
ok
());
}
TEST_F
(
DISABLED_
MySQLTest
,
ARCHIVE_TEST_DAYS
)
{
TEST_F
(
MySQLTest
,
ARCHIVE_TEST_DAYS
)
{
srand
(
time
(
0
));
DBMetaOptions
options
;
try
{
...
...
@@ -211,7 +211,7 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) {
ASSERT_TRUE
(
status
.
ok
());
}
TEST_F
(
DISABLED_
MySQLTest
,
ARCHIVE_TEST_DISK
)
{
TEST_F
(
MySQLTest
,
ARCHIVE_TEST_DISK
)
{
DBMetaOptions
options
;
try
{
options
=
getDBMetaOptions
();
...
...
@@ -269,7 +269,7 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DISK) {
ASSERT_TRUE
(
status
.
ok
());
}
TEST_F
(
DISABLED_
MySQLTest
,
TABLE_FILES_TEST
)
{
TEST_F
(
MySQLTest
,
TABLE_FILES_TEST
)
{
DBMetaOptions
options
;
try
{
options
=
getDBMetaOptions
();
...
...
cpp/unittest/db/utils.cpp
浏览文件 @
a2a9e16d
...
...
@@ -103,7 +103,7 @@ void MetaTest::TearDown() {
impl_
->
DropAll
();
}
zilliz
::
milvus
::
engine
::
DBMetaOptions
DISABLED_
MySQLTest
::
getDBMetaOptions
()
{
zilliz
::
milvus
::
engine
::
DBMetaOptions
MySQLTest
::
getDBMetaOptions
()
{
// std::string path = "/tmp/milvus_test";
// engine::DBMetaOptions options = engine::DBMetaOptionsFactory::Build(path);
zilliz
::
milvus
::
engine
::
DBMetaOptions
options
;
...
...
@@ -111,17 +111,16 @@ zilliz::milvus::engine::DBMetaOptions DISABLED_MySQLTest::getDBMetaOptions() {
options
.
backend_uri
=
DBTestEnvironment
::
getURI
();
if
(
options
.
backend_uri
.
empty
())
{
// throw std::exception();
options
.
backend_uri
=
"mysql://root:Fantast1c@192.168.1.194:3306/"
;
}
return
options
;
}
zilliz
::
milvus
::
engine
::
Options
DISABLED_
MySQLDBTest
::
GetOptions
()
{
zilliz
::
milvus
::
engine
::
Options
MySQLDBTest
::
GetOptions
()
{
auto
options
=
engine
::
OptionsFactory
::
Build
();
options
.
meta
.
path
=
"/tmp/milvus_test"
;
options
.
meta
.
backend_uri
=
DBTestEnvironment
::
getURI
()
;
options
.
meta
.
backend_uri
=
"mysql://root:Fantast1c@192.168.1.194:3306/"
;
return
options
;
}
...
...
cpp/unittest/db/utils.h
浏览文件 @
a2a9e16d
...
...
@@ -79,13 +79,13 @@ class MetaTest : public DBTest {
virtual
void
TearDown
()
override
;
};
class
DISABLED_
MySQLTest
:
public
::
testing
::
Test
{
class
MySQLTest
:
public
::
testing
::
Test
{
protected:
// std::shared_ptr<zilliz::milvus::engine::meta::MySQLMetaImpl> impl_;
zilliz
::
milvus
::
engine
::
DBMetaOptions
getDBMetaOptions
();
};
class
DISABLED_MySQLDBTest
:
public
::
testing
::
Test
{
class
MySQLDBTest
:
public
DB
Test
{
protected:
zilliz
::
milvus
::
engine
::
Options
GetOptions
();
};
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录