Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

从无法访问的项目Fork

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 daf9eda9 编写于 作者: P peng.xu
浏览文件
操作

Merge branch 'mysql-0.3.0' into 'branch-0.3.0' 

MS-105 - Add MySQL

See merge request megasearch/vecwise_engine!143

Former-commit-id: c202a0a82d02bff1528db05e6c1b1f9a6788b9bb
上级 ea7aa4f3 5c65f2a8
隐藏空白更改
内联 并排
Showing with 85 addition and 227 deletion
cpp/conf/server_config.template
浏览文件 @ daf9eda9
...@@ -15,6 +15,7 @@ db_config: ...@@ -15,6 +15,7 @@ db_config:
index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB
archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB
archive_days_threshold: 30 # files older than x days will be archived, unit: day archive_days_threshold: 30 # files older than x days will be archived, unit: day
sql_echo: on # print sql statement in debug log
metric_config: metric_config:
is_startup: off # if monitoring start: on, off is_startup: off # if monitoring start: on, off
......
cpp/coverage.sh
浏览文件 @ daf9eda9
#!/bin/bash #!/bin/bash
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib
LCOV_CMD="lcov" LCOV_CMD="lcov"
LCOV_GEN_CMD="genhtml" LCOV_GEN_CMD="genhtml"
...@@ -14,6 +12,26 @@ DIR_LCOV_OUTPUT="lcov_out" ...@@ -14,6 +12,26 @@ DIR_LCOV_OUTPUT="lcov_out"
DIR_GCNO="cmake_build" DIR_GCNO="cmake_build"
DIR_UNITTEST="milvus/bin" DIR_UNITTEST="milvus/bin"
MYSQL_USER_NAME=root
MYSQL_PASSWORD=Fantast1c
MYSQL_HOST='192.168.1.194'
MYSQL_PORT='3306'
MYSQL_DB_NAME=`date +%s%N`
function mysql_exc()
{
cmd=$1
mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}"
if [ $? -ne 0 ]; then
echo "mysql $cmd run failed"
fi
}
mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};"
mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';"
mysql_exc "FLUSH PRIVILEGES;"
# get baseline # get baseline
${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
...@@ -22,14 +40,13 @@ if [ $? -ne 0 ]; then ...@@ -22,14 +40,13 @@ if [ $? -ne 0 ]; then
fi fi
for test in `ls ${DIR_UNITTEST}`; do for test in `ls ${DIR_UNITTEST}`; do
echo $test
case ${test} in case ${test} in
db_test) db_test)
# set run args for db_test # set run args for db_test
args="mysql://root:Fantast1c@192.168.1.194:3306/test" args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}"
;;
*_test) *_test)
args="" args=""
;;
esac esac
# run unittest # run unittest
./${DIR_UNITTEST}/${test} "${args}" ./${DIR_UNITTEST}/${test} "${args}"
...@@ -38,6 +55,8 @@ for test in `ls ${DIR_UNITTEST}`; do ...@@ -38,6 +55,8 @@ for test in `ls ${DIR_UNITTEST}`; do
fi fi
done done
mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};"
# gen test converage # gen test converage
${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c
# merge coverage # merge coverage
...@@ -50,4 +69,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ ...@@ -50,4 +69,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
"*/cmake_build/*_ep-prefix/*" \ "*/cmake_build/*_ep-prefix/*" \
# gen html report # gen html report
${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/
\ No newline at end of file
cpp/src/db/MySQLConnectionPool.h
浏览文件 @ daf9eda9
...@@ -69,6 +69,10 @@ public: ...@@ -69,6 +69,10 @@ public:
max_idle_time_ = max_idle; max_idle_time_ = max_idle;
} }
std::string getDB() {
return db_;
}
protected: protected:
// Superclass overrides // Superclass overrides
......
cpp/src/db/MySQLMetaImpl.cpp
浏览文件 @ daf9eda9
...@@ -1733,17 +1733,30 @@ namespace meta { ...@@ -1733,17 +1733,30 @@ namespace meta {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// } // }
ENGINE_LOG_DEBUG << "Remove table file type as NEW";
Query cleanUpQuery = connectionPtr->query(); Query cleanUpQuery = connectionPtr->query();
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; cleanUpQuery << "SELECT table_name " <<
"FROM information_schema.tables " <<
"WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " <<
"AND table_name = " << quote << "TableFiles" << quote << ";";
if (options_.sql_echo) { if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
} }
if (!cleanUpQuery.exec()) { StoreQueryResult res = cleanUpQuery.store();
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; assert(res);
return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); if (!res.empty()) {
ENGINE_LOG_DEBUG << "Remove table file type as NEW";
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
}
if (!cleanUpQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
return Status::DBTransactionError("Clean up Error", cleanUpQuery.error());
}
} }
} catch (const BadQuery& er) { } catch (const BadQuery& er) {
......
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @ daf9eda9
...@@ -21,21 +21,8 @@ ...@@ -21,21 +21,8 @@
using namespace zilliz::milvus::engine; using namespace zilliz::milvus::engine;
//TEST_F(MySQLTest, InitializeTest) { TEST_F(MySQLTest, TABLE_TEST) {
// DBMetaOptions options;
// //dialect+driver://username:password@host:port/database
// options.backend_uri = "mysql://root:1234@:/test";
// meta::MySQLMetaImpl impl(options);
// auto status = impl.Initialize();
// std::cout << status.ToString() << std::endl;
// ASSERT_TRUE(status.ok());
//}
TEST_F(MySQLTest, core) {
DBMetaOptions options; DBMetaOptions options;
// //dialect+driver://username:password@host:port/database
// options.backend_uri = "mysql://root:1234@:/test";
// options.path = "/tmp/vecwise_test";
try { try {
options = getDBMetaOptions(); options = getDBMetaOptions();
} catch(std::exception& ex) { } catch(std::exception& ex) {
...@@ -45,203 +32,38 @@ TEST_F(MySQLTest, core) { ...@@ -45,203 +32,38 @@ TEST_F(MySQLTest, core) {
int mode = Options::MODE::SINGLE; int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode); meta::MySQLMetaImpl impl(options, mode);
// auto status = impl.Initialize();
// ASSERT_TRUE(status.ok());
meta::TableSchema schema1; auto table_id = "meta_test_table";
schema1.table_id_ = "test1";
schema1.dimension_ = 123;
auto status = impl.CreateTable(schema1); meta::TableSchema table;
// std::cout << status.ToString() << std::endl; table.table_id_ = table_id;
auto status = impl.CreateTable(table);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
meta::TableSchema schema2; auto gid = table.id_;
schema2.table_id_ = "test2"; table.id_ = -1;
schema2.dimension_ = 321; status = impl.DescribeTable(table);
status = impl.CreateTable(schema2);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(table.id_, gid);
ASSERT_EQ(table.table_id_, table_id);
status = impl.CreateTable(schema2); table.table_id_ = "not_found";
// std::cout << status.ToString() << std::endl; status = impl.DescribeTable(table);
// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery);
ASSERT_TRUE(status.ok());
status = impl.DeleteTable(schema2.table_id_);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
size_t id1 = schema1.id_;
long created_on1 = schema1.created_on_;
status = impl.DescribeTable(schema1);
ASSERT_TRUE(status.ok());
ASSERT_EQ(schema1.id_, id1);
ASSERT_EQ(schema1.table_id_, "test1");
ASSERT_EQ(schema1.created_on_, created_on1);
ASSERT_EQ(schema1.files_cnt_, 0);
ASSERT_EQ(schema1.engine_type_, 1);
ASSERT_EQ(schema1.store_raw_data_, false);
bool check;
status = impl.HasTable("test1", check);
ASSERT_TRUE(status.ok());
ASSERT_EQ(check, true);
std::vector<meta::TableSchema> table_schema_array;
status = impl.AllTables(table_schema_array);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_schema_array.size(), 1);
meta::TableSchema resultSchema = table_schema_array[0];
ASSERT_EQ(resultSchema.id_, id1);
ASSERT_EQ(resultSchema.table_id_, "test1");
ASSERT_EQ(resultSchema.dimension_, 123);
ASSERT_EQ(resultSchema.files_cnt_, 0);
ASSERT_EQ(resultSchema.engine_type_, 1);
ASSERT_EQ(resultSchema.store_raw_data_, false);
meta::TableFileSchema tableFileSchema;
tableFileSchema.table_id_ = "test1";
status = impl.CreateTableFile(tableFileSchema);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX;
status = impl.UpdateTableFile(tableFileSchema);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
meta::TableFilesSchema filesToIndex;
status = impl.FilesToIndex(filesToIndex);
ASSERT_TRUE(status.ok());
ASSERT_EQ(filesToIndex.size(), 1);
meta::TableFileSchema fileToIndex = filesToIndex[0];
ASSERT_EQ(fileToIndex.table_id_, "test1");
ASSERT_EQ(fileToIndex.dimension_, 123);
// meta::TableFilesSchema filesToIndex;
// status = impl.FilesToIndex(filesToIndex);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(filesToIndex.size(), 0);
meta::DatesT partition;
partition.push_back(tableFileSchema.date_);
meta::DatePartionedTableFilesSchema filesToSearch;
status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch);
ASSERT_TRUE(status.ok());
ASSERT_EQ(filesToSearch.size(), 1);
ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1);
meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0];
ASSERT_EQ(fileToSearch.table_id_, "test1");
ASSERT_EQ(fileToSearch.dimension_, 123);
tableFileSchema.file_type_ = meta::TableFileSchema::RAW;
status = impl.UpdateTableFile(tableFileSchema);
ASSERT_TRUE(status.ok());
meta::DatePartionedTableFilesSchema filesToMerge;
status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
ASSERT_EQ(filesToMerge.size(), 1);
ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1);
meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0];
ASSERT_EQ(fileToMerge.table_id_, "test1");
ASSERT_EQ(fileToMerge.dimension_, 123);
meta::TableFilesSchema resultTableFilesSchema;
std::vector<size_t> ids;
ids.push_back(tableFileSchema.id_);
status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema);
ASSERT_TRUE(status.ok());
ASSERT_EQ(resultTableFilesSchema.size(), 1);
meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0];
// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_);
ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_);
ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_);
ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_);
ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_);
ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_);
ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_);
ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_);
tableFileSchema.size_ = 234;
meta::TableSchema schema3;
schema3.table_id_ = "test3";
schema3.dimension_ = 321;
status = impl.CreateTable(schema3);
ASSERT_TRUE(status.ok());
meta::TableFileSchema tableFileSchema2;
tableFileSchema2.table_id_ = "test3";
tableFileSchema2.size_ = 345;
status = impl.CreateTableFile(tableFileSchema2);
ASSERT_TRUE(status.ok());
meta::TableFilesSchema filesToUpdate;
filesToUpdate.emplace_back(tableFileSchema);
filesToUpdate.emplace_back(tableFileSchema2);
status = impl.UpdateTableFile(tableFileSchema);
ASSERT_TRUE(status.ok());
uint64_t resultSize;
status = impl.Size(resultSize);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_);
uint64_t countResult;
status = impl.Count(tableFileSchema.table_id_, countResult);
ASSERT_TRUE(status.ok());
status = impl.DropAll();
ASSERT_TRUE(status.ok());
}
TEST_F(MySQLTest, GROUP_TEST) {
DBMetaOptions options;
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
ASSERT_TRUE(status.ok());
auto gid = group.id_;
group.id_ = -1;
status = impl.DescribeTable(group);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group.id_, gid);
ASSERT_EQ(group.table_id_, table_id);
group.table_id_ = "not_found";
status = impl.DescribeTable(group);
ASSERT_TRUE(!status.ok()); ASSERT_TRUE(!status.ok());
group.table_id_ = table_id; table.table_id_ = table_id;
status = impl.CreateTable(group); status = impl.CreateTable(table);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
group.table_id_ = ""; table.table_id_ = "";
status = impl.CreateTable(group); status = impl.CreateTable(table);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = impl.DropAll(); status = impl.DropAll();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(MySQLTest, table_file_TEST) { TEST_F(MySQLTest, TABLE_FILE_TEST) {
DBMetaOptions options; DBMetaOptions options;
try { try {
options = getDBMetaOptions(); options = getDBMetaOptions();
...@@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { ...@@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) {
int mode = Options::MODE::SINGLE; int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode); meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_group"; auto table_id = "meta_test_table";
meta::TableSchema group; meta::TableSchema table;
group.table_id_ = table_id; table.table_id_ = table_id;
group.dimension_ = 256; table.dimension_ = 256;
auto status = impl.CreateTable(group); auto status = impl.CreateTable(table);
meta::TableFileSchema table_file; meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_; table_file.table_id_ = table.table_id_;
status = impl.CreateTableFile(table_file); status = impl.CreateTableFile(table_file);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
...@@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { ...@@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
int mode = Options::MODE::SINGLE; int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode); meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_group"; auto table_id = "meta_test_table";
meta::TableSchema group; meta::TableSchema table;
group.table_id_ = table_id; table.table_id_ = table_id;
auto status = impl.CreateTable(group); auto status = impl.CreateTable(table);
meta::TableFilesSchema files; meta::TableFilesSchema files;
meta::TableFileSchema table_file; meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_; table_file.table_id_ = table.table_id_;
auto cnt = 100; auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp(); long ts = utils::GetMicroSecTimeStamp();
...@@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { ...@@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
auto impl = meta::MySQLMetaImpl(options, mode); auto impl = meta::MySQLMetaImpl(options, mode);
auto table_id = "meta_test_group"; auto table_id = "meta_test_group";
meta::TableSchema group; meta::TableSchema table;
group.table_id_ = table_id; table.table_id_ = table_id;
auto status = impl.CreateTable(group); auto status = impl.CreateTable(table);
meta::TableFilesSchema files; meta::TableFilesSchema files;
meta::TableFileSchema table_file; meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_; table_file.table_id_ = table.table_id_;
auto cnt = 10; auto cnt = 10;
auto each_size = 2UL; auto each_size = 2UL;
...@@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ...@@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group"; auto table_id = "meta_test_group";
meta::TableSchema group; meta::TableSchema table;
group.table_id_ = table_id; table.table_id_ = table_id;
auto status = impl.CreateTable(group); auto status = impl.CreateTable(table);
int new_files_cnt = 4; int new_files_cnt = 4;
int raw_files_cnt = 5; int raw_files_cnt = 5;
...@@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ...@@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
int index_files_cnt = 7; int index_files_cnt = 7;
meta::TableFileSchema table_file; meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_; table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_files_cnt; ++i) { for (auto i=0; i<new_files_cnt; ++i) {
status = impl.CreateTableFile(table_file); status = impl.CreateTableFile(table_file);
...@@ -488,7 +309,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ...@@ -488,7 +309,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
ASSERT_EQ(files.size(), to_index_files_cnt); ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files; meta::DatePartionedTableFilesSchema dated_files;
status = impl.FilesToMerge(group.table_id_, dated_files); status = impl.FilesToMerge(table.table_id_, dated_files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或