diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index c2ed775601f875ed8c35bda5ff30ba9c4002756a..b5ba2134b9204c40d6c4b90b6450b96ec97b21d3 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,6 +15,7 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/coverage.sh b/cpp/coverage.sh index cc509b611b999a51f09acbed8340270a4169ae20..dd2b9418f5107d3aa829fb8015285916e49ffa05 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,7 +1,5 @@ #!/bin/bash -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib - LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" @@ -14,6 +12,26 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" +MYSQL_USER_NAME=root +MYSQL_PASSWORD=Fantast1c +MYSQL_HOST='192.168.1.194' +MYSQL_PORT='3306' + +MYSQL_DB_NAME=`date +%s%N` + +function mysql_exc() +{ + cmd=$1 + mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" + if [ $? -ne 0 ]; then + echo "mysql $cmd run failed" + fi +} + +mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" +mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" +mysql_exc "FLUSH PRIVILEGES;" + # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -22,14 +40,13 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do + echo $test case ${test} in db_test) # set run args for db_test - args="mysql://root:Fantast1c@192.168.1.194:3306/test" - ;; + args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" *_test) args="" - ;; esac # run unittest ./${DIR_UNITTEST}/${test} "${args}" @@ -38,6 +55,8 @@ for test in `ls ${DIR_UNITTEST}`; do fi done +mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" + # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -50,4 +69,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 8992ba274c7bc48c1374bf2b6e88e7804aa98b6a..6a763a972990b7665cc9940b9860c37ea49252d4 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -69,6 +69,10 @@ public: max_idle_time_ = max_idle; } + std::string getDB() { + return db_; + } + protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 0b0cc01e5d2ab1f52848bff8f301185bedb0ef00..cb701680757e7389d9656b65884c22183e0f81a4 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,17 +1733,30 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << + "AND table_name = " << quote << "TableFiles" << quote << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + StoreQueryResult res = cleanUpQuery.store(); + assert(res); + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 436086acb3651af46d4c744dae422199bcc743a8..76d7846362f8bf447fd09ccdce0b8ffbb89cc5ed 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,21 +21,8 @@ using namespace zilliz::milvus::engine; -//TEST_F(MySQLTest, InitializeTest) { -// DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// meta::MySQLMetaImpl impl(options); -// auto status = impl.Initialize(); -// std::cout << status.ToString() << std::endl; -// ASSERT_TRUE(status.ok()); -//} - -TEST_F(MySQLTest, core) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; try { options = getDBMetaOptions(); } catch(std::exception& ex) { @@ -45,203 +32,38 @@ TEST_F(MySQLTest, core) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); -// auto status = impl.Initialize(); -// ASSERT_TRUE(status.ok()); - meta::TableSchema schema1; - schema1.table_id_ = "test1"; - schema1.dimension_ = 123; + auto table_id = "meta_test_table"; - auto status = impl.CreateTable(schema1); -// std::cout << status.ToString() << std::endl; + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - meta::TableSchema schema2; - schema2.table_id_ = "test2"; - schema2.dimension_ = 321; - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; + auto gid = table.id_; + table.id_ = -1; + status = impl.DescribeTable(table); ASSERT_TRUE(status.ok()); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; -// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_TRUE(status.ok()); - - status = impl.DeleteTable(schema2.table_id_); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - size_t id1 = schema1.id_; - long created_on1 = schema1.created_on_; - status = impl.DescribeTable(schema1); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(schema1.id_, id1); - ASSERT_EQ(schema1.table_id_, "test1"); - ASSERT_EQ(schema1.created_on_, created_on1); - ASSERT_EQ(schema1.files_cnt_, 0); - ASSERT_EQ(schema1.engine_type_, 1); - ASSERT_EQ(schema1.store_raw_data_, false); - - bool check; - status = impl.HasTable("test1", check); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(check, true); - - std::vector table_schema_array; - status = impl.AllTables(table_schema_array); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_schema_array.size(), 1); - meta::TableSchema resultSchema = table_schema_array[0]; - ASSERT_EQ(resultSchema.id_, id1); - ASSERT_EQ(resultSchema.table_id_, "test1"); - ASSERT_EQ(resultSchema.dimension_, 123); - ASSERT_EQ(resultSchema.files_cnt_, 0); - ASSERT_EQ(resultSchema.engine_type_, 1); - ASSERT_EQ(resultSchema.store_raw_data_, false); - - meta::TableFileSchema tableFileSchema; - tableFileSchema.table_id_ = "test1"; - - status = impl.CreateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; - status = impl.UpdateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableFilesSchema filesToIndex; - status = impl.FilesToIndex(filesToIndex); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToIndex.size(), 1); - meta::TableFileSchema fileToIndex = filesToIndex[0]; - ASSERT_EQ(fileToIndex.table_id_, "test1"); - ASSERT_EQ(fileToIndex.dimension_, 123); - -// meta::TableFilesSchema filesToIndex; -// status = impl.FilesToIndex(filesToIndex); -// ASSERT_TRUE(status.ok()); -// ASSERT_EQ(filesToIndex.size(), 0); - - meta::DatesT partition; - partition.push_back(tableFileSchema.date_); - meta::DatePartionedTableFilesSchema filesToSearch; - status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToSearch.size(), 1); - ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; - ASSERT_EQ(fileToSearch.table_id_, "test1"); - ASSERT_EQ(fileToSearch.dimension_, 123); - - tableFileSchema.file_type_ = meta::TableFileSchema::RAW; - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - meta::DatePartionedTableFilesSchema filesToMerge; - status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToMerge.size(), 1); - ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; - ASSERT_EQ(fileToMerge.table_id_, "test1"); - ASSERT_EQ(fileToMerge.dimension_, 123); - - meta::TableFilesSchema resultTableFilesSchema; - std::vector ids; - ids.push_back(tableFileSchema.id_); - status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFilesSchema.size(), 1); - meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; -// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); - ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); - ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); - ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); - ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); - ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); - ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); - ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); - - tableFileSchema.size_ = 234; - meta::TableSchema schema3; - schema3.table_id_ = "test3"; - schema3.dimension_ = 321; - status = impl.CreateTable(schema3); - ASSERT_TRUE(status.ok()); - meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test3"; - tableFileSchema2.size_ = 345; - status = impl.CreateTableFile(tableFileSchema2); - ASSERT_TRUE(status.ok()); - meta::TableFilesSchema filesToUpdate; - filesToUpdate.emplace_back(tableFileSchema); - filesToUpdate.emplace_back(tableFileSchema2); - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - uint64_t resultSize; - status = impl.Size(resultSize); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); - - uint64_t countResult; - status = impl.Count(tableFileSchema.table_id_, countResult); - ASSERT_TRUE(status.ok()); - - status = impl.DropAll(); - ASSERT_TRUE(status.ok()); - -} - -TEST_F(MySQLTest, GROUP_TEST) { - DBMetaOptions options; - try { - options = getDBMetaOptions(); - } catch(std::exception& ex) { - ASSERT_TRUE(false); - return; - } - - int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); - - auto table_id = "meta_test_group"; - - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); - ASSERT_TRUE(status.ok()); - - auto gid = group.id_; - group.id_ = -1; - status = impl.DescribeTable(group); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); - - group.table_id_ = "not_found"; - status = impl.DescribeTable(group); + table.table_id_ = "not_found"; + status = impl.DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl.CreateTable(group); + table.table_id_ = table_id; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - group.table_id_ = ""; - status = impl.CreateTable(group); + table.table_id_ = ""; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, table_file_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - group.dimension_ = 256; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + table.dimension_ = 256; + auto status = impl.CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl.CreateTableFile(table_file); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; i