diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b5ba2134b9204c40d6c4b90b6450b96ec97b21d3..c2ed775601f875ed8c35bda5ff30ba9c4002756a 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,7 +15,6 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day - sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/coverage.sh b/cpp/coverage.sh index dd2b9418f5107d3aa829fb8015285916e49ffa05..cc509b611b999a51f09acbed8340270a4169ae20 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,5 +1,7 @@ #!/bin/bash +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib + LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" @@ -12,26 +14,6 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" -MYSQL_USER_NAME=root -MYSQL_PASSWORD=Fantast1c -MYSQL_HOST='192.168.1.194' -MYSQL_PORT='3306' - -MYSQL_DB_NAME=`date +%s%N` - -function mysql_exc() -{ - cmd=$1 - mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" - if [ $? -ne 0 ]; then - echo "mysql $cmd run failed" - fi -} - -mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" -mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" -mysql_exc "FLUSH PRIVILEGES;" - # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -40,13 +22,14 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do - echo $test case ${test} in db_test) # set run args for db_test - args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" + args="mysql://root:Fantast1c@192.168.1.194:3306/test" + ;; *_test) args="" + ;; esac # run unittest ./${DIR_UNITTEST}/${test} "${args}" @@ -55,8 +38,6 @@ for test in `ls ${DIR_UNITTEST}`; do fi done -mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" - # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -69,4 +50,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 6a763a972990b7665cc9940b9860c37ea49252d4..8992ba274c7bc48c1374bf2b6e88e7804aa98b6a 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -69,10 +69,6 @@ public: max_idle_time_ = max_idle; } - std::string getDB() { - return db_; - } - protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index cb701680757e7389d9656b65884c22183e0f81a4..0b0cc01e5d2ab1f52848bff8f301185bedb0ef00 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,30 +1733,17 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << - "AND table_name = " << quote << "TableFiles" << quote << ";"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - StoreQueryResult res = cleanUpQuery.store(); - assert(res); - if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - } - - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); - } + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 76d7846362f8bf447fd09ccdce0b8ffbb89cc5ed..436086acb3651af46d4c744dae422199bcc743a8 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,8 +21,21 @@ using namespace zilliz::milvus::engine; -TEST_F(MySQLTest, TABLE_TEST) { +//TEST_F(MySQLTest, InitializeTest) { +// DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// meta::MySQLMetaImpl impl(options); +// auto status = impl.Initialize(); +// std::cout << status.ToString() << std::endl; +// ASSERT_TRUE(status.ok()); +//} + +TEST_F(MySQLTest, core) { DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; try { options = getDBMetaOptions(); } catch(std::exception& ex) { @@ -32,38 +45,203 @@ TEST_F(MySQLTest, TABLE_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); +// auto status = impl.Initialize(); +// ASSERT_TRUE(status.ok()); - auto table_id = "meta_test_table"; + meta::TableSchema schema1; + schema1.table_id_ = "test1"; + schema1.dimension_ = 123; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + auto status = impl.CreateTable(schema1); +// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); - auto gid = table.id_; - table.id_ = -1; - status = impl.DescribeTable(table); + meta::TableSchema schema2; + schema2.table_id_ = "test2"; + schema2.dimension_ = 321; + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); - ASSERT_EQ(table.id_, gid); - ASSERT_EQ(table.table_id_, table_id); - table.table_id_ = "not_found"; - status = impl.DescribeTable(table); + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; +// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); + ASSERT_TRUE(status.ok()); + + status = impl.DeleteTable(schema2.table_id_); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + size_t id1 = schema1.id_; + long created_on1 = schema1.created_on_; + status = impl.DescribeTable(schema1); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(schema1.id_, id1); + ASSERT_EQ(schema1.table_id_, "test1"); + ASSERT_EQ(schema1.created_on_, created_on1); + ASSERT_EQ(schema1.files_cnt_, 0); + ASSERT_EQ(schema1.engine_type_, 1); + ASSERT_EQ(schema1.store_raw_data_, false); + + bool check; + status = impl.HasTable("test1", check); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(check, true); + + std::vector table_schema_array; + status = impl.AllTables(table_schema_array); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_schema_array.size(), 1); + meta::TableSchema resultSchema = table_schema_array[0]; + ASSERT_EQ(resultSchema.id_, id1); + ASSERT_EQ(resultSchema.table_id_, "test1"); + ASSERT_EQ(resultSchema.dimension_, 123); + ASSERT_EQ(resultSchema.files_cnt_, 0); + ASSERT_EQ(resultSchema.engine_type_, 1); + ASSERT_EQ(resultSchema.store_raw_data_, false); + + meta::TableFileSchema tableFileSchema; + tableFileSchema.table_id_ = "test1"; + + status = impl.CreateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; + status = impl.UpdateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableFilesSchema filesToIndex; + status = impl.FilesToIndex(filesToIndex); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToIndex.size(), 1); + meta::TableFileSchema fileToIndex = filesToIndex[0]; + ASSERT_EQ(fileToIndex.table_id_, "test1"); + ASSERT_EQ(fileToIndex.dimension_, 123); + +// meta::TableFilesSchema filesToIndex; +// status = impl.FilesToIndex(filesToIndex); +// ASSERT_TRUE(status.ok()); +// ASSERT_EQ(filesToIndex.size(), 0); + + meta::DatesT partition; + partition.push_back(tableFileSchema.date_); + meta::DatePartionedTableFilesSchema filesToSearch; + status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToSearch.size(), 1); + ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; + ASSERT_EQ(fileToSearch.table_id_, "test1"); + ASSERT_EQ(fileToSearch.dimension_, 123); + + tableFileSchema.file_type_ = meta::TableFileSchema::RAW; + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + meta::DatePartionedTableFilesSchema filesToMerge; + status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToMerge.size(), 1); + ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; + ASSERT_EQ(fileToMerge.table_id_, "test1"); + ASSERT_EQ(fileToMerge.dimension_, 123); + + meta::TableFilesSchema resultTableFilesSchema; + std::vector ids; + ids.push_back(tableFileSchema.id_); + status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultTableFilesSchema.size(), 1); + meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; +// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); + ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); + ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); + ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); + ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); + ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); + ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); + ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); + + tableFileSchema.size_ = 234; + meta::TableSchema schema3; + schema3.table_id_ = "test3"; + schema3.dimension_ = 321; + status = impl.CreateTable(schema3); + ASSERT_TRUE(status.ok()); + meta::TableFileSchema tableFileSchema2; + tableFileSchema2.table_id_ = "test3"; + tableFileSchema2.size_ = 345; + status = impl.CreateTableFile(tableFileSchema2); + ASSERT_TRUE(status.ok()); + meta::TableFilesSchema filesToUpdate; + filesToUpdate.emplace_back(tableFileSchema); + filesToUpdate.emplace_back(tableFileSchema2); + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + uint64_t resultSize; + status = impl.Size(resultSize); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); + + uint64_t countResult; + status = impl.Count(tableFileSchema.table_id_, countResult); + ASSERT_TRUE(status.ok()); + + status = impl.DropAll(); + ASSERT_TRUE(status.ok()); + +} + +TEST_F(MySQLTest, GROUP_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(options, mode); + + auto table_id = "meta_test_group"; + + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); + ASSERT_TRUE(status.ok()); + + auto gid = group.id_; + group.id_ = -1; + status = impl.DescribeTable(group); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(group.id_, gid); + ASSERT_EQ(group.table_id_, table_id); + + group.table_id_ = "not_found"; + status = impl.DescribeTable(group); ASSERT_TRUE(!status.ok()); - table.table_id_ = table_id; - status = impl.CreateTable(table); + group.table_id_ = table_id; + status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); - table.table_id_ = ""; - status = impl.CreateTable(table); + group.table_id_ = ""; + status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, TABLE_FILE_TEST) { +TEST_F(MySQLTest, table_file_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -75,16 +253,17 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - table.dimension_ = 256; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + group.dimension_ = 256; + auto status = impl.CreateTable(group); meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; status = impl.CreateTableFile(table_file); +// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -153,15 +332,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -212,13 +391,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -266,9 +445,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -276,7 +455,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; for (auto i=0; i