From 45a777b4d5c676a309c95b3bd25a7db1eecbb63a Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 22 Aug 2019 09:21:15 +0800 Subject: [PATCH] merge 0.3.1 Former-commit-id: 6fa5ce5365752369865360d715659c56fc367f89 --- cpp/CHANGELOG.md | 1 + cpp/src/db/Utils.cpp | 24 +++++++++------ cpp/src/db/Utils.h | 2 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 47 ++++++++++++++++++++++++----- cpp/src/db/meta/SqliteMetaImpl.cpp | 24 ++++++++++++++- cpp/unittest/db/db_tests.cpp | 7 +++++ cpp/unittest/db/mem_test.cpp | 39 +++++++++++++++++++++++- cpp/unittest/db/mysql_meta_test.cpp | 7 +++++ cpp/unittest/db/scheduler_test.cpp | 1 + cpp/unittest/server/config_test.cpp | 3 ++ cpp/unittest/server/util_test.cpp | 26 ++++++++++++++-- 11 files changed, 159 insertions(+), 22 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index a7ddee10..de878a86 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -61,6 +61,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-257 - Update bzip2 download url - MS-288 - Update compile scripts - MS-330 - Stability test failed caused by server core dumped +- MS-347 - Build index hangs again ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 5a0d3caf..1a1355d5 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -85,16 +85,20 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id return Status::OK(); } -Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id) { - std::string db_path = options.path; - std::string table_path = db_path + TABLES_FOLDER + table_id; - boost::filesystem::remove_all(table_path); - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - - for(auto& path : options.slave_paths) { - table_path = path + TABLES_FOLDER + table_id; - boost::filesystem::remove_all(table_path); - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; +Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { + std::vector paths = options.slave_paths; + paths.push_back(options.path); + + for(auto& path : paths) { + std::string table_path = path + TABLES_FOLDER + table_id; + if(force) { + boost::filesystem::remove_all(table_path); + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + } else if(boost::filesystem::exists(table_path) && + boost::filesystem::is_empty(table_path)) { + boost::filesystem::remove_all(table_path); + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + } } return Status::OK(); diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 47a8fca9..101d849c 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -19,7 +19,7 @@ namespace utils { long GetMicroSecTimeStamp(); Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id); -Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id); +Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true); Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index fa2697ec..26f38938 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -1652,15 +1652,14 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { - - auto now = utils::GetMicroSecTimeStamp(); + std::set table_ids; + + //remove to_delete files try { MetricCollector metric; { - - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { @@ -1700,6 +1699,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; idsToDelete.emplace_back(std::to_string(table_file.id_)); + + table_ids.insert(table_file.table_id_); } if (!idsToDelete.empty()) { @@ -1734,12 +1735,11 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } + //remove to_delete tables try { MetricCollector metric; { - - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { @@ -1765,7 +1765,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { std::string table_id; resRow["table_id"].to_string(table_id); - utils::DeleteTablePath(options_, table_id); + utils::DeleteTablePath(options_, table_id, false);//only delete empty folder idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } @@ -1794,6 +1794,39 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } + //remove deleted table folder + //don't remove table folder until all its files has been deleted + try { + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + for(auto& table_id : table_ids) { + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT file_id " << + "FROM TableFiles " << + "WHERE table_id = " << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + if (res.empty()) { + utils::DeleteTablePath(options_, table_id); + } + } + } + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", er.what()); + } + return Status::OK(); } diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 9118eadd..081c7569 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -1069,6 +1069,9 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); + std::set table_ids; + + //remove to_delete files try { MetricCollector metric; @@ -1098,6 +1101,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; ConnectorPtr->remove(table_file.id_); + table_ids.insert(table_file.table_id_); } return true; }); @@ -1111,6 +1115,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("Encounter exception when clean table files", e); } + //remove to_delete tables try { MetricCollector metric; @@ -1123,7 +1128,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &table : tables) { - utils::DeleteTablePath(options_, std::get<1>(table)); + utils::DeleteTablePath(options_, std::get<1>(table), false);//only delete empty folder ConnectorPtr->remove(std::get<0>(table)); } @@ -1139,6 +1144,23 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("Encounter exception when clean table files", e); } + //remove deleted table folder + //don't remove table folder until all its files has been deleted + try { + MetricCollector metric; + + for(auto& table_id : table_ids) { + auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), + where(c(&TableFileSchema::table_id_) == table_id)); + if(selected.size() == 0) { + utils::DeleteTablePath(options_, table_id); + } + } + + } catch (std::exception &e) { + return HandleException("Encounter exception when delete table folder", e); + } + return Status::OK(); } diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 8b36d2ef..9a32d519 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -93,6 +93,8 @@ TEST_F(DBTest, CONFIG_TEST) { TEST_F(DBTest, DB_TEST) { + db_->Open(GetOptions(), &db_); + engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -161,6 +163,11 @@ TEST_F(DBTest, DB_TEST) { } search.join(); + + uint64_t count; + stat = db_->GetTableRowCount(TABLE_NAME, count); + ASSERT_STATS(stat); + ASSERT_TRUE(count > 0); }; TEST_F(DBTest, SEARCH_TEST) { diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index e5618370..cf84d745 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -445,4 +445,41 @@ TEST_F(DBTest, VECTOR_IDS_TEST) for (auto i = 0; i < nb; i++) { ASSERT_EQ(vector_ids[i], i + nb); } -} \ No newline at end of file +} + +TEST_F(NewMemManagerTest, MEMMANAGER_TEST) { + int setenv_res = setenv("MILVUS_USE_OLD_MEM_MANAGER", "ON", 1); + ASSERT_TRUE(setenv_res == 0); + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + auto start_time = METRICS_NOW_TIME; + + int insert_loop = 20; + for (int i = 0; i < insert_loop; ++i) { + int64_t nb = 40960; + std::vector xb; + BuildVectors(nb, xb); + engine::IDNumbers vector_ids; + engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(status.ok()); + } + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time; + + delete db_; + boost::filesystem::remove_all(options.meta.path); +} diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 80a9ddf4..07e0eada 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -194,6 +194,13 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) { i++; } + bool has; + status = impl.HasNonIndexFiles(table_id, has); + ASSERT_TRUE(status.ok()); + + status = impl.UpdateTableFilesToIndex(table_id); + ASSERT_TRUE(status.ok()); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp index 0937ef19..4826fa4b 100644 --- a/cpp/unittest/db/scheduler_test.cpp +++ b/cpp/unittest/db/scheduler_test.cpp @@ -56,6 +56,7 @@ TEST(DBSchedulerTest, TASK_QUEUE_TEST) { ptr = queue.Back(); ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); + load_task->Execute(); } TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) { diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 2172bdd9..51e3feac 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -87,6 +87,9 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH); ASSERT_EQ(err, server::SERVER_SUCCESS); + err = server::ServerConfig::GetInstance().ValidateConfig(); + ASSERT_EQ(err, server::SERVER_SUCCESS); + server::ConfigNode node1 = config.GetConfig("server_config"); server::ConfigNode& node2 = config.GetConfig("cache_config"); node1.Combine(node2); diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index c3a47182..b75673e5 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -178,12 +178,15 @@ TEST(UtilTest, VALIDATE_TABLENAME_TEST) { res = server::ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); - table_name = "中文"; + table_name = "_!@#!@"; res = server::ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + table_name = "中文"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); - table_name = std::string('a', 32768); + table_name = std::string(10000, 'a'); res = server::ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); } @@ -204,3 +207,22 @@ TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) { ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE); } +TEST(ValidationUtilTest, ValidateGpuTest) { + ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS); + + size_t memory = 0; + ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), server::SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), server::SERVER_SUCCESS); +} + +TEST(UtilTest, TIMERECORDER_TEST) { + for(int64_t log_level = 0; log_level <= 6; log_level++) { + if(log_level == 5) { + continue; //skip fatal + } + server::TimeRecorder rc("time", log_level); + rc.RecordSection("end"); + } +} + -- GitLab