diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index b9241a5e0fc28271cc941b57a2d9740f96b1080b..960b732d9dfa2600d604147a02743d08c5d97c17 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -14,8 +14,8 @@ Please mark all change in change log and use the ticket from JIRA. ## Improvement - MS-156 - Add unittest for merge result functions - - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl +- MS-204 - Support multi db_path ## New Feature - MS-195 - Add nlist and use_blas_threshold conf diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index cb997497844b6497bae067cfb438a847e122bf7e..ec5bde9d832546d918bf6fe3b8fc6bc2c96c79f1 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -6,6 +6,7 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path + db_slave_path: # secondry data storage path, split by semicolon # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 6981ca93e2aee828162ce48a57f8e04f16efbd3c..8a8c2b117e7d31298ad4d2c7e5205f74f50f2e31 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -461,6 +461,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; + table_file.file_type_ = meta::TableFileSchema::INDEX; //for multi-db-path, distribute index file averagely to each path Status status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { return status; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 400335d6ca0bf70b6b4907d0c91405d449188fac..fc67c6b36029248f9a279ec8fcf97a85bde745a9 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -83,26 +83,6 @@ using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL); -std::string DBMetaImpl::GetTablePath(const std::string &table_id) { - return options_.path + "/tables/" + table_id; -} - -std::string DBMetaImpl::GetTableDatePartitionPath(const std::string &table_id, DateT &date) { - std::stringstream ss; - ss << GetTablePath(table_id) << "/" << date; - return ss.str(); -} - -void DBMetaImpl::GetTableFilePath(TableFileSchema &group_file) { - if (group_file.date_ == EmptyDate) { - group_file.date_ = Meta::GetDate(); - } - std::stringstream ss; - ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_) - << "/" << group_file.file_id_; - group_file.location_ = ss.str(); -} - Status DBMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; @@ -213,15 +193,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { return Status::DBTransactionError("Add Table Error"); } - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - if (!boost::filesystem::is_directory(table_path)) { - auto ret = boost::filesystem::create_directories(table_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status::Error("Failed to create table path"); - } - } + return utils::CreateTablePath(options_, table_schema.table_id_); } catch (std::exception &e) { return HandleException("Encounter exception when create table", e); @@ -307,9 +279,6 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - } catch (std::exception &e) { return HandleException("Encounter exception when describe table", e); } @@ -411,20 +380,11 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; - GetTableFilePath(file_schema); auto id = ConnectorPtr->insert(file_schema); file_schema.id_ = id; - auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); - - if (!boost::filesystem::is_directory(partition_path)) { - auto ret = boost::filesystem::create_directory(partition_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; - return Status::DBTransactionError("Failed to create partition directory"); - } - } + return utils::CreateTableFilePath(options_, file_schema); } catch (std::exception& ex) { return HandleException("Encounter exception when create table file", ex); @@ -461,7 +421,7 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto groupItr = groups.find(table_file.table_id_); if (groupItr == groups.end()) { TableSchema table_schema; @@ -524,7 +484,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); @@ -566,7 +526,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); @@ -616,7 +576,7 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, table_file.size_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); @@ -662,7 +622,7 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, file_schema.date_ = std::get<4>(file); file_schema.engine_type_ = std::get<5>(file); file_schema.dimension_ = table_schema.dimension_; - GetTableFilePath(file_schema); + utils::GetTableFilePath(options_, file_schema); table_files.emplace_back(file_schema); } @@ -895,10 +855,9 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.date_ = std::get<3>(file); - GetTableFilePath(table_file); + utils::DeleteTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; - boost::filesystem::remove(table_file.location_); ConnectorPtr->remove(table_file.id_); } @@ -922,10 +881,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &table : tables) { - auto table_path = GetTablePath(std::get<1>(table)); - - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); + utils::DeleteTablePath(options_, std::get<1>(table)); ConnectorPtr->remove(std::get<0>(table)); } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index f0b17c4106d0f0c5dc1d612e603816d95c437c31..98a42c000dfa46242ecef9b8a1e2e0b3e0cbccea 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -70,9 +70,6 @@ private: Status NextFileId(std::string& file_id); Status NextTableId(std::string& table_id); Status DiscardFiles(long to_discard_size); - std::string GetTablePath(const std::string& table_id); - std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); - void GetTableFilePath(TableFileSchema& group_file); Status Initialize(); const DBMetaOptions options_; diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index b20d3a05d02f008faf5998a3d9c4beaa8a803a11..641712e9d52aabca2c2f1a6ec33ef0a642ae3098 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -31,7 +31,6 @@ struct TableSchema { int state_ = (int)NORMAL; size_t files_cnt_ = 0; uint16_t dimension_ = 0; - std::string location_; long created_on_ = 0; int engine_type_ = (int)EngineType::FAISS_IDMAP; bool store_raw_data_ = false; diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 14ce38c743114bba3175923102f195267da11cb7..bd82e7e6a1f76dd5aa5c6da2cae39ac763425496 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -67,26 +67,6 @@ namespace meta { } - std::string MySQLMetaImpl::GetTablePath(const std::string &table_id) { - return options_.path + "/tables/" + table_id; - } - - std::string MySQLMetaImpl::GetTableDatePartitionPath(const std::string &table_id, DateT &date) { - std::stringstream ss; - ss << GetTablePath(table_id) << "/" << date; - return ss.str(); - } - - void MySQLMetaImpl::GetTableFilePath(TableFileSchema &group_file) { - if (group_file.date_ == EmptyDate) { - group_file.date_ = Meta::GetDate(); - } - std::stringstream ss; - ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_) - << "/" << group_file.file_id_; - group_file.location_ = ss.str(); - } - Status MySQLMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; @@ -412,15 +392,8 @@ namespace meta { // auto total_time = METRICS_MICROSECONDS(start_time, end_time); // server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - if (!boost::filesystem::is_directory(table_path)) { - auto ret = boost::filesystem::create_directories(table_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status::Error("Failed to create table path"); - } - } + return utils::CreateTablePath(options_, table_schema.table_id_); + } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); @@ -588,9 +561,6 @@ namespace meta { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); @@ -743,7 +713,7 @@ namespace meta { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; - GetTableFilePath(file_schema); + utils::GetTableFilePath(options_, file_schema); std::string id = "NULL"; //auto-increment std::string table_id = file_schema.table_id_; @@ -788,15 +758,7 @@ namespace meta { } } // Scoped Connection - auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); - - if (!boost::filesystem::is_directory(partition_path)) { - auto ret = boost::filesystem::create_directory(partition_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; - return Status::DBTransactionError("Failed to create partition directory"); - } - } + return utils::CreateTableFilePath(options_, file_schema); } catch (const BadQuery& er) { // Handle any query errors @@ -881,7 +843,7 @@ namespace meta { } table_file.dimension_ = groups[table_file.table_id_].dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); files.push_back(table_file); } @@ -992,7 +954,7 @@ namespace meta { table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -1078,7 +1040,7 @@ namespace meta { table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -1173,7 +1135,7 @@ namespace meta { file_schema.dimension_ = table_schema.dimension_; - GetTableFilePath(file_schema); + utils::GetTableFilePath(options_, file_schema); table_files.emplace_back(file_schema); } @@ -1633,11 +1595,10 @@ namespace meta { table_file.date_ = resRow["date"]; - GetTableFilePath(table_file); + utils::DeleteTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; - boost::filesystem::remove(table_file.location_); idsToDelete.emplace_back(std::to_string(table_file.id_)); } @@ -1710,10 +1671,7 @@ namespace meta { std::string table_id; resRow["table_id"].to_string(table_id); - auto table_path = GetTablePath(table_id); - - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); + utils::DeleteTablePath(options_, table_id); idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 9daa9f83cbc3d881a44807095f889d70003355ab..403d5b0b594e953a491fc3e83ff6a7c0f3b73ab1 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -75,9 +75,6 @@ namespace meta { Status NextFileId(std::string& file_id); Status NextTableId(std::string& table_id); Status DiscardFiles(long long to_discard_size); - std::string GetTablePath(const std::string& table_id); - std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); - void GetTableFilePath(TableFileSchema& group_file); Status Initialize(); const DBMetaOptions options_; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 39d0a15019ab2bb5cfb422a06095193a8a30bc78..1c3c28b4ac6ff2fa653a0b53b5d1aad6ae406bbf 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace zilliz { namespace milvus { @@ -43,6 +44,7 @@ private: struct DBMetaOptions { std::string path; + std::vector slave_paths; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 5f38a9c72a2a967f8065f63b4cc3c990bd826852..aa533559bdf5268b797cc866a8fee51b235bf563 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -4,14 +4,58 @@ * Proprietary and confidential. ******************************************************************************/ #include "Utils.h" +#include "utils/CommonUtil.h" +#include "Log.h" +#include #include +#include namespace zilliz { namespace milvus { namespace engine { namespace utils { +namespace { + +static const std::string TABLES_FOLDER = "/tables/"; + +static uint64_t index_file_counter = 0; +static std::mutex index_file_counter_mutex; + +std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { + std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; + std::string partition_path = table_path + "/" + std::to_string(table_file.date_); + return partition_path; +} + +std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { + uint64_t path_count = options.slave_paths.size() + 1; + std::string target_path = options.path; + uint64_t index = 0; + + if(meta::TableFileSchema::INDEX == table_file.file_type_) { + // index file is large file and to be persisted permanently + // we need to distribute index files to each db_path averagely + // round robin according to a file counter + std::lock_guard lock(index_file_counter_mutex); + index = index_file_counter % path_count; + index_file_counter++; + } else { + // for other type files, they could be merged or deleted + // so we round robin according to their file id + index = table_file.id_ % path_count; + } + + if (index > 0) { + target_path = options.slave_paths[index - 1]; + } + + return ConstructParentFolder(target_path, table_file); +} + +} + long GetMicroSecTimeStamp() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( @@ -20,6 +64,82 @@ long GetMicroSecTimeStamp() { return micros; } +Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { + std::string db_path = options.path; + std::string table_path = db_path + TABLES_FOLDER + table_id; + auto status = server::CommonUtil::CreateDirectory(table_path); + if (status != 0) { + ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + return Status::Error("Failed to create table path"); + } + + for(auto& path : options.slave_paths) { + table_path = path + TABLES_FOLDER + table_id; + status = server::CommonUtil::CreateDirectory(table_path); + if (status != 0) { + ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + return Status::Error("Failed to create table path"); + } + } + + return Status::OK(); +} + +Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id) { + std::string db_path = options.path; + std::string table_path = db_path + TABLES_FOLDER + table_id; + boost::filesystem::remove_all(table_path); + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + + for(auto& path : options.slave_paths) { + table_path = path + TABLES_FOLDER + table_id; + boost::filesystem::remove_all(table_path); + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + } + + return Status::OK(); +} + +Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { + std::string parent_path = GetTableFileParentFolder(options, table_file); + + auto status = server::CommonUtil::CreateDirectory(parent_path); + if (status != 0) { + ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error"; + return Status::DBTransactionError("Failed to create partition directory"); + } + + table_file.location_ = parent_path + "/" + table_file.file_id_; + + return Status::OK(); +} + +Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { + std::string parent_path = ConstructParentFolder(options.path, table_file); + std::string file_path = parent_path + "/" + table_file.file_id_; + if(boost::filesystem::exists(file_path)) { + table_file.location_ = file_path; + return Status::OK(); + } else { + for(auto& path : options.slave_paths) { + parent_path = ConstructParentFolder(path, table_file); + file_path = parent_path + "/" + table_file.file_id_; + if(boost::filesystem::exists(file_path)) { + table_file.location_ = file_path; + return Status::OK(); + } + } + } + + return Status::Error("Table file doesn't exist: " + table_file.file_id_); +} + +Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { + utils::GetTableFilePath(options, table_file); + boost::filesystem::remove(table_file.location_); + return Status::OK(); +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 481808d6308f2b16a7dcf9e1de4ff472e52f6475..a5a0f8ff8c278827ac60ebb4ed117d741cbe7788 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -5,6 +5,10 @@ ******************************************************************************/ #pragma once +#include "Options.h" +#include "MetaTypes.h" + +#include namespace zilliz { namespace milvus { @@ -13,6 +17,13 @@ namespace utils { long GetMicroSecTimeStamp(); +Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id); +Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id); + +Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index fca15cb65a6c68eb051b2f272900ae995ba2e84c..05d8b7debe944ee410a8358982638e7d0b24ed45 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -8,6 +8,7 @@ #include "ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" namespace zilliz { namespace milvus { @@ -19,6 +20,10 @@ DBWrapper::DBWrapper() { opt.meta.backend_uri = config.GetValue(CONFIG_DB_URL); std::string db_path = config.GetValue(CONFIG_DB_PATH); opt.meta.path = db_path + "/db"; + + std::string db_slave_path = config.GetValue(CONFIG_DB_SLAVE_PATH); + StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); + int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; @@ -59,6 +64,14 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } + for(auto& path : opt.meta.slave_paths) { + err = CommonUtil::CreateDirectory(path); + if(err != SERVER_SUCCESS) { + std::cout << "ERROR! Failed to create database slave path: " << path << std::endl; + kill(0, SIGUSR1); + } + } + std::string msg = opt.meta.path; try { zilliz::milvus::engine::DB::Open(opt, &db_); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index c5565b7282b4d87714040620837af690658225c1..1fa5bb65eaa2df525c39d202a6538515e0dc73ef 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -23,6 +23,7 @@ static const std::string CONFIG_CLUSTER_MODE = "mode"; static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; +static const std::string CONFIG_DB_SLAVE_PATH = "db_slave_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 19157af9f2249ff70f69aeabc995def16f3a230f..00e3a8c94e90cb8f953d5b4fb1a89307f843650e 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -51,8 +51,7 @@ bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) { return true; } -bool CommonUtil::IsDirectoryExist(const std::string &path) -{ +bool CommonUtil::IsDirectoryExist(const std::string &path) { DIR *dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { return false; @@ -63,6 +62,10 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) } ServerError CommonUtil::CreateDirectory(const std::string &path) { + if(path.empty()) { + return SERVER_SUCCESS; + } + struct stat directoryStat; int statOK = stat(path.c_str(), &directoryStat); if (statOK == 0) { @@ -120,6 +123,10 @@ namespace { } ServerError CommonUtil::DeleteDirectory(const std::string &path) { + if(path.empty()) { + return SERVER_SUCCESS; + } + struct stat directoryStat; int statOK = stat(path.c_str(), &directoryStat); if (statOK != 0) diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 36ba017beb60211ec80b013199d7ca9580052263..03c689d82c8972c4a97f8a80134c00e941ce5ba4 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -27,6 +27,10 @@ void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str, const std::string &delimeter, std::vector &result) { + if(str.empty()) { + return SERVER_SUCCESS; + } + size_t last = 0; size_t index = str.find_first_of(delimeter, last); while (index != std::string::npos) { diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index bd17081af8270d08c1b289bb223b86efc419f27b..729317011db002304419ef9ae38bae277ebe3262 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -272,7 +272,9 @@ TEST_F(DBTest2, DELETE_TEST) { stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + bool has_table = false; + db_->HasTable(TABLE_NAME, has_table); + ASSERT_TRUE(has_table); engine::IDNumbers vector_ids; @@ -293,5 +295,7 @@ TEST_F(DBTest2, DELETE_TEST) { stat = db_->DeleteTable(TABLE_NAME, dates); std::this_thread::sleep_for(std::chrono::seconds(2)); ASSERT_TRUE(stat.ok()); - ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + db_->HasTable(TABLE_NAME, has_table); + ASSERT_FALSE(has_table); }; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 0f3a92af09be83d164c0c95e61c9baca43e63439..5bce4058b133604665d600356131fcb98132e7a9 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -38,7 +38,7 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status.IsAlreadyExist()); table.table_id_ = ""; status = impl_->CreateTable(table); diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 7fdb30a2042441488308cb4abd89ec4265d319d3..02ce3f0fa7b77209186ef0e1601118357fd144b1 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -271,8 +271,9 @@ TEST_F(MySQLDBTest, DELETE_TEST) { stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); -// std::cout << "location: " << table_info_get.location_ << std::endl; - ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + bool has_table = false; + db_->HasTable(TABLE_NAME, has_table); + ASSERT_TRUE(has_table); engine::IDNumbers vector_ids; @@ -295,7 +296,9 @@ TEST_F(MySQLDBTest, DELETE_TEST) { std::this_thread::sleep_for(std::chrono::seconds(5)); // std::cout << "5 sec finish" << std::endl; ASSERT_TRUE(stat.ok()); -// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + db_->HasTable(TABLE_NAME, has_table); + ASSERT_FALSE(has_table); delete db_;