diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 58124c5296926954def0d2dd40d2dd8183024161..116f30026d9247d53732589f6b08696066f7d970 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -113,21 +113,13 @@ link_directories(${MILVUS_BINARY_DIR}) set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) -#set(MILVUS_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) -#set(MILVUS_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) add_compile_definitions(PROFILER=${PROFILER}) include_directories(${MILVUS_ENGINE_INCLUDE}) include_directories(${MILVUS_ENGINE_SRC}) -include_directories(/usr/local/cuda/include) -#include_directories(${MILVUS_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) -#link_directories(${MILVUS_THIRD_PARTY_BUILD}/lib) -#link_directories(${MILVUS_THIRD_PARTY_BUILD}/lib64) -#execute_process(COMMAND bash build.sh -# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) add_subdirectory(src) diff --git a/cpp/mysqlNotes b/cpp/mysqlNotes deleted file mode 100644 index 8aa151efff28c1b8d5c35950d6772aa21309f4ba..0000000000000000000000000000000000000000 --- a/cpp/mysqlNotes +++ /dev/null @@ -1,8 +0,0 @@ -sudo apt-get install mysql-server -sudo apt-get install libmysqlclient-dev -sudo ln -s libmysqlclient.so libmysqlclient_r.so - -Install MySQL++ -./configure --enable-thread-check LDFLAGS='-pthread' -make -sudo make install diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index def216358bad26ed56f986549c2e719dc55731cb..92bbff2b84105411ff4c4ac0ea47bcb63c36ffe2 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -139,7 +139,7 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); - if (options.mode != "read_only") { + if (options.mode != Options::MODE::READ_ONLY) { StartTimerTasks(); } } @@ -600,7 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 1; - if (options_.mode == "cluster") { + if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; // ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 97e89e0994c200d004c5456c2139bf192641f1ba..231c3cce4fb26e24bf22236d027fbb9edf2502ac 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -29,15 +29,8 @@ DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { p = ss.str(); } -// std::string uri; -// const char* uri_p = getenv("MILVUS_DB_META_URI"); -// if (uri_p) { -// uri = uri_p; -// } - DBMetaOptions meta; meta.path = p; -// meta.backend_uri = uri; return meta; } @@ -54,14 +47,9 @@ std::shared_ptr DBMetaImplFactory::Build() { } std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, - const std::string& mode) { + const int& mode) { std::string uri = metaOptions.backend_uri; -// if (uri.empty()) { -// //Default to sqlite if uri is empty -//// return std::make_shared(new meta::DBMetaImpl(metaOptions)); -// return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); -// } std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 48bea9b2911c4ddebc31fa571249e220df940457..889922b17a59cb9bb96df73971a0952abe6079e7 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -28,7 +28,7 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions, const std::string& mode); + static std::shared_ptr Build(const DBMetaOptions& metaOptions, const int& mode); }; struct DBFactory { diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index c1ea2e83bcda07d4c3654979ea89e9167be65b31..0978d77a7c99fd8f0cfa370224d551d311eb3c3f 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -20,12 +20,12 @@ public: password_(passWord), server_(serverIp), port_(port), - maxPoolSize_(maxPoolSize) + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - maxIdleTime_ = 300; //300ms + max_idle_time_ = 10; //10 seconds } // The destructor. We _must_ call ConnectionPool::clear() here, @@ -40,12 +40,10 @@ public: // connections actually in use, not those created. Also note that // we keep our own count; ConnectionPool::size() isn't the same! mysqlpp::Connection* grab() override { - while (conns_in_use_ > maxPoolSize_) { -// cout.put('R'); cout.flush(); // indicate waiting for release + while (conns_in_use_ > max_pool_size_) { sleep(1); } -// ENGINE_LOG_DEBUG << "conns_in_use_ in grab: " << conns_in_use_ << std::endl; ++conns_in_use_; return mysqlpp::ConnectionPool::grab(); } @@ -63,7 +61,7 @@ public: } void set_max_idle_time(int max_idle) { - maxIdleTime_ = max_idle; + max_idle_time_ = max_idle; } protected: @@ -72,7 +70,6 @@ protected: mysqlpp::Connection* create() override { // Create connection using the parameters we were passed upon // creation. -// cout.put('C'); cout.flush(); // indicate connection creation mysqlpp::Connection* conn = new mysqlpp::Connection(); conn->set_option(new mysqlpp::ReconnectOption(true)); conn->connect(db_.empty() ? 0 : db_.c_str(), @@ -86,12 +83,11 @@ protected: void destroy(mysqlpp::Connection* cp) override { // Our superclass can't know how we created the Connection, so // it delegates destruction to us, to be safe. -// cout.put('D'); cout.flush(); // indicate connection destruction delete cp; } unsigned int max_idle_time() override { - return maxIdleTime_; + return max_idle_time_; } private: @@ -102,7 +98,7 @@ private: std::string db_, user_, password_, server_; int port_; - int maxPoolSize_; + int max_pool_size_; - unsigned int maxIdleTime_; + unsigned int max_idle_time_; }; \ No newline at end of file diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 7ee9231b41beb25189f8183506ec3e27872d3e98..25d2c77777aa6fcacdd99501a58fd4addbb79d04 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -103,7 +103,7 @@ namespace meta { return Status::OK(); } - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const std::string& mode) + MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int& mode) : options_(options_), mode_(mode) { Initialize(); @@ -157,7 +157,7 @@ namespace meta { // connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); int threadHint = std::thread::hardware_concurrency(); int maxPoolSize = threadHint == 0 ? 8 : threadHint; - mySQLConnectionPool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); + mysql_connection_pool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); // std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; try { @@ -165,7 +165,9 @@ namespace meta { CleanUp(); { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse(); // if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { // return Status::Error("DB connection failed: ", connectionPtr->error()); // } @@ -190,6 +192,11 @@ namespace meta { "files_cnt BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "store_raw_data BOOL DEFAULT false NOT NULL);"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + } + if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } @@ -204,6 +211,11 @@ namespace meta { "updated_time BIGINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "date INT DEFAULT -1 NOT NULL);"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + } + if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } @@ -280,7 +292,11 @@ namespace meta { dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query dropPartitionsByDatesQuery = connectionPtr->query(); @@ -289,6 +305,10 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "date in (" << dateListStr << ");"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); + } + if (!dropPartitionsByDatesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", @@ -318,7 +338,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query createTableQuery = connectionPtr->query(); ENGINE_LOG_DEBUG << "Create Table in"; @@ -328,6 +352,11 @@ namespace meta { createTableQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + } + StoreQueryResult res = createTableQuery.store(); assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { @@ -360,6 +389,11 @@ namespace meta { "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + } + if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; @@ -410,7 +444,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } //soft delete table Query deleteTableQuery = connectionPtr->query(); @@ -419,6 +457,10 @@ namespace meta { "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); + } + if (!deleteTableQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); @@ -427,9 +469,7 @@ namespace meta { } //Scoped Connection -// ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); -// opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (mode_ != "single") { + if (mode_ != Options::MODE::SINGLE) { DeleteTableFiles(table_id); } @@ -451,7 +491,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } //soft delete table files Query deleteTableFilesQuery = connectionPtr->query(); @@ -462,6 +506,10 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); + } + if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); @@ -491,13 +539,22 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query describeTableQuery = connectionPtr->query(); describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << "FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); + } + res = describeTableQuery.store(); } //Scoped Connection @@ -513,7 +570,7 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -546,7 +603,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not @@ -555,6 +616,11 @@ namespace meta { "WHERE table_id = " << quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); + } + res = hasTableQuery.store(); } //Scoped Connection @@ -586,12 +652,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << "FROM Tables " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); + } + res = allTablesQuery.store(); } //Scoped Connection @@ -610,7 +685,7 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); table_schema_array.emplace_back(table_schema); } @@ -665,7 +740,11 @@ namespace meta { std::string date = std::to_string(file_schema.date_); { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query createTableFileQuery = connectionPtr->query(); @@ -674,6 +753,10 @@ namespace meta { quote << file_id << ", " << file_type << ", " << size << ", " << updated_time << ", " << created_on << ", " << date << ");"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + } + if (SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -725,12 +808,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); + } + res = filesToIndexQuery.store(); } //Scoped Connection @@ -801,7 +893,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } if (partition.empty()) { @@ -812,6 +908,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + } + res = filesToSearchQuery.store(); } else { @@ -832,6 +933,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + } + res = filesToSearchQuery.store(); } @@ -902,7 +1008,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query filesToMergeQuery = connectionPtr->query(); filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << @@ -910,6 +1020,11 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY size DESC" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); + } + res = filesToMergeQuery.store(); } //Scoped Connection @@ -987,13 +1102,22 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query getTableFileQuery = connectionPtr->query(); getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + } + res = getTableFileQuery.store(); } //Scoped Connection @@ -1062,13 +1186,22 @@ namespace meta { try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query archiveQuery = connectionPtr->query(); archiveQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); + } + if (!archiveQuery.exec()) { return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); } @@ -1105,12 +1238,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT SUM(size) AS sum " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); + } + res = getSizeQuery.store(); } //Scoped Connection @@ -1158,7 +1300,11 @@ namespace meta { bool status; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query discardFilesQuery = connectionPtr->query(); discardFilesQuery << "SELECT id, size " << @@ -1166,7 +1312,12 @@ namespace meta { "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << "LIMIT 10;"; -// std::cout << discardFilesQuery.str() << std::endl; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } + + // std::cout << discardFilesQuery.str() << std::endl; StoreQueryResult res = discardFilesQuery.store(); assert(res); @@ -1196,6 +1347,10 @@ namespace meta { "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } + status = discardFilesQuery.exec(); if (!status) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; @@ -1227,7 +1382,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query updateTableFileQuery = connectionPtr->query(); @@ -1235,6 +1394,11 @@ namespace meta { //clean thread will delete the file later updateTableFileQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + } + StoreQueryResult res = updateTableFileQuery.store(); assert(res && res.num_rows() <= 1); @@ -1268,7 +1432,11 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; -// std::cout << updateTableFileQuery.str() << std::endl; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + } + + // std::cout << updateTableFileQuery.str() << std::endl; if (!updateTableFileQuery.exec()) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; @@ -1300,7 +1468,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query updateTableFilesQuery = connectionPtr->query(); @@ -1316,6 +1488,11 @@ namespace meta { "WHERE table_id = " << quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + } + StoreQueryResult res = updateTableFilesQuery.store(); assert(res && res.num_rows() == 1); @@ -1351,6 +1528,10 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + } + if (!updateTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", @@ -1382,13 +1563,27 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use before creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); +// } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); @@ -1430,6 +1625,11 @@ namespace meta { idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << idsToDeleteStr << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("CleanUpFilesWithTTL Error", @@ -1452,12 +1652,25 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use before creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); +// } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id " << "FROM Tables " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); // std::cout << res.num_rows() << std::endl; @@ -1481,6 +1694,11 @@ namespace meta { idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << idsToDeleteStr << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", @@ -1507,12 +1725,20 @@ namespace meta { // std::lock_guard lock(mysql_mutex); try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + if (!cleanUpQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); @@ -1549,7 +1775,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query countQuery = connectionPtr->query(); countQuery << "SELECT size " << @@ -1558,6 +1788,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); + } + res = countQuery.store(); } //Scoped Connection @@ -1592,10 +1827,19 @@ namespace meta { } try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + } + if (dropTableQuery.exec()) { return Status::OK(); } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 4805076d450ac3f9bb8bfa06527c02838229a378..9ff8254b6006d6cc404dd77ae398d90c6963be97 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -22,7 +22,7 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions& options_, const std::string& mode); + MySQLMetaImpl(const DBMetaOptions& options_, const int& mode); virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -77,9 +77,9 @@ namespace meta { Status Initialize(); const DBMetaOptions options_; - const std::string mode_; + const int mode_; - std::shared_ptr mySQLConnectionPool_; + std::shared_ptr mysql_connection_pool_; bool safe_grab = false; // std::mutex connectionMutex_; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index dbe80f8d5f27546c2ab184e766ef485b613f3f46..609e3ca24520e65f89122083fea0779689288049 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -45,15 +45,23 @@ struct DBMetaOptions { std::string path; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); + bool sql_echo = false; }; // DBMetaOptions struct Options { + + typedef enum { + SINGLE, + CLUSTER, + READ_ONLY + } MODE; + Options(); uint16_t memory_sync_interval = 1; //unit: second uint16_t merge_trigger_number = 2; size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; - std::string mode; + int mode = MODE::SINGLE; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index ceaa6e813053c7c6a473735d569a8c6ddcc5a3cd..a3db0bf1103822d5d0babb5d5894719ff2b2d905 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,9 +23,30 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + std::string sql_echo = config.GetValue(CONFIG_DB_SQL_ECHO, "off"); + if (sql_echo == "on") { + opt.meta.sql_echo = true; + } + else if (sql_echo == "off") { + opt.meta.sql_echo = false; + } + else { + std::cout << "ERROR: sql_echo specified in db_config is not one of ['on', 'off']" << std::endl; + kill(0, SIGUSR1); + } + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (opt.mode != "single" && opt.mode != "cluster" && opt.mode != "read_only") { + std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + if (mode == "single") { + opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE; + } + else if (mode == "cluster") { + opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER; + } + else if (mode == "read_only") { + opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; + } + else { std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index f337275a4688721b2209ef07593795d8b4bf18a0..768430f02378e81267f6ea12e44d434cdd1ac566 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,6 +27,7 @@ static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; +static const std::string CONFIG_DB_SQL_ECHO = "sql_echo"; static const std::string CONFIG_LOG = "log_config";