diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 157103df22432acbd9016b41af6d7e8dea544be3..4a3288526c76692cdd7b96d5b9d81d24653a9674 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -14,6 +14,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-190 - use env variable to switch mem manager and fix cmake - MS-217 - Fix SQ8 row count bug - MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists +- MS-232 - add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 +- MS-233 - remove mem manager log ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index dbf0703173a12c223664a8f3607efe6e74e6608b..366188b547bb408fecb298dfa8f6ef5a64f90271 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -125,9 +125,6 @@ Status MemManager::InsertVectors(const std::string &table_id_, const float *vectors_, IDNumbers &vector_ids_) { - LOG(DEBUG) << "MemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << - ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); - std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 9aa5a85e9e99712fddf1dcd89103bff2d90ea094..dc22c0307c4498036cfd974241332c99ab24fc4c 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -24,251 +24,127 @@ #include "mysql++/mysql++.h" + namespace zilliz { namespace milvus { namespace engine { namespace meta { - using namespace mysqlpp; +using namespace mysqlpp; -// static std::unique_ptr connectionPtr(new Connection()); -// std::recursive_mutex mysql_mutex; -// -// std::unique_ptr& MySQLMetaImpl::getConnectionPtr() { -//// static std::recursive_mutex connectionMutex_; -// std::lock_guard lock(connectionMutex_); -// return connectionPtr; -// } - namespace { - Status HandleException(const std::string& desc, std::exception &e) { - ENGINE_LOG_ERROR << desc << ": " << e.what(); - return Status::DBTransactionError(desc, e.what()); - } +// - class MetricCollector { - public: - MetricCollector() { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - start_time_ = METRICS_NOW_TIME; - } +// - ~MetricCollector() { - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time_, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - } - private: - using TIME_POINT = std::chrono::system_clock::time_point; - TIME_POINT start_time_; - }; - } - Status MySQLMetaImpl::NextTableId(std::string &table_id) { - std::stringstream ss; - SimpleIDGenerator g; - ss << g.GetNextIDNumber(); - table_id = ss.str(); - return Status::OK(); - } +namespace { - Status MySQLMetaImpl::NextFileId(std::string &file_id) { - std::stringstream ss; - SimpleIDGenerator g; - ss << g.GetNextIDNumber(); - file_id = ss.str(); - return Status::OK(); - } +Status HandleException(const std::string &desc, std::exception &e) { + ENGINE_LOG_ERROR << desc << ": " << e.what(); + return Status::DBTransactionError(desc, e.what()); +} - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int& mode) - : options_(options_), - mode_(mode) { - Initialize(); +class MetricCollector { + public: + MetricCollector() { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + start_time_ = METRICS_NOW_TIME; } - Status MySQLMetaImpl::Initialize() { - -// std::lock_guard lock(mysql_mutex); - - if (!boost::filesystem::is_directory(options_.path)) { - auto ret = boost::filesystem::create_directory(options_.path); - if (!ret) { - ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; - return Status::DBTransactionError("Failed to create db directory", options_.path); - } - } - - std::string uri = options_.backend_uri; - - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; - - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") == std::string::npos) { - return Status::Error("URI's dialect is not MySQL"); - } - std::string username = pieces_match[2].str(); - std::string password = pieces_match[3].str(); - std::string serverAddress = pieces_match[4].str(); - unsigned int port = 0; - if (!pieces_match[5].str().empty()) { - port = std::stoi(pieces_match[5].str()); - } - std::string dbName = pieces_match[6].str(); -// std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; -// connectionPtr->set_option(new MultiStatementsOption(true)); -// connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); - int threadHint = std::thread::hardware_concurrency(); - int maxPoolSize = threadHint == 0 ? 8 : threadHint; - mysql_connection_pool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); -// std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; - ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); - try { - - if (mode_ != Options::MODE::READ_ONLY) { - CleanUp(); - } - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse(); -// if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { -// return Status::Error("DB connection failed: ", connectionPtr->error()); -// } - if (!connectionPtr->thread_aware()) { - ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; - return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); - } - Query InitializeQuery = connectionPtr->query(); - -// InitializeQuery << "SET max_allowed_packet=67108864;"; -// if (!InitializeQuery.exec()) { -// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); -// } - -// InitializeQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; - InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) UNIQUE NOT NULL, " << - "state INT NOT NULL, " << - "dimension SMALLINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "files_cnt BIGINT DEFAULT 0 NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "store_raw_data BOOL DEFAULT false NOT NULL);"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - - if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); - } - - InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "file_id VARCHAR(255) NOT NULL, " << - "file_type INT DEFAULT 0 NOT NULL, " << - "size BIGINT DEFAULT 0 NOT NULL, " << - "updated_time BIGINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "date INT DEFAULT -1 NOT NULL);"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - - if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); - } - } //Scoped Connection - -// //Consume all results to avoid "Commands out of sync" error -// while (InitializeQuery.more_results()) { -// InitializeQuery.store_next(); -// } - return Status::OK(); + ~MetricCollector() { + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time_, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } -// if (InitializeQuery.exec()) { -// std::cout << "XXXXXXXXXXXXXXXXXXXXXXXXX" << std::endl; -// while (InitializeQuery.more_results()) { -// InitializeQuery.store_next(); -// } -// return Status::OK(); -// } else { -// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); -// } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); - } catch (std::exception &e) { - return HandleException("Encounter exception during initialization", e); - } - } - else { - ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; - return Status::Error("Wrong URI format"); + private: + using TIME_POINT = std::chrono::system_clock::time_point; + TIME_POINT start_time_; +}; + +} + +Status MySQLMetaImpl::NextTableId(std::string &table_id) { + std::stringstream ss; + SimpleIDGenerator g; + ss << g.GetNextIDNumber(); + table_id = ss.str(); + return Status::OK(); +} + +Status MySQLMetaImpl::NextFileId(std::string &file_id) { + std::stringstream ss; + SimpleIDGenerator g; + ss << g.GetNextIDNumber(); + file_id = ss.str(); + return Status::OK(); +} + +MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode) + : options_(options_), + mode_(mode) { + Initialize(); +} + +Status MySQLMetaImpl::Initialize() { + + + if (!boost::filesystem::is_directory(options_.path)) { + auto ret = boost::filesystem::create_directory(options_.path); + if (!ret) { + ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; + return Status::DBTransactionError("Failed to create db directory", options_.path); } } -// PXU TODO: Temp solution. Will fix later - Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { - -// std::lock_guard lock(mysql_mutex); - - if (dates.empty()) { - return Status::OK(); + std::string uri = options_.backend_uri; + + std::string dialectRegex = "(.*)"; + std::string usernameRegex = "(.*)"; + std::string passwordRegex = "(.*)"; + std::string hostRegex = "(.*)"; + std::string portRegex = "(.*)"; + std::string dbNameRegex = "(.*)"; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + + usernameRegex + "\\:" + + passwordRegex + "\\@" + + hostRegex + "\\:" + + portRegex + "\\/" + + dbNameRegex; + std::regex uriRegex(uriRegexStr); + std::smatch pieces_match; + + if (std::regex_match(uri, pieces_match, uriRegex)) { + std::string dialect = pieces_match[1].str(); + std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); + if (dialect.find("mysql") == std::string::npos) { + return Status::Error("URI's dialect is not MySQL"); } - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; + std::string username = pieces_match[2].str(); + std::string password = pieces_match[3].str(); + std::string serverAddress = pieces_match[4].str(); + unsigned int port = 0; + if (!pieces_match[5].str().empty()) { + port = std::stoi(pieces_match[5].str()); } + std::string dbName = pieces_match[6].str(); - try { - auto yesterday = GetDateWithDelta(-1); + int threadHint = std::thread::hardware_concurrency(); + int maxPoolSize = threadHint == 0 ? 8 : threadHint; + mysql_connection_pool_ = + std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); - for (auto &date : dates) { - if (date >= yesterday) { - return Status::Error("Could not delete partitions within 2 days"); - } - } + ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); + try { - std::stringstream dateListSS; - for (auto &date : dates) { - dateListSS << std::to_string(date) << ", "; + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); } - std::string dateListStr = dateListSS.str(); - dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); @@ -277,1599 +153,1639 @@ namespace meta { return Status::Error("Failed to connect to database server"); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "date in (" << dateListStr << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); - - if (!dropPartitionsByDatesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", - dropPartitionsByDatesQuery.error()); + if (!connectionPtr->thread_aware()) { + ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; + return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); } - } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); - } - return Status::OK(); - } + Query InitializeQuery = connectionPtr->query(); - Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { -// std::lock_guard lock(mysql_mutex); + InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) UNIQUE NOT NULL, " << + "state INT NOT NULL, " << + "dimension SMALLINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "files_cnt BIGINT DEFAULT 0 NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "store_raw_data BOOL DEFAULT false NOT NULL);"; -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); - try { - - MetricCollector metric; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query createTableQuery = connectionPtr->query(); -// ENGINE_LOG_DEBUG << "Create Table in"; - if (table_schema.table_id_.empty()) { - NextTableId(table_schema.table_id_); - } else { - createTableQuery << "SELECT state FROM Tables " << - "WHERE table_id = " << quote << table_schema.table_id_ << ";"; -// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - - StoreQueryResult res = createTableQuery.store(); - - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (TableSchema::TO_DELETE == state) { - return Status::Error("Table already exists and it is in delete state, please wait a second"); - } - else { - return Status::AlreadyExist("Table already exists"); - } - } + InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "file_id VARCHAR(255) NOT NULL, " << + "file_type INT DEFAULT 0 NOT NULL, " << + "size BIGINT DEFAULT 0 NOT NULL, " << + "updated_time BIGINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "date INT DEFAULT -1 NOT NULL);"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } -// ENGINE_LOG_DEBUG << "Create Table start"; - - table_schema.files_cnt_ = 0; - table_schema.id_ = -1; - table_schema.created_on_ = utils::GetMicroSecTimeStamp(); - -// auto start_time = METRICS_NOW_TIME; + } //Scoped Connection - std::string id = "NULL"; //auto-increment - std::string table_id = table_schema.table_id_; - std::string state = std::to_string(table_schema.state_); - std::string dimension = std::to_string(table_schema.dimension_); - std::string created_on = std::to_string(table_schema.created_on_); - std::string files_cnt = "0"; - std::string engine_type = std::to_string(table_schema.engine_type_); - std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; - createTableQuery << "INSERT INTO Tables VALUES" << - "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; -// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - if (SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? -// std::cout << table_schema.id_ << std::endl; - //Consume all results to avoid "Commands out of sync" error -// while (createTableQuery.more_results()) { -// createTableQuery.store_next(); -// } - } else { - ENGINE_LOG_ERROR << "Add Table Error"; - return Status::DBTransactionError("Add Table Error", createTableQuery.error()); - } - } //Scoped Connection -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + return Status::OK(); - return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (const BadQuery& er) { + } catch (const BadQuery &er) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); - } catch (const Exception& er) { + ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); + } catch (const Exception &er) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); } catch (std::exception &e) { - return HandleException("Encounter exception when create table", e); + return HandleException("Encounter exception during initialization", e); } - - return Status::OK(); + } else { + ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; + return Status::Error("Wrong URI format"); } +} + +// PXU TODO: Temp solution. Will fix later +Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { + - Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { - // TODO + if (dates.empty()) { return Status::OK(); } - Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { - -// std::lock_guard lock(mysql_mutex); + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } - try { + try { - MetricCollector metric; + auto yesterday = GetDateWithDelta(-1); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + for (auto &date : dates) { + if (date >= yesterday) { + return Status::Error("Could not delete partitions within 2 days"); + } + } - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + std::stringstream dateListSS; + for (auto &date : dates) { + dateListSS << std::to_string(date) << ", "; + } + std::string dateListStr = dateListSS.str(); + dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - //soft delete table - Query deleteTableQuery = connectionPtr->query(); -// - deleteTableQuery << "UPDATE Tables " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << ";"; + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); - if (!deleteTableQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); - } + Query dropPartitionsByDatesQuery = connectionPtr->query(); - } //Scoped Connection + dropPartitionsByDatesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "date in (" << dateListStr << ");"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); - if (mode_ == Options::MODE::CLUSTER) { - DeleteTableFiles(table_id); + if (!dropPartitionsByDatesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", + dropPartitionsByDatesQuery.error()); } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); - } - - return Status::OK(); + } //Scoped Connection + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); } + return Status::OK(); +} - Status MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { - try { - MetricCollector metric; +Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + try { -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + MetricCollector metric; - //soft delete table files - Query deleteTableFilesQuery = connectionPtr->query(); - // - deleteTableFilesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - if (!deleteTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); - } - } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); - } - return Status::OK(); - } + Query createTableQuery = connectionPtr->query(); - Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { + if (table_schema.table_id_.empty()) { + NextTableId(table_schema.table_id_); + } else { + createTableQuery << "SELECT state FROM Tables " << + "WHERE table_id = " << quote << table_schema.table_id_ << ";"; -// std::lock_guard lock(mysql_mutex); - try { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - MetricCollector metric; + StoreQueryResult res = createTableQuery.store(); - StoreQueryResult res; + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (TableSchema::TO_DELETE == state) { + return Status::Error("Table already exists and it is in delete state, please wait a second"); + } else { + return Status::AlreadyExist("Table already exists"); + } + } + } - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + table_schema.files_cnt_ = 0; + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM Tables " << - "WHERE table_id = " << quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + std::string id = "NULL"; //auto-increment + std::string table_id = table_schema.table_id_; + std::string state = std::to_string(table_schema.state_); + std::string dimension = std::to_string(table_schema.dimension_); + std::string created_on = std::to_string(table_schema.created_on_); + std::string files_cnt = "0"; + std::string engine_type = std::to_string(table_schema.engine_type_); + std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); + createTableQuery << "INSERT INTO Tables VALUES" << + "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; - res = describeTableQuery.store(); - } //Scoped Connection - if (res.num_rows() == 1) { - const Row& resRow = res[0]; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - table_schema.id_ = resRow["id"]; //implicit conversion + if (SimpleResult res = createTableQuery.execute()) { + table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? - table_schema.dimension_ = resRow["dimension"]; + //Consume all results to avoid "Commands out of sync" error - table_schema.files_cnt_ = resRow["files_cnt"]; - table_schema.engine_type_ = resRow["engine_type"]; - int store_raw_data = resRow["store_raw_data"]; - table_schema.store_raw_data_ = (store_raw_data == 1); - } - else { - return Status::NotFound("Table " + table_schema.table_id_ + " not found"); + } else { + ENGINE_LOG_ERROR << "Add Table Error"; + return Status::DBTransactionError("Add Table Error", createTableQuery.error()); } + } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); - } - return Status::OK(); - } - Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { -// std::lock_guard lock(mysql_mutex); - try { + return utils::CreateTablePath(options_, table_schema.table_id_); - MetricCollector metric; + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + } catch (std::exception &e) { + return HandleException("Encounter exception when create table", e); + } - StoreQueryResult res; + return Status::OK(); +} - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); +Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { + // TODO + return Status::OK(); +} - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } +Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query hasTableQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << - "WHERE table_id = " << quote << table_id << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; + try { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); + MetricCollector metric; - res = hasTableQuery.store(); - } //Scoped Connection + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - int check = res[0]["check"]; - has_or_not = (check == 1); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); - } - return Status::OK(); - } - Status MySQLMetaImpl::AllTables(std::vector& table_schema_array) { -// std::lock_guard lock(mysql_mutex); - try { + //soft delete table + Query deleteTableQuery = connectionPtr->query(); +// + deleteTableQuery << "UPDATE Tables " << + "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << ";"; - MetricCollector metric; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); - StoreQueryResult res; + if (!deleteTableQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); + } - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + } //Scoped Connection - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + if (mode_ == Options::MODE::CLUSTER) { + DeleteTableFiles(table_id); + } - Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM Tables " << - "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); + } - ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); + return Status::OK(); +} - res = allTablesQuery.store(); - } //Scoped Connection +Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { + try { + MetricCollector metric; - for (auto& resRow : res) { - TableSchema table_schema; + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - table_schema.id_ = resRow["id"]; //implicit conversion - std::string table_id; - resRow["table_id"].to_string(table_id); - table_schema.table_id_ = table_id; - table_schema.dimension_ = resRow["dimension"]; - table_schema.files_cnt_ = resRow["files_cnt"]; - table_schema.engine_type_ = resRow["engine_type"]; + //soft delete table files + Query deleteTableFilesQuery = connectionPtr->query(); + // + deleteTableFilesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - int store_raw_data = resRow["store_raw_data"]; - table_schema.store_raw_data_ = (store_raw_data == 1); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); - table_schema_array.emplace_back(table_schema); + if (!deleteTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); - } - - return Status::OK(); + } //Scoped Connection + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); } - Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); +Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { - if (file_schema.date_ == EmptyDate) { - file_schema.date_ = Meta::GetDate(); - } - TableSchema table_schema; - table_schema.table_id_ = file_schema.table_id_; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - try { + try { - MetricCollector metric; + MetricCollector metric; - NextFileId(file_schema.file_id_); - file_schema.file_type_ = TableFileSchema::NEW; - file_schema.dimension_ = table_schema.dimension_; - file_schema.size_ = 0; - file_schema.created_on_ = utils::GetMicroSecTimeStamp(); - file_schema.updated_time_ = file_schema.created_on_; - file_schema.engine_type_ = table_schema.engine_type_; - utils::GetTableFilePath(options_, file_schema); + StoreQueryResult res; - std::string id = "NULL"; //auto-increment - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + Query describeTableQuery = connectionPtr->query(); + describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM Tables " << + "WHERE table_id = " << quote << table_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); - Query createTableFileQuery = connectionPtr->query(); + res = describeTableQuery.store(); + } //Scoped Connection - createTableFileQuery << "INSERT INTO TableFiles VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << size << ", " << - updated_time << ", " << created_on << ", " << date << ");"; + if (res.num_rows() == 1) { + const Row &resRow = res[0]; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + table_schema.id_ = resRow["id"]; //implicit conversion - if (SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + table_schema.dimension_ = resRow["dimension"]; - //Consume all results to avoid "Commands out of sync" error -// while (createTableFileQuery.more_results()) { -// createTableFileQuery.store_next(); -// } - } else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; - return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); - } - } // Scoped Connection + table_schema.files_cnt_ = resRow["files_cnt"]; - return utils::CreateTableFilePath(options_, file_schema); + table_schema.engine_type_ = resRow["engine_type"]; - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (std::exception& ex) { - return HandleException("Encounter exception when create table file", ex); + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); + } else { + return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } - return Status::OK(); + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); } - Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { - -// std::lock_guard lock(mysql_mutex); + return Status::OK(); +} - files.clear(); +Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { - try { - MetricCollector metric; + try { - StoreQueryResult res; + MetricCollector metric; - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + StoreQueryResult res; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); + Query hasTableQuery = connectionPtr->query(); + //since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS " << + "(SELECT 1 FROM Tables " << + "WHERE table_id = " << quote << table_id << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; - res = filesToIndexQuery.store(); - } //Scoped Connection + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); - std::map groups; - TableFileSchema table_file; - for (auto& resRow : res) { + res = hasTableQuery.store(); + } //Scoped Connection - table_file.id_ = resRow["id"]; //implicit conversion + int check = res[0]["check"]; + has_or_not = (check == 1); - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + } - table_file.engine_type_ = resRow["engine_type"]; + return Status::OK(); +} - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; +Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { - table_file.file_type_ = resRow["file_type"]; - table_file.size_ = resRow["size"]; + try { - table_file.date_ = resRow["date"]; + MetricCollector metric; - auto groupItr = groups.find(table_file.table_id_); - if (groupItr == groups.end()) { - TableSchema table_schema; - table_schema.table_id_ = table_file.table_id_; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - groups[table_file.table_id_] = table_schema; -// std::cout << table_schema.dimension_ << std::endl; - } - table_file.dimension_ = groups[table_file.table_id_].dimension_; + StoreQueryResult res; - utils::GetTableFilePath(options_, table_file); + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - files.push_back(table_file); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); - } - return Status::OK(); - } - Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { + Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM Tables " << + "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; -// std::lock_guard lock(mysql_mutex); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); - files.clear(); + res = allTablesQuery.store(); + } //Scoped Connection - try { + for (auto &resRow : res) { + TableSchema table_schema; - MetricCollector metric; + table_schema.id_ = resRow["id"]; //implicit conversion - StoreQueryResult res; + std::string table_id; + resRow["table_id"].to_string(table_id); + table_schema.table_id_ = table_id; - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + table_schema.dimension_ = resRow["dimension"]; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + table_schema.files_cnt_ = resRow["files_cnt"]; - if (partition.empty()) { + table_schema.engine_type_ = resRow["engine_type"]; - Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + table_schema_array.emplace_back(table_schema); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); + } - res = filesToSearchQuery.store(); + return Status::OK(); +} - } else { +Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { - Query filesToSearchQuery = connectionPtr->query(); - std::stringstream partitionListSS; - for (auto &date : partition) { - partitionListSS << std::to_string(date) << ", "; - } - std::string partitionListStr = partitionListSS.str(); - partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " + if (file_schema.date_ == EmptyDate) { + file_schema.date_ = Meta::GetDate(); + } + TableSchema table_schema; + table_schema.table_id_ = file_schema.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "date IN (" << partitionListStr << ") AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + try { + + MetricCollector metric; + + NextFileId(file_schema.file_id_); + file_schema.file_type_ = TableFileSchema::NEW; + file_schema.dimension_ = table_schema.dimension_; + file_schema.size_ = 0; + file_schema.created_on_ = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = file_schema.created_on_; + file_schema.engine_type_ = table_schema.engine_type_; + utils::GetTableFilePath(options_, file_schema); + + std::string id = "NULL"; //auto-increment + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - res = filesToSearchQuery.store(); - } - } //Scoped Connection + Query createTableFileQuery = connectionPtr->query(); - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; + createTableFileQuery << "INSERT INTO TableFiles VALUES" << + "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + quote << file_id << ", " << file_type << ", " << size << ", " << + updated_time << ", " << created_on << ", " << date << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + + if (SimpleResult res = createTableFileQuery.execute()) { + file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + + //Consume all results to avoid "Commands out of sync" error + + + + } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; + return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); } + } // Scoped Connection + + return utils::CreateTableFilePath(options_, file_schema); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (std::exception &ex) { + return HandleException("Encounter exception when create table file", ex); + } - TableFileSchema table_file; - for (auto& resRow : res) { + return Status::OK(); +} - table_file.id_ = resRow["id"]; //implicit conversion +Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { - std::string table_id_str; - resRow["table_id"].to_string(table_id_str); - table_file.table_id_ = table_id_str; - table_file.engine_type_ = resRow["engine_type"]; + files.clear(); - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; + try { - table_file.file_type_ = resRow["file_type"]; + MetricCollector metric; - table_file.size_ = resRow["size"]; + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - table_file.date_ = resRow["date"]; - table_file.dimension_ = table_schema.dimension_; + Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - utils::GetTableFilePath(options_, table_file); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); + res = filesToIndexQuery.store(); + } //Scoped Connection + + std::map groups; + TableFileSchema table_file; + for (auto &resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; + + table_file.engine_type_ = resRow["engine_type"]; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + auto groupItr = groups.find(table_file.table_id_); + if (groupItr == groups.end()) { + TableSchema table_schema; + table_schema.table_id_ = table_file.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; } + groups[table_file.table_id_] = table_schema; - files[table_file.date_].push_back(table_file); } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); - } + table_file.dimension_ = groups[table_file.table_id_].dimension_; - return Status::OK(); + utils::GetTableFilePath(options_, table_file); + + files.push_back(table_file); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); } - Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, + return Status::OK(); +} + +Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, + const DatesT &partition, DatePartionedTableFilesSchema &files) { -// std::lock_guard lock(mysql_mutex); - files.clear(); + files.clear(); - try { - MetricCollector metric; + try { - StoreQueryResult res; + MetricCollector metric; - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + StoreQueryResult res; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY size DESC" << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); + if (partition.empty()) { - res = filesToMergeQuery.store(); - } //Scoped Connection + Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + + res = filesToSearchQuery.store(); + + } else { + + Query filesToSearchQuery = connectionPtr->query(); + + std::stringstream partitionListSS; + for (auto &date : partition) { + partitionListSS << std::to_string(date) << ", "; + } + std::string partitionListStr = partitionListSS.str(); + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " + + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "date IN (" << partitionListStr << ") AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + + res = filesToSearchQuery.store(); - if (!status.ok()) { - return status; } + } //Scoped Connection - TableFileSchema table_file; - for (auto& resRow : res) { + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } - table_file.id_ = resRow["id"]; //implicit conversion + TableFileSchema table_file; + for (auto &resRow : res) { - std::string table_id_str; - resRow["table_id"].to_string(table_id_str); - table_file.table_id_ = table_id_str; + table_file.id_ = resRow["id"]; //implicit conversion - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; + std::string table_id_str; + resRow["table_id"].to_string(table_id_str); + table_file.table_id_ = table_id_str; - table_file.file_type_ = resRow["file_type"]; + table_file.engine_type_ = resRow["engine_type"]; - table_file.size_ = resRow["size"]; + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; - table_file.date_ = resRow["date"]; + table_file.file_type_ = resRow["file_type"]; - table_file.dimension_ = table_schema.dimension_; + table_file.size_ = resRow["size"]; - utils::GetTableFilePath(options_, table_file); + table_file.date_ = resRow["date"]; - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); - } + table_file.dimension_ = table_schema.dimension_; + + utils::GetTableFilePath(options_, table_file); - files[table_file.date_].push_back(table_file); + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + files[table_file.date_].push_back(table_file); } - - return Status::OK(); + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); } - Status MySQLMetaImpl::GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); +Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { - if (ids.empty()) { - return Status::OK(); - } - std::stringstream idSS; - for (auto& id : ids) { - idSS << "id = " << std::to_string(id) << " OR "; - } - std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + files.clear(); - try { - - StoreQueryResult res; + try { + MetricCollector metric; - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + StoreQueryResult res; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ");"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << " " << + "ORDER BY size DESC" << ";"; - res = getTableFileQuery.store(); - } //Scoped Connection + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } + res = filesToMergeQuery.store(); + } //Scoped Connection - for (auto& resRow : res) { + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); - TableFileSchema file_schema; + if (!status.ok()) { + return status; + } - file_schema.id_ = resRow["id"]; + TableFileSchema table_file; + for (auto &resRow : res) { - file_schema.table_id_ = table_id; + table_file.id_ = resRow["id"]; //implicit conversion - file_schema.engine_type_ = resRow["engine_type"]; + std::string table_id_str; + resRow["table_id"].to_string(table_id_str); + table_file.table_id_ = table_id_str; - std::string file_id; - resRow["file_id"].to_string(file_id); - file_schema.file_id_ = file_id; + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; - file_schema.file_type_ = resRow["file_type"]; + table_file.file_type_ = resRow["file_type"]; - file_schema.size_ = resRow["size"]; + table_file.size_ = resRow["size"]; - file_schema.date_ = resRow["date"]; + table_file.date_ = resRow["date"]; - file_schema.dimension_ = table_schema.dimension_; + table_file.dimension_ = table_schema.dimension_; - utils::GetTableFilePath(options_, file_schema); + utils::GetTableFilePath(options_, table_file); - table_files.emplace_back(file_schema); + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); + + files[table_file.date_].push_back(table_file); } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { + + + if (ids.empty()) { return Status::OK(); } -// PXU TODO: Support Swap - Status MySQLMetaImpl::Archive() { + std::stringstream idSS; + for (auto &id : ids) { + idSS << "id = " << std::to_string(id) << " OR "; + } + std::string idStr = idSS.str(); + idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " -// std::lock_guard lock(mysql_mutex); + try { - auto &criterias = options_.archive_conf.GetCriterias(); - if (criterias.empty()) { - return Status::OK(); + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(" << idStr << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + + res = getTableFileQuery.store(); + } //Scoped Connection + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; } - for (auto& kv : criterias) { - auto &criteria = kv.first; - auto &limit = kv.second; - if (criteria == "days") { - size_t usecs = limit * D_SEC * US_PS; - long now = utils::GetMicroSecTimeStamp(); + for (auto &resRow : res) { - try { + TableFileSchema file_schema; - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + file_schema.id_ = resRow["id"]; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + file_schema.table_id_ = table_id; -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + file_schema.engine_type_ = resRow["engine_type"]; - Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE created_on < " << std::to_string(now - usecs) << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + std::string file_id; + resRow["file_id"].to_string(file_id); + file_schema.file_id_ = file_id; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); + file_schema.file_type_ = resRow["file_type"]; - if (!archiveQuery.exec()) { - return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); - } + file_schema.size_ = resRow["size"]; - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); - } - } - if (criteria == "disk") { - uint64_t sum = 0; - Size(sum); + file_schema.date_ = resRow["date"]; - auto to_delete = (sum - limit * G); - DiscardFiles(to_delete); - } - } + file_schema.dimension_ = table_schema.dimension_; - return Status::OK(); + utils::GetTableFilePath(options_, file_schema); + + table_files.emplace_back(file_schema); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); } - Status MySQLMetaImpl::Size(uint64_t &result) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); +// PXU TODO: Support Swap +Status MySQLMetaImpl::Archive() { - result = 0; - try { - StoreQueryResult res; + auto &criterias = options_.archive_conf.GetCriterias(); + if (criterias.empty()) { + return Status::OK(); + } + + for (auto &kv : criterias) { + auto &criteria = kv.first; + auto &limit = kv.second; + if (criteria == "days") { + size_t usecs = limit * D_SEC * US_PS; + long now = utils::GetMicroSecTimeStamp(); + + try { - { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { return Status::Error("Failed to connect to database server"); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << - "FROM TableFiles " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + Query archiveQuery = connectionPtr->query(); + archiveQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE created_on < " << std::to_string(now - usecs) << " AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); - res = getSizeQuery.store(); - } //Scoped Connection + if (!archiveQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); + } -// if (!res) { -//// std::cout << "result is NULL" << std::endl; -// return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", getSizeQuery.error()); -// } - if (res.empty()) { - result = 0; -// std::cout << "result = 0" << std::endl; - } - else { - result = res[0]["sum"]; -// std::cout << "result = " << std::to_string(result) << std::endl; + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); } + if (criteria == "disk") { + uint64_t sum = 0; + Size(sum); - return Status::OK(); + auto to_delete = (sum - limit * G); + DiscardFiles(to_delete); + } } - Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); +Status MySQLMetaImpl::Size(uint64_t &result) { - if (to_discard_size <= 0) { -// std::cout << "in" << std::endl; - return Status::OK(); - } - ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; - try { + result = 0; + try { - MetricCollector metric; + StoreQueryResult res; - bool status; + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + Query getSizeQuery = connectionPtr->query(); + getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << + "FROM TableFiles " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, size " << - "FROM TableFiles " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "ORDER BY id ASC " << - "LIMIT 10;"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + res = getSizeQuery.store(); + } //Scoped Connection - // std::cout << discardFilesQuery.str() << std::endl; - StoreQueryResult res = discardFilesQuery.store(); - if (res.num_rows() == 0) { - return Status::OK(); - } +// - TableFileSchema table_file; - std::stringstream idsToDiscardSS; - for (auto &resRow : res) { - if (to_discard_size <= 0) { - break; - } - table_file.id_ = resRow["id"]; - table_file.size_ = resRow["size"]; - idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; - ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ - << " table_file.size=" << table_file.size_; - to_discard_size -= table_file.size_; - } - std::string idsToDiscardStr = idsToDiscardSS.str(); - idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " + if (res.empty()) { + result = 0; - discardFilesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE " << idsToDiscardStr << ";"; + } else { + result = res[0]["sum"]; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } - status = discardFilesQuery.exec(); - if (!status) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); - } - } //Scoped Connection + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); + } - return DiscardFiles(to_discard_size); + return Status::OK(); +} - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); - } +Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { + + + if (to_discard_size <= 0) { + + return Status::OK(); } + ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; - //ZR: this function assumes all fields in file_schema have value - Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { + try { -// std::lock_guard lock(mysql_mutex); + MetricCollector metric; - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - try { + bool status; - MetricCollector metric; + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + Query discardFilesQuery = connectionPtr->query(); + discardFilesQuery << "SELECT id, size " << + "FROM TableFiles " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "ORDER BY id ASC " << + "LIMIT 10;"; - Query updateTableFileQuery = connectionPtr->query(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - //if the table has been deleted, just mark the table file as TO_DELETE - //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM Tables " << - "WHERE table_id = " << quote << file_schema.table_id_ << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + StoreQueryResult res = discardFilesQuery.store(); - StoreQueryResult res = updateTableFileQuery.store(); + if (res.num_rows() == 0) { + return Status::OK(); + } - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (state == TableSchema::TO_DELETE) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - } else { - file_schema.file_type_ = TableFileSchema::TO_DELETE; + TableFileSchema table_file; + std::stringstream idsToDiscardSS; + for (auto &resRow : res) { + if (to_discard_size <= 0) { + break; } + table_file.id_ = resRow["id"]; + table_file.size_ = resRow["size"]; + idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; + ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ + << " table_file.size=" << table_file.size_; + to_discard_size -= table_file.size_; + } - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); + std::string idsToDiscardStr = idsToDiscardSS.str(); + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " - updateTableFileQuery << "UPDATE TableFiles " << - "SET table_id = " << quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << - "file_type = " << file_type << ", " << - "size = " << size << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - - // std::cout << updateTableFileQuery.str() << std::endl; - - if (!updateTableFileQuery.exec()) { - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", - updateTableFileQuery.error()); - } - } //Scoped Connection + discardFilesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << + "WHERE " << idsToDiscardStr << ";"; - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); - } - return Status::OK(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { - // TODO - return Status::OK(); + status = discardFilesQuery.exec(); + if (!status) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); + } + } //Scoped Connection + + return DiscardFiles(to_discard_size); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); } +} - Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +//ZR: this function assumes all fields in file_schema have value +Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { -// std::lock_guard lock(mysql_mutex); - try { - MetricCollector metric; + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + try { - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + MetricCollector metric; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - Query updateTableFilesQuery = connectionPtr->query(); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - std::map has_tables; - for (auto &file_schema : files) { - if (has_tables.find(file_schema.table_id_) != has_tables.end()) { - continue; - } + Query updateTableFileQuery = connectionPtr->query(); - updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << - "WHERE table_id = " << quote << file_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; + //if the table has been deleted, just mark the table file as TO_DELETE + //clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM Tables " << + "WHERE table_id = " << quote << file_schema.table_id_ << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - StoreQueryResult res = updateTableFilesQuery.store(); + StoreQueryResult res = updateTableFileQuery.store(); - int check = res[0]["check"]; - has_tables[file_schema.table_id_] = (check == 1); + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (state == TableSchema::TO_DELETE) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; } + } else { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); - for (auto &file_schema : files) { + updateTableFileQuery << "UPDATE TableFiles " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + + + if (!updateTableFileQuery.exec()) { + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", + updateTableFileQuery.error()); + } + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); + } + return Status::OK(); +} - if (!has_tables[file_schema.table_id_]) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); - - updateTableFilesQuery << "UPDATE TableFiles " << - "SET table_id = " << quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << - "file_type = " << file_type << ", " << - "size = " << size << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - - if (!updateTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", - updateTableFilesQuery.error()); - } - } - } //Scoped Connection +Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { + try { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } - return Status::OK(); - } - Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { -// static int b_count = 0; -// b_count++; -// std::cout << "CleanUpFilesWithTTL: " << b_count << std::endl; -// std::lock_guard lock(mysql_mutex); + Query updateTableFilesToIndexQuery = connectionPtr->query(); - auto now = utils::GetMicroSecTimeStamp(); - try { - MetricCollector metric; + updateTableFilesToIndexQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; - { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFilesToIndexQuery.str(); -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use before creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what()); + } - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + return Status::OK(); +} - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } +Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); -// } - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << - "FROM TableFiles " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << - "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + try { + MetricCollector metric; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - TableFileSchema table_file; - std::vector idsToDelete; + Query updateTableFilesQuery = connectionPtr->query(); - for (auto &resRow : res) { + std::map has_tables; + for (auto &file_schema : files) { - table_file.id_ = resRow["id"]; //implicit conversion + if (has_tables.find(file_schema.table_id_) != has_tables.end()) { + continue; + } - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; + updateTableFilesQuery << "SELECT EXISTS " << + "(SELECT 1 FROM Tables " << + "WHERE table_id = " << quote << file_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - table_file.date_ = resRow["date"]; + StoreQueryResult res = updateTableFilesQuery.store(); - utils::DeleteTableFilePath(options_, table_file); + int check = res[0]["check"]; + has_tables[file_schema.table_id_] = (check == 1); + } - ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " - << table_file.location_ << std::endl; + for (auto &file_schema : files) { - idsToDelete.emplace_back(std::to_string(table_file.id_)); + if (!has_tables[file_schema.table_id_]) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; } + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - if (!idsToDelete.empty()) { + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); - std::stringstream idsToDeleteSS; - for (auto &id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } + updateTableFilesQuery << "UPDATE TableFiles " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + + if (!updateTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", + updateTableFilesQuery.error()); + } + } + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); + } + return Status::OK(); +} - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; +Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("CleanUpFilesWithTTL Error", - cleanUpFilesWithTTLQuery.error()); - } - } - } //Scoped Connection + auto now = utils::GetMicroSecTimeStamp(); + try { + MetricCollector metric; - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } + { - try { - MetricCollector metric; - { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use before creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); -// } + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << + "FROM TableFiles " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << + "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id " << - "FROM Tables " << - "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); -// std::cout << res.num_rows() << std::endl; + TableFileSchema table_file; + std::vector idsToDelete; - if (!res.empty()) { + for (auto &resRow : res) { - std::stringstream idsToDeleteSS; - for (auto &resRow : res) { - size_t id = resRow["id"]; - std::string table_id; - resRow["table_id"].to_string(table_id); + table_file.id_ = resRow["id"]; //implicit conversion - utils::DeleteTablePath(options_, table_id); + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; - idsToDeleteSS << "id = " << std::to_string(id) << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + table_file.date_ = resRow["date"]; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); - } + utils::DeleteTableFilePath(options_, table_file); + + ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " + << table_file.location_ << std::endl; + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + } + + if (!idsToDelete.empty()) { + + std::stringstream idsToDeleteSS; + for (auto &id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; } - } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << + idsToDeleteStr << ";"; - return Status::OK(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("CleanUpFilesWithTTL Error", + cleanUpFilesWithTTLQuery.error()); + } + } + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } - Status MySQLMetaImpl::CleanUp() { + try { + MetricCollector metric; + + { -// std::lock_guard lock(mysql_mutex); - try { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { return Status::Error("Failed to connect to database server"); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << "TableFiles" << ";"; + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id " << + "FROM Tables " << + "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - StoreQueryResult res = cleanUpQuery.store(); if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + std::stringstream idsToDeleteSS; + for (auto &resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + utils::DeleteTablePath(options_, table_id); + + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } - } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << + idsToDeleteStr << ";"; - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - return Status::OK(); + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } + } + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } - Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { - -// std::lock_guard lock(mysql_mutex); - - try { - MetricCollector metric; + return Status::OK(); +} - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); +Status MySQLMetaImpl::CleanUp() { - if (!status.ok()) { - return status; - } - StoreQueryResult res; + try { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } + Query cleanUpQuery = connectionPtr->query(); + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << + "AND table_name = " << quote << "TableFiles" << ";"; - Query countQuery = connectionPtr->query(); - countQuery << "SELECT size " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); + StoreQueryResult res = cleanUpQuery.store(); - res = countQuery.store(); - } //Scoped Connection + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - result = 0; - for (auto &resRow : res) { - size_t size = resRow["size"]; - result += size; - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - if (table_schema.dimension_ <= 0) { - std::stringstream errorMsg; - errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) << ", table_id = " << table_id; - ENGINE_LOG_ERROR << errorMsg.str(); - return Status::Error(errorMsg.str()); + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } - result /= table_schema.dimension_; - result /= sizeof(float); - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); } - return Status::OK(); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); } - Status MySQLMetaImpl::DropAll() { + return Status::OK(); +} + +Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { -// std::lock_guard lock(mysql_mutex); - if (boost::filesystem::is_directory(options_.path)) { - boost::filesystem::remove_all(options_.path); + try { + MetricCollector metric; + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + + if (!status.ok()) { + return status; } - try { + StoreQueryResult res; + + { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { return Status::Error("Failed to connect to database server"); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + Query countQuery = connectionPtr->query(); + countQuery << "SELECT size " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); - if (dropTableQuery.exec()) { - return Status::OK(); - } - else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; - return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); - } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); + res = countQuery.store(); + } //Scoped Connection + + result = 0; + for (auto &resRow : res) { + size_t size = resRow["size"]; + result += size; } - return Status::OK(); + + if (table_schema.dimension_ <= 0) { + std::stringstream errorMsg; + errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) + << ", table_id = " << table_id; + ENGINE_LOG_ERROR << errorMsg.str(); + return Status::Error(errorMsg.str()); + } + result /= table_schema.dimension_; + result /= sizeof(float); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); } + return Status::OK(); +} + +Status MySQLMetaImpl::DropAll() { - MySQLMetaImpl::~MySQLMetaImpl() { -// std::lock_guard lock(mysql_mutex); - if (mode_ != Options::MODE::READ_ONLY) { - CleanUp(); + + if (boost::filesystem::is_directory(options_.path)) { + boost::filesystem::remove_all(options_.path); + } + try { + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } + + + Query dropTableQuery = connectionPtr->query(); + dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + + if (dropTableQuery.exec()) { + return Status::OK(); + } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; + return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); + } + return Status::OK(); +} + +MySQLMetaImpl::~MySQLMetaImpl() { + + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); } +} } // namespace meta } // namespace engine diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index e4903d75a9457bf22bd95244bcd18730ea4870a1..405d3771fd4009d0f347c5977625c1724db82a24 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -29,9 +29,6 @@ Status NewMemManager::InsertVectors(const std::string &table_id_, std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - LOG(DEBUG) << "NewMemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << - ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); - std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index b8e049f9d4fb01711e890f02d67d06e1c6362622..a12afbfd80d77caaa5e2b4a07158257312480fc3 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -63,7 +63,7 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; - float maximum_memory = 4 * ONE_GB; + size_t maximum_memory = 4 * ONE_GB; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 66a1d83479ef34e1d2f2c88e371445e4aca582fc..c01e1acda3abee8bd4c34ae9486cbf1cddc3e214 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -28,11 +28,11 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - float maximum_memory = config.GetFloatValue(CONFIG_MAXMIMUM_MEMORY); + int64_t maximum_memory = config.GetInt64Value(CONFIG_MAXMIMUM_MEMORY); if (maximum_memory > 1.0) { opt.maximum_memory = maximum_memory * engine::ONE_GB; } - else { + else if (maximum_memory != 0) { //if maximum_memory = 0, set it to default std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; kill(0, SIGUSR1); }