Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

从无法访问的项目Fork

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 e950fff6 编写于 作者: Z zhiru
浏览文件
操作

add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 


Former-commit-id: f8943149b9737b2f0107308aa96a7f77acdf9ce4
上级 5f3533ba
隐藏空白更改
内联 并排
Showing with 1299 addition and 1383 deletion
cpp/src/db/MySQLMetaImpl.cpp
浏览文件 @ e950fff6
...@@ -24,251 +24,127 @@ ...@@ -24,251 +24,127 @@
#include "mysql++/mysql++.h" #include "mysql++/mysql++.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
namespace meta { namespace meta {
using namespace mysqlpp; using namespace mysqlpp;
// static std::unique_ptr<Connection> connectionPtr(new Connection());
// std::recursive_mutex mysql_mutex;
//
// std::unique_ptr<Connection>& MySQLMetaImpl::getConnectionPtr() {
//// static std::recursive_mutex connectionMutex_;
// std::lock_guard<std::recursive_mutex> lock(connectionMutex_);
// return connectionPtr;
// }
namespace {
Status HandleException(const std::string& desc, std::exception &e) { //
ENGINE_LOG_ERROR << desc << ": " << e.what();
return Status::DBTransactionError(desc, e.what());
}
class MetricCollector { //
public:
MetricCollector() {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
start_time_ = METRICS_NOW_TIME;
}
~MetricCollector() {
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time_, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
}
private:
using TIME_POINT = std::chrono::system_clock::time_point;
TIME_POINT start_time_;
};
}
Status MySQLMetaImpl::NextTableId(std::string &table_id) { namespace {
std::stringstream ss;
SimpleIDGenerator g;
ss << g.GetNextIDNumber();
table_id = ss.str();
return Status::OK();
}
Status MySQLMetaImpl::NextFileId(std::string &file_id) { Status HandleException(const std::string &desc, std::exception &e) {
std::stringstream ss; ENGINE_LOG_ERROR << desc << ": " << e.what();
SimpleIDGenerator g; return Status::DBTransactionError(desc, e.what());
ss << g.GetNextIDNumber(); }
file_id = ss.str();
return Status::OK();
}
MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int& mode) class MetricCollector {
: options_(options_), public:
mode_(mode) { MetricCollector() {
Initialize(); server::Metrics::GetInstance().MetaAccessTotalIncrement();
start_time_ = METRICS_NOW_TIME;
} }
Status MySQLMetaImpl::Initialize() { ~MetricCollector() {
auto end_time = METRICS_NOW_TIME;
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); auto total_time = METRICS_MICROSECONDS(start_time_, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
if (!boost::filesystem::is_directory(options_.path)) { }
auto ret = boost::filesystem::create_directory(options_.path);
if (!ret) {
ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path;
return Status::DBTransactionError("Failed to create db directory", options_.path);
}
}
std::string uri = options_.backend_uri;
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + "\\:\\/\\/" +
usernameRegex + "\\:" +
passwordRegex + "\\@" +
hostRegex + "\\:" +
portRegex + "\\/" +
dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos) {
return Status::Error("URI's dialect is not MySQL");
}
std::string username = pieces_match[2].str();
std::string password = pieces_match[3].str();
std::string serverAddress = pieces_match[4].str();
unsigned int port = 0;
if (!pieces_match[5].str().empty()) {
port = std::stoi(pieces_match[5].str());
}
std::string dbName = pieces_match[6].str();
// std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl;
// connectionPtr->set_option(new MultiStatementsOption(true));
// connectionPtr->set_option(new mysqlpp::ReconnectOption(true));
int threadHint = std::thread::hardware_concurrency();
int maxPoolSize = threadHint == 0 ? 8 : threadHint;
mysql_connection_pool_ = std::make_shared<MySQLConnectionPool>(dbName, username, password, serverAddress, port, maxPoolSize);
// std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl;
ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize);
try {
if (mode_ != Options::MODE::READ_ONLY) {
CleanUp();
}
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse();
// if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) {
// return Status::Error("DB connection failed: ", connectionPtr->error());
// }
if (!connectionPtr->thread_aware()) {
ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it.";
return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it.");
}
Query InitializeQuery = connectionPtr->query();
// InitializeQuery << "SET max_allowed_packet=67108864;";
// if (!InitializeQuery.exec()) {
// return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
// }
// InitializeQuery << "DROP TABLE IF EXISTS Tables, TableFiles;";
InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" <<
"id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
"table_id VARCHAR(255) UNIQUE NOT NULL, " <<
"state INT NOT NULL, " <<
"dimension SMALLINT NOT NULL, " <<
"created_on BIGINT NOT NULL, " <<
"files_cnt BIGINT DEFAULT 0 NOT NULL, " <<
"engine_type INT DEFAULT 1 NOT NULL, " <<
"store_raw_data BOOL DEFAULT false NOT NULL);";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
if (!InitializeQuery.exec()) {
return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
}
InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" <<
"id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
"table_id VARCHAR(255) NOT NULL, " <<
"engine_type INT DEFAULT 1 NOT NULL, " <<
"file_id VARCHAR(255) NOT NULL, " <<
"file_type INT DEFAULT 0 NOT NULL, " <<
"size BIGINT DEFAULT 0 NOT NULL, " <<
"updated_time BIGINT NOT NULL, " <<
"created_on BIGINT NOT NULL, " <<
"date INT DEFAULT -1 NOT NULL);";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
if (!InitializeQuery.exec()) {
return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
}
} //Scoped Connection
// //Consume all results to avoid "Commands out of sync" error
// while (InitializeQuery.more_results()) {
// InitializeQuery.store_next();
// }
return Status::OK();
// if (InitializeQuery.exec()) { private:
// std::cout << "XXXXXXXXXXXXXXXXXXXXXXXXX" << std::endl; using TIME_POINT = std::chrono::system_clock::time_point;
// while (InitializeQuery.more_results()) { TIME_POINT start_time_;
// InitializeQuery.store_next(); };
// }
// return Status::OK(); }
// } else {
// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); Status MySQLMetaImpl::NextTableId(std::string &table_id) {
// } std::stringstream ss;
} catch (const BadQuery& er) { SimpleIDGenerator g;
// Handle any query errors ss << g.GetNextIDNumber();
ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); table_id = ss.str();
return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); return Status::OK();
} catch (const Exception& er) { }
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); Status MySQLMetaImpl::NextFileId(std::string &file_id) {
return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); std::stringstream ss;
} catch (std::exception &e) { SimpleIDGenerator g;
return HandleException("Encounter exception during initialization", e); ss << g.GetNextIDNumber();
} file_id = ss.str();
} return Status::OK();
else { }
ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri;
return Status::Error("Wrong URI format"); MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode)
: options_(options_),
mode_(mode) {
Initialize();
}
Status MySQLMetaImpl::Initialize() {
if (!boost::filesystem::is_directory(options_.path)) {
auto ret = boost::filesystem::create_directory(options_.path);
if (!ret) {
ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path;
return Status::DBTransactionError("Failed to create db directory", options_.path);
} }
} }
// PXU TODO: Temp solution. Will fix later std::string uri = options_.backend_uri;
Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id,
const DatesT &dates) { std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
if (dates.empty()) { std::string portRegex = "(.*)";
return Status::OK(); std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + "\\:\\/\\/" +
usernameRegex + "\\:" +
passwordRegex + "\\@" +
hostRegex + "\\:" +
portRegex + "\\/" +
dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos) {
return Status::Error("URI's dialect is not MySQL");
} }
std::string username = pieces_match[2].str();
TableSchema table_schema; std::string password = pieces_match[3].str();
table_schema.table_id_ = table_id; std::string serverAddress = pieces_match[4].str();
auto status = DescribeTable(table_schema); unsigned int port = 0;
if (!status.ok()) { if (!pieces_match[5].str().empty()) {
return status; port = std::stoi(pieces_match[5].str());
} }
std::string dbName = pieces_match[6].str();
try {
auto yesterday = GetDateWithDelta(-1); int threadHint = std::thread::hardware_concurrency();
int maxPoolSize = threadHint == 0 ? 8 : threadHint;
mysql_connection_pool_ =
std::make_shared<MySQLConnectionPool>(dbName, username, password, serverAddress, port, maxPoolSize);
for (auto &date : dates) { ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize);
if (date >= yesterday) { try {
return Status::Error("Could not delete partitions within 2 days");
}
}
std::stringstream dateListSS; if (mode_ != Options::MODE::READ_ONLY) {
for (auto &date : dates) { CleanUp();
dateListSS << std::to_string(date) << ", ";
} }
std::string dateListStr = dateListSS.str();
dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", "
{ {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
...@@ -277,1599 +153,1639 @@ namespace meta { ...@@ -277,1599 +153,1639 @@ namespace meta {
return Status::Error("Failed to connect to database server"); return Status::Error("Failed to connect to database server");
} }
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query dropPartitionsByDatesQuery = connectionPtr->query();
dropPartitionsByDatesQuery << "UPDATE TableFiles " << if (!connectionPtr->thread_aware()) {
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it.";
"WHERE table_id = " << quote << table_id << " AND " << return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it.");
"date in (" << dateListStr << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str();
if (!dropPartitionsByDatesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES";
return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES",
dropPartitionsByDatesQuery.error());
} }
} //Scoped Connection Query InitializeQuery = connectionPtr->query();
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what());
}
return Status::OK();
}
Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" <<
"id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
"table_id VARCHAR(255) UNIQUE NOT NULL, " <<
"state INT NOT NULL, " <<
"dimension SMALLINT NOT NULL, " <<
"created_on BIGINT NOT NULL, " <<
"files_cnt BIGINT DEFAULT 0 NOT NULL, " <<
"engine_type INT DEFAULT 1 NOT NULL, " <<
"store_raw_data BOOL DEFAULT false NOT NULL);";
// server::Metrics::GetInstance().MetaAccessTotalIncrement(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
try {
MetricCollector metric;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { if (!InitializeQuery.exec()) {
return Status::Error("Failed to connect to database server"); return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
} }
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" <<
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); "id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
// } "table_id VARCHAR(255) NOT NULL, " <<
"engine_type INT DEFAULT 1 NOT NULL, " <<
Query createTableQuery = connectionPtr->query(); "file_id VARCHAR(255) NOT NULL, " <<
// ENGINE_LOG_DEBUG << "Create Table in"; "file_type INT DEFAULT 0 NOT NULL, " <<
if (table_schema.table_id_.empty()) { "size BIGINT DEFAULT 0 NOT NULL, " <<
NextTableId(table_schema.table_id_); "updated_time BIGINT NOT NULL, " <<
} else { "created_on BIGINT NOT NULL, " <<
createTableQuery << "SELECT state FROM Tables " << "date INT DEFAULT -1 NOT NULL);";
"WHERE table_id = " << quote << table_schema.table_id_ << ";";
// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); if (!InitializeQuery.exec()) {
return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
StoreQueryResult res = createTableQuery.store();
if (res.num_rows() == 1) {
int state = res[0]["state"];
if (TableSchema::TO_DELETE == state) {
return Status::Error("Table already exists and it is in delete state, please wait a second");
}
else {
return Status::AlreadyExist("Table already exists");
}
}
} }
// ENGINE_LOG_DEBUG << "Create Table start"; } //Scoped Connection
table_schema.files_cnt_ = 0;
table_schema.id_ = -1;
table_schema.created_on_ = utils::GetMicroSecTimeStamp();
// auto start_time = METRICS_NOW_TIME;
std::string id = "NULL"; //auto-increment
std::string table_id = table_schema.table_id_;
std::string state = std::to_string(table_schema.state_);
std::string dimension = std::to_string(table_schema.dimension_);
std::string created_on = std::to_string(table_schema.created_on_);
std::string files_cnt = "0";
std::string engine_type = std::to_string(table_schema.engine_type_);
std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false";
createTableQuery << "INSERT INTO Tables VALUES" <<
"(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " <<
created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");";
// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str();
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
if (SimpleResult res = createTableQuery.execute()) {
table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()?
// std::cout << table_schema.id_ << std::endl;
//Consume all results to avoid "Commands out of sync" error
// while (createTableQuery.more_results()) {
// createTableQuery.store_next();
// }
} else {
ENGINE_LOG_ERROR << "Add Table Error";
return Status::DBTransactionError("Add Table Error", createTableQuery.error());
}
} //Scoped Connection
// auto end_time = METRICS_NOW_TIME; return Status::OK();
// auto total_time = METRICS_MICROSECONDS(start_time, end_time);
// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
return utils::CreateTablePath(options_, table_schema.table_id_);
} catch (const BadQuery& er) { } catch (const BadQuery &er) {
// Handle any query errors // Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what());
} catch (const Exception& er) { } catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions // Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what());
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when create table", e); return HandleException("Encounter exception during initialization", e);
} }
} else {
return Status::OK(); ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri;
return Status::Error("Wrong URI format");
} }
}
// PXU TODO: Temp solution. Will fix later
Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id,
const DatesT &dates) {
Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { if (dates.empty()) {
// TODO
return Status::OK(); return Status::OK();
} }
Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { TableSchema table_schema;
table_schema.table_id_ = table_id;
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
try { try {
MetricCollector metric; auto yesterday = GetDateWithDelta(-1);
{ for (auto &date : dates) {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (date >= yesterday) {
return Status::Error("Could not delete partitions within 2 days");
}
}
if (connectionPtr == nullptr) { std::stringstream dateListSS;
return Status::Error("Failed to connect to database server"); for (auto &date : dates) {
} dateListSS << std::to_string(date) << ", ";
}
std::string dateListStr = dateListSS.str();
dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", "
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
// }
//soft delete table if (connectionPtr == nullptr) {
Query deleteTableQuery = connectionPtr->query(); return Status::Error("Failed to connect to database server");
// }
deleteTableQuery << "UPDATE Tables " <<
"SET state = " << std::to_string(TableSchema::TO_DELETE) << " " <<
"WHERE table_id = " << quote << table_id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str();
if (!deleteTableQuery.exec()) { Query dropPartitionsByDatesQuery = connectionPtr->query();
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE";
return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error());
}
} //Scoped Connection dropPartitionsByDatesQuery << "UPDATE TableFiles " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"date in (" << dateListStr << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str();
if (mode_ == Options::MODE::CLUSTER) { if (!dropPartitionsByDatesQuery.exec()) {
DeleteTableFiles(table_id); ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES";
return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES",
dropPartitionsByDatesQuery.error());
} }
} //Scoped Connection
} catch (const BadQuery& er) { } catch (const BadQuery &er) {
// Handle any query errors // Handle any query errors
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what());
} catch (const Exception& er) { } catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions // Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what());
}
return Status::OK();
} }
return Status::OK();
}
Status MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) {
try {
MetricCollector metric;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { try {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { MetricCollector metric;
// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
//soft delete table files {
Query deleteTableFilesQuery = connectionPtr->query(); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
//
deleteTableFilesQuery << "UPDATE TableFiles " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " <<
"updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
if (!deleteTableFilesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES";
return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error());
}
} //Scoped Connection
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what());
}
return Status::OK(); Query createTableQuery = connectionPtr->query();
}
Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { if (table_schema.table_id_.empty()) {
NextTableId(table_schema.table_id_);
} else {
createTableQuery << "SELECT state FROM Tables " <<
"WHERE table_id = " << quote << table_schema.table_id_ << ";";
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
try { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
MetricCollector metric; StoreQueryResult res = createTableQuery.store();
StoreQueryResult res; if (res.num_rows() == 1) {
int state = res[0]["state"];
if (TableSchema::TO_DELETE == state) {
return Status::Error("Table already exists and it is in delete state, please wait a second");
} else {
return Status::AlreadyExist("Table already exists");
}
}
}
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { table_schema.files_cnt_ = 0;
return Status::Error("Failed to connect to database server"); table_schema.id_ = -1;
} table_schema.created_on_ = utils::GetMicroSecTimeStamp();
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query describeTableQuery = connectionPtr->query(); std::string id = "NULL"; //auto-increment
describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << std::string table_id = table_schema.table_id_;
"FROM Tables " << std::string state = std::to_string(table_schema.state_);
"WHERE table_id = " << quote << table_schema.table_id_ << " " << std::string dimension = std::to_string(table_schema.dimension_);
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; std::string created_on = std::to_string(table_schema.created_on_);
std::string files_cnt = "0";
std::string engine_type = std::to_string(table_schema.engine_type_);
std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); createTableQuery << "INSERT INTO Tables VALUES" <<
"(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " <<
created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");";
res = describeTableQuery.store();
} //Scoped Connection
if (res.num_rows() == 1) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
const Row& resRow = res[0];
table_schema.id_ = resRow["id"]; //implicit conversion if (SimpleResult res = createTableQuery.execute()) {
table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()?
table_schema.dimension_ = resRow["dimension"]; //Consume all results to avoid "Commands out of sync" error
table_schema.files_cnt_ = resRow["files_cnt"];
table_schema.engine_type_ = resRow["engine_type"];
int store_raw_data = resRow["store_raw_data"]; } else {
table_schema.store_raw_data_ = (store_raw_data == 1); ENGINE_LOG_ERROR << "Add Table Error";
} return Status::DBTransactionError("Add Table Error", createTableQuery.error());
else {
return Status::NotFound("Table " + table_schema.table_id_ + " not found");
} }
} //Scoped Connection
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what());
}
return Status::OK();
}
Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
try { return utils::CreateTablePath(options_, table_schema.table_id_);
MetricCollector metric; } catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what());
} catch (std::exception &e) {
return HandleException("Encounter exception when create table", e);
}
StoreQueryResult res; return Status::OK();
}
{ Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); // TODO
return Status::OK();
}
if (connectionPtr == nullptr) { Status MySQLMetaImpl::DeleteTable(const std::string &table_id) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query hasTableQuery = connectionPtr->query(); try {
//since table_id is a unique column we just need to check whether it exists or not
hasTableQuery << "SELECT EXISTS " <<
"(SELECT 1 FROM Tables " <<
"WHERE table_id = " << quote << table_id << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); MetricCollector metric;
res = hasTableQuery.store(); {
} //Scoped Connection ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
int check = res[0]["check"]; if (connectionPtr == nullptr) {
has_or_not = (check == 1); return Status::Error("Failed to connect to database server");
}
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what());
}
return Status::OK();
}
Status MySQLMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
try { //soft delete table
Query deleteTableQuery = connectionPtr->query();
//
deleteTableQuery << "UPDATE Tables " <<
"SET state = " << std::to_string(TableSchema::TO_DELETE) << " " <<
"WHERE table_id = " << quote << table_id << ";";
MetricCollector metric; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str();
StoreQueryResult res; if (!deleteTableQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE";
return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error());
}
{ } //Scoped Connection
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { if (mode_ == Options::MODE::CLUSTER) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); DeleteTableFiles(table_id);
// } }
Query allTablesQuery = connectionPtr->query(); } catch (const BadQuery &er) {
allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << // Handle any query errors
"FROM Tables " << ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what();
"WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what());
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); return Status::OK();
}
res = allTablesQuery.store(); Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) {
} //Scoped Connection try {
MetricCollector metric;
for (auto& resRow : res) { {
TableSchema table_schema; ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
table_schema.id_ = resRow["id"]; //implicit conversion
std::string table_id;
resRow["table_id"].to_string(table_id);
table_schema.table_id_ = table_id;
table_schema.dimension_ = resRow["dimension"];
table_schema.files_cnt_ = resRow["files_cnt"];
table_schema.engine_type_ = resRow["engine_type"]; //soft delete table files
Query deleteTableFilesQuery = connectionPtr->query();
//
deleteTableFilesQuery << "UPDATE TableFiles " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " <<
"updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
int store_raw_data = resRow["store_raw_data"]; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str();
table_schema.store_raw_data_ = (store_raw_data == 1);
table_schema_array.emplace_back(table_schema); if (!deleteTableFilesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES";
return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error());
} }
} catch (const BadQuery& er) { } //Scoped Connection
// Handle any query errors } catch (const BadQuery &er) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); // Handle any query errors
return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what();
} catch (const Exception& er) { return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what());
// Catch-all for any other MySQL++ exceptions } catch (const Exception &er) {
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); // Catch-all for any other MySQL++ exceptions
return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what();
} return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what());
return Status::OK();
} }
Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { return Status::OK();
}
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) {
if (file_schema.date_ == EmptyDate) {
file_schema.date_ = Meta::GetDate();
}
TableSchema table_schema;
table_schema.table_id_ = file_schema.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
try { try {
MetricCollector metric; MetricCollector metric;
NextFileId(file_schema.file_id_); StoreQueryResult res;
file_schema.file_type_ = TableFileSchema::NEW;
file_schema.dimension_ = table_schema.dimension_;
file_schema.size_ = 0;
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.engine_type_ = table_schema.engine_type_;
utils::GetTableFilePath(options_, file_schema);
std::string id = "NULL"; //auto-increment {
std::string table_id = file_schema.table_id_; ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
std::string engine_type = std::to_string(file_schema.engine_type_);
std::string file_id = file_schema.file_id_;
std::string file_type = std::to_string(file_schema.file_type_);
std::string size = std::to_string(file_schema.size_);
std::string updated_time = std::to_string(file_schema.updated_time_);
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
{ if (connectionPtr == nullptr) {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); return Status::Error("Failed to connect to database server");
}
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { Query describeTableQuery = connectionPtr->query();
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " <<
// } "FROM Tables " <<
"WHERE table_id = " << quote << table_schema.table_id_ << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str();
Query createTableFileQuery = connectionPtr->query(); res = describeTableQuery.store();
} //Scoped Connection
createTableFileQuery << "INSERT INTO TableFiles VALUES" << if (res.num_rows() == 1) {
"(" << id << ", " << quote << table_id << ", " << engine_type << ", " << const Row &resRow = res[0];
quote << file_id << ", " << file_type << ", " << size << ", " <<
updated_time << ", " << created_on << ", " << date << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); table_schema.id_ = resRow["id"]; //implicit conversion
if (SimpleResult res = createTableFileQuery.execute()) { table_schema.dimension_ = resRow["dimension"];
file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()?
//Consume all results to avoid "Commands out of sync" error table_schema.files_cnt_ = resRow["files_cnt"];
// while (createTableFileQuery.more_results()) {
// createTableFileQuery.store_next();
// }
} else {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE";
return Status::DBTransactionError("Add file Error", createTableFileQuery.error());
}
} // Scoped Connection
return utils::CreateTableFilePath(options_, file_schema); table_schema.engine_type_ = resRow["engine_type"];
} catch (const BadQuery& er) { int store_raw_data = resRow["store_raw_data"];
// Handle any query errors table_schema.store_raw_data_ = (store_raw_data == 1);
ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); } else {
return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); return Status::NotFound("Table " + table_schema.table_id_ + " not found");
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what());
} catch (std::exception& ex) {
return HandleException("Encounter exception when create table file", ex);
} }
return Status::OK(); } catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what());
} }
Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { return Status::OK();
}
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
files.clear(); Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
try {
MetricCollector metric; try {
StoreQueryResult res; MetricCollector metric;
{ StoreQueryResult res;
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { {
return Status::Error("Failed to connect to database server"); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { if (connectionPtr == nullptr) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); return Status::Error("Failed to connect to database server");
// } }
Query filesToIndexQuery = connectionPtr->query();
filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); Query hasTableQuery = connectionPtr->query();
//since table_id is a unique column we just need to check whether it exists or not
hasTableQuery << "SELECT EXISTS " <<
"(SELECT 1 FROM Tables " <<
"WHERE table_id = " << quote << table_id << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
res = filesToIndexQuery.store(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str();
} //Scoped Connection
std::map<std::string, TableSchema> groups; res = hasTableQuery.store();
TableFileSchema table_file; } //Scoped Connection
for (auto& resRow : res) {
table_file.id_ = resRow["id"]; //implicit conversion int check = res[0]["check"];
has_or_not = (check == 1);
std::string table_id; } catch (const BadQuery &er) {
resRow["table_id"].to_string(table_id); // Handle any query errors
table_file.table_id_ = table_id; ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what());
}
table_file.engine_type_ = resRow["engine_type"]; return Status::OK();
}
std::string file_id; Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
table_file.file_type_ = resRow["file_type"];
table_file.size_ = resRow["size"]; try {
table_file.date_ = resRow["date"]; MetricCollector metric;
auto groupItr = groups.find(table_file.table_id_); StoreQueryResult res;
if (groupItr == groups.end()) {
TableSchema table_schema;
table_schema.table_id_ = table_file.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
groups[table_file.table_id_] = table_schema;
// std::cout << table_schema.dimension_ << std::endl;
}
table_file.dimension_ = groups[table_file.table_id_].dimension_;
utils::GetTableFilePath(options_, table_file); {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
files.push_back(table_file); if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
} }
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what());
}
return Status::OK();
}
Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, Query allTablesQuery = connectionPtr->query();
const DatesT &partition, allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " <<
DatePartionedTableFilesSchema &files) { "FROM Tables " <<
"WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str();
files.clear(); res = allTablesQuery.store();
} //Scoped Connection
try { for (auto &resRow : res) {
TableSchema table_schema;
MetricCollector metric; table_schema.id_ = resRow["id"]; //implicit conversion
StoreQueryResult res; std::string table_id;
resRow["table_id"].to_string(table_id);
table_schema.table_id_ = table_id;
{ table_schema.dimension_ = resRow["dimension"];
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { table_schema.files_cnt_ = resRow["files_cnt"];
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
if (partition.empty()) { table_schema.engine_type_ = resRow["engine_type"];
Query filesToSearchQuery = connectionPtr->query(); int store_raw_data = resRow["store_raw_data"];
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << table_schema.store_raw_data_ = (store_raw_data == 1);
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); table_schema_array.emplace_back(table_schema);
}
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what());
}
res = filesToSearchQuery.store(); return Status::OK();
}
} else { Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
Query filesToSearchQuery = connectionPtr->query();
std::stringstream partitionListSS; if (file_schema.date_ == EmptyDate) {
for (auto &date : partition) { file_schema.date_ = Meta::GetDate();
partitionListSS << std::to_string(date) << ", "; }
} TableSchema table_schema;
std::string partitionListStr = partitionListSS.str(); table_schema.table_id_ = file_schema.table_id_;
partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << try {
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " << MetricCollector metric;
"date IN (" << partitionListStr << ") AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << NextFileId(file_schema.file_id_);
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << file_schema.file_type_ = TableFileSchema::NEW;
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; file_schema.dimension_ = table_schema.dimension_;
file_schema.size_ = 0;
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.engine_type_ = table_schema.engine_type_;
utils::GetTableFilePath(options_, file_schema);
std::string id = "NULL"; //auto-increment
std::string table_id = file_schema.table_id_;
std::string engine_type = std::to_string(file_schema.engine_type_);
std::string file_id = file_schema.file_id_;
std::string file_type = std::to_string(file_schema.file_type_);
std::string size = std::to_string(file_schema.size_);
std::string updated_time = std::to_string(file_schema.updated_time_);
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
res = filesToSearchQuery.store();
} Query createTableFileQuery = connectionPtr->query();
} //Scoped Connection
TableSchema table_schema; createTableFileQuery << "INSERT INTO TableFiles VALUES" <<
table_schema.table_id_ = table_id; "(" << id << ", " << quote << table_id << ", " << engine_type << ", " <<
auto status = DescribeTable(table_schema); quote << file_id << ", " << file_type << ", " << size << ", " <<
if (!status.ok()) { updated_time << ", " << created_on << ", " << date << ");";
return status;
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str();
if (SimpleResult res = createTableFileQuery.execute()) {
file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()?
//Consume all results to avoid "Commands out of sync" error
} else {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE";
return Status::DBTransactionError("Add file Error", createTableFileQuery.error());
} }
} // Scoped Connection
return utils::CreateTableFilePath(options_, file_schema);
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what());
} catch (std::exception &ex) {
return HandleException("Encounter exception when create table file", ex);
}
TableFileSchema table_file; return Status::OK();
for (auto& resRow : res) { }
table_file.id_ = resRow["id"]; //implicit conversion Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
std::string table_id_str;
resRow["table_id"].to_string(table_id_str);
table_file.table_id_ = table_id_str;
table_file.engine_type_ = resRow["engine_type"]; files.clear();
std::string file_id; try {
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
table_file.file_type_ = resRow["file_type"]; MetricCollector metric;
table_file.size_ = resRow["size"]; StoreQueryResult res;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
table_file.date_ = resRow["date"];
table_file.dimension_ = table_schema.dimension_; Query filesToIndexQuery = connectionPtr->query();
filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";";
utils::GetTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str();
auto dateItr = files.find(table_file.date_); res = filesToIndexQuery.store();
if (dateItr == files.end()) { } //Scoped Connection
files[table_file.date_] = TableFilesSchema();
std::map<std::string, TableSchema> groups;
TableFileSchema table_file;
for (auto &resRow : res) {
table_file.id_ = resRow["id"]; //implicit conversion
std::string table_id;
resRow["table_id"].to_string(table_id);
table_file.table_id_ = table_id;
table_file.engine_type_ = resRow["engine_type"];
std::string file_id;
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
table_file.file_type_ = resRow["file_type"];
table_file.size_ = resRow["size"];
table_file.date_ = resRow["date"];
auto groupItr = groups.find(table_file.table_id_);
if (groupItr == groups.end()) {
TableSchema table_schema;
table_schema.table_id_ = table_file.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
} }
groups[table_file.table_id_] = table_schema;
files[table_file.date_].push_back(table_file);
} }
} catch (const BadQuery& er) { table_file.dimension_ = groups[table_file.table_id_].dimension_;
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what());
}
return Status::OK(); utils::GetTableFilePath(options_, table_file);
files.push_back(table_file);
}
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what());
} }
Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, return Status::OK();
}
Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
const DatesT &partition,
DatePartionedTableFilesSchema &files) { DatePartionedTableFilesSchema &files) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
files.clear(); files.clear();
try { try {
MetricCollector metric;
StoreQueryResult res; MetricCollector metric;
{ StoreQueryResult res;
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { {
return Status::Error("Failed to connect to database server"); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { if (connectionPtr == nullptr) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); return Status::Error("Failed to connect to database server");
// } }
Query filesToMergeQuery = connectionPtr->query();
filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type = " << std::to_string(TableFileSchema::RAW) << " " <<
"ORDER BY size DESC" << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); if (partition.empty()) {
res = filesToMergeQuery.store(); Query filesToSearchQuery = connectionPtr->query();
} //Scoped Connection filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
TableSchema table_schema; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str();
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema); res = filesToSearchQuery.store();
} else {
Query filesToSearchQuery = connectionPtr->query();
std::stringstream partitionListSS;
for (auto &date : partition) {
partitionListSS << std::to_string(date) << ", ";
}
std::string partitionListStr = partitionListSS.str();
partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", "
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"date IN (" << partitionListStr << ") AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str();
res = filesToSearchQuery.store();
if (!status.ok()) {
return status;
} }
} //Scoped Connection
TableFileSchema table_file; TableSchema table_schema;
for (auto& resRow : res) { table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
table_file.id_ = resRow["id"]; //implicit conversion TableFileSchema table_file;
for (auto &resRow : res) {
std::string table_id_str; table_file.id_ = resRow["id"]; //implicit conversion
resRow["table_id"].to_string(table_id_str);
table_file.table_id_ = table_id_str;
std::string file_id; std::string table_id_str;
resRow["file_id"].to_string(file_id); resRow["table_id"].to_string(table_id_str);
table_file.file_id_ = file_id; table_file.table_id_ = table_id_str;
table_file.file_type_ = resRow["file_type"]; table_file.engine_type_ = resRow["engine_type"];
table_file.size_ = resRow["size"]; std::string file_id;
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
table_file.date_ = resRow["date"]; table_file.file_type_ = resRow["file_type"];
table_file.dimension_ = table_schema.dimension_; table_file.size_ = resRow["size"];
utils::GetTableFilePath(options_, table_file); table_file.date_ = resRow["date"];
auto dateItr = files.find(table_file.date_); table_file.dimension_ = table_schema.dimension_;
if (dateItr == files.end()) {
files[table_file.date_] = TableFilesSchema(); utils::GetTableFilePath(options_, table_file);
}
files[table_file.date_].push_back(table_file); auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date_] = TableFilesSchema();
} }
} catch (const BadQuery& er) { files[table_file.date_].push_back(table_file);
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what());
} }
} catch (const BadQuery &er) {
return Status::OK(); // Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what());
} }
Status MySQLMetaImpl::GetTableFiles(const std::string& table_id, return Status::OK();
const std::vector<size_t>& ids, }
TableFilesSchema& table_files) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
DatePartionedTableFilesSchema &files) {
if (ids.empty()) {
return Status::OK();
}
std::stringstream idSS; files.clear();
for (auto& id : ids) {
idSS << "id = " << std::to_string(id) << " OR ";
}
std::string idStr = idSS.str();
idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR "
try { try {
MetricCollector metric;
StoreQueryResult res;
{ StoreQueryResult res;
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { {
return Status::Error("Failed to connect to database server"); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { if (connectionPtr == nullptr) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); return Status::Error("Failed to connect to database server");
// } }
Query getTableFileQuery = connectionPtr->query();
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(" << idStr << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); Query filesToMergeQuery = connectionPtr->query();
filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type = " << std::to_string(TableFileSchema::RAW) << " " <<
"ORDER BY size DESC" << ";";
res = getTableFileQuery.store(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str();
} //Scoped Connection
TableSchema table_schema; res = filesToMergeQuery.store();
table_schema.table_id_ = table_id; } //Scoped Connection
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
for (auto& resRow : res) { TableSchema table_schema;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
TableFileSchema file_schema; if (!status.ok()) {
return status;
}
file_schema.id_ = resRow["id"]; TableFileSchema table_file;
for (auto &resRow : res) {
file_schema.table_id_ = table_id; table_file.id_ = resRow["id"]; //implicit conversion
file_schema.engine_type_ = resRow["engine_type"]; std::string table_id_str;
resRow["table_id"].to_string(table_id_str);
table_file.table_id_ = table_id_str;
std::string file_id; std::string file_id;
resRow["file_id"].to_string(file_id); resRow["file_id"].to_string(file_id);
file_schema.file_id_ = file_id; table_file.file_id_ = file_id;
file_schema.file_type_ = resRow["file_type"]; table_file.file_type_ = resRow["file_type"];
file_schema.size_ = resRow["size"]; table_file.size_ = resRow["size"];
file_schema.date_ = resRow["date"]; table_file.date_ = resRow["date"];
file_schema.dimension_ = table_schema.dimension_; table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, file_schema); utils::GetTableFilePath(options_, table_file);
table_files.emplace_back(file_schema); auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date_] = TableFilesSchema();
} }
} catch (const BadQuery& er) {
// Handle any query errors files[table_file.date_].push_back(table_file);
ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what());
} }
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what());
}
return Status::OK();
}
Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) {
if (ids.empty()) {
return Status::OK(); return Status::OK();
} }
// PXU TODO: Support Swap std::stringstream idSS;
Status MySQLMetaImpl::Archive() { for (auto &id : ids) {
idSS << "id = " << std::to_string(id) << " OR ";
}
std::string idStr = idSS.str();
idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR "
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); try {
auto &criterias = options_.archive_conf.GetCriterias(); StoreQueryResult res;
if (criterias.empty()) {
return Status::OK(); {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
Query getTableFileQuery = connectionPtr->query();
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(" << idStr << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str();
res = getTableFileQuery.store();
} //Scoped Connection
TableSchema table_schema;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
} }
for (auto& kv : criterias) { for (auto &resRow : res) {
auto &criteria = kv.first;
auto &limit = kv.second;
if (criteria == "days") {
size_t usecs = limit * D_SEC * US_PS;
long now = utils::GetMicroSecTimeStamp();
try { TableFileSchema file_schema;
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); file_schema.id_ = resRow["id"];
if (connectionPtr == nullptr) { file_schema.table_id_ = table_id;
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { file_schema.engine_type_ = resRow["engine_type"];
// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query archiveQuery = connectionPtr->query(); std::string file_id;
archiveQuery << "UPDATE TableFiles " << resRow["file_id"].to_string(file_id);
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << file_schema.file_id_ = file_id;
"WHERE created_on < " << std::to_string(now - usecs) << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); file_schema.file_type_ = resRow["file_type"];
if (!archiveQuery.exec()) { file_schema.size_ = resRow["size"];
return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error());
}
} catch (const BadQuery& er) { file_schema.date_ = resRow["date"];
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what());
}
}
if (criteria == "disk") {
uint64_t sum = 0;
Size(sum);
auto to_delete = (sum - limit * G); file_schema.dimension_ = table_schema.dimension_;
DiscardFiles(to_delete);
}
}
return Status::OK(); utils::GetTableFilePath(options_, file_schema);
table_files.emplace_back(file_schema);
}
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what());
} }
Status MySQLMetaImpl::Size(uint64_t &result) { return Status::OK();
}
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); // PXU TODO: Support Swap
Status MySQLMetaImpl::Archive() {
result = 0;
try {
StoreQueryResult res; auto &criterias = options_.archive_conf.GetCriterias();
if (criterias.empty()) {
return Status::OK();
}
for (auto &kv : criterias) {
auto &criteria = kv.first;
auto &limit = kv.second;
if (criteria == "days") {
size_t usecs = limit * D_SEC * US_PS;
long now = utils::GetMicroSecTimeStamp();
try {
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server"); return Status::Error("Failed to connect to database server");
} }
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query getSizeQuery = connectionPtr->query(); Query archiveQuery = connectionPtr->query();
getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << archiveQuery << "UPDATE TableFiles " <<
"FROM TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; "WHERE created_on < " << std::to_string(now - usecs) << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str();
res = getSizeQuery.store(); if (!archiveQuery.exec()) {
} //Scoped Connection return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error());
}
// if (!res) { } catch (const BadQuery &er) {
//// std::cout << "result is NULL" << std::endl; // Handle any query errors
// return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", getSizeQuery.error()); ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what();
// } return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what());
if (res.empty()) { } catch (const Exception &er) {
result = 0; // Catch-all for any other MySQL++ exceptions
// std::cout << "result = 0" << std::endl; ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what();
} return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what());
else {
result = res[0]["sum"];
// std::cout << "result = " << std::to_string(result) << std::endl;
} }
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what());
} }
if (criteria == "disk") {
uint64_t sum = 0;
Size(sum);
return Status::OK(); auto to_delete = (sum - limit * G);
DiscardFiles(to_delete);
}
} }
Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { return Status::OK();
}
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); Status MySQLMetaImpl::Size(uint64_t &result) {
if (to_discard_size <= 0) {
// std::cout << "in" << std::endl;
return Status::OK();
}
ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size;
try { result = 0;
try {
MetricCollector metric; StoreQueryResult res;
bool status; {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
{ if (connectionPtr == nullptr) {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); return Status::Error("Failed to connect to database server");
}
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { Query getSizeQuery = connectionPtr->query();
// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " <<
// } "FROM TableFiles " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
Query discardFilesQuery = connectionPtr->query(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str();
discardFilesQuery << "SELECT id, size " <<
"FROM TableFiles " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " <<
"ORDER BY id ASC " <<
"LIMIT 10;";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); res = getSizeQuery.store();
} //Scoped Connection
// std::cout << discardFilesQuery.str() << std::endl;
StoreQueryResult res = discardFilesQuery.store();
if (res.num_rows() == 0) { //
return Status::OK();
}
TableFileSchema table_file;
std::stringstream idsToDiscardSS;
for (auto &resRow : res) {
if (to_discard_size <= 0) {
break;
}
table_file.id_ = resRow["id"];
table_file.size_ = resRow["size"];
idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR ";
ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_
<< " table_file.size=" << table_file.size_;
to_discard_size -= table_file.size_;
}
std::string idsToDiscardStr = idsToDiscardSS.str(); if (res.empty()) {
idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " result = 0;
discardFilesQuery << "UPDATE TableFiles " << } else {
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << result = res[0]["sum"];
"updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"WHERE " << idsToDiscardStr << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); }
status = discardFilesQuery.exec(); } catch (const BadQuery &er) {
if (!status) { // Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what());
} } catch (const Exception &er) {
} //Scoped Connection // Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what());
}
return DiscardFiles(to_discard_size); return Status::OK();
}
} catch (const BadQuery& er) { Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); if (to_discard_size <= 0) {
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions return Status::OK();
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what());
}
} }
ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size;
//ZR: this function assumes all fields in file_schema have value try {
Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); MetricCollector metric;
file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); bool status;
try {
MetricCollector metric; {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
{ if (connectionPtr == nullptr) {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); return Status::Error("Failed to connect to database server");
}
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { Query discardFilesQuery = connectionPtr->query();
// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); discardFilesQuery << "SELECT id, size " <<
// } "FROM TableFiles " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " <<
"ORDER BY id ASC " <<
"LIMIT 10;";
Query updateTableFileQuery = connectionPtr->query(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str();
//if the table has been deleted, just mark the table file as TO_DELETE
//clean thread will delete the file later
updateTableFileQuery << "SELECT state FROM Tables " <<
"WHERE table_id = " << quote << file_schema.table_id_ << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); StoreQueryResult res = discardFilesQuery.store();
StoreQueryResult res = updateTableFileQuery.store(); if (res.num_rows() == 0) {
return Status::OK();
}
if (res.num_rows() == 1) { TableFileSchema table_file;
int state = res[0]["state"]; std::stringstream idsToDiscardSS;
if (state == TableSchema::TO_DELETE) { for (auto &resRow : res) {
file_schema.file_type_ = TableFileSchema::TO_DELETE; if (to_discard_size <= 0) {
} break;
} else {
file_schema.file_type_ = TableFileSchema::TO_DELETE;
} }
table_file.id_ = resRow["id"];
table_file.size_ = resRow["size"];
idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR ";
ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_
<< " table_file.size=" << table_file.size_;
to_discard_size -= table_file.size_;
}
std::string id = std::to_string(file_schema.id_); std::string idsToDiscardStr = idsToDiscardSS.str();
std::string table_id = file_schema.table_id_; idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR "
std::string engine_type = std::to_string(file_schema.engine_type_);
std::string file_id = file_schema.file_id_;
std::string file_type = std::to_string(file_schema.file_type_);
std::string size = std::to_string(file_schema.size_);
std::string updated_time = std::to_string(file_schema.updated_time_);
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
updateTableFileQuery << "UPDATE TableFiles " << discardFilesQuery << "UPDATE TableFiles " <<
"SET table_id = " << quote << table_id << ", " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " <<
"engine_type = " << engine_type << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"file_id = " << quote << file_id << ", " << "WHERE " << idsToDiscardStr << ";";
"file_type = " << file_type << ", " <<
"size = " << size << ", " <<
"updated_time = " << updated_time << ", " <<
"created_on = " << created_on << ", " <<
"date = " << date << " " <<
"WHERE id = " << id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
// std::cout << updateTableFileQuery.str() << std::endl;
if (!updateTableFileQuery.exec()) {
ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE";
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE",
updateTableFileQuery.error());
}
} //Scoped Connection
} catch (const BadQuery& er) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str();
// Handle any query errors
ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what());
}
return Status::OK();
}
Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { status = discardFilesQuery.exec();
// TODO if (!status) {
return Status::OK(); ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES";
return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error());
}
} //Scoped Connection
return DiscardFiles(to_discard_size);
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what());
} }
}
Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { //ZR: this function assumes all fields in file_schema have value
Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
try { file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
MetricCollector metric; try {
{ MetricCollector metric;
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { {
return Status::Error("Failed to connect to database server"); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query updateTableFilesQuery = connectionPtr->query(); if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
std::map<std::string, bool> has_tables;
for (auto &file_schema : files) {
if (has_tables.find(file_schema.table_id_) != has_tables.end()) { Query updateTableFileQuery = connectionPtr->query();
continue;
}
updateTableFilesQuery << "SELECT EXISTS " << //if the table has been deleted, just mark the table file as TO_DELETE
"(SELECT 1 FROM Tables " << //clean thread will delete the file later
"WHERE table_id = " << quote << file_schema.table_id_ << " " << updateTableFileQuery << "SELECT state FROM Tables " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "WHERE table_id = " << quote << file_schema.table_id_ << ";";
"AS " << quote << "check" << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
StoreQueryResult res = updateTableFilesQuery.store(); StoreQueryResult res = updateTableFileQuery.store();
int check = res[0]["check"]; if (res.num_rows() == 1) {
has_tables[file_schema.table_id_] = (check == 1); int state = res[0]["state"];
if (state == TableSchema::TO_DELETE) {
file_schema.file_type_ = TableFileSchema::TO_DELETE;
} }
} else {
file_schema.file_type_ = TableFileSchema::TO_DELETE;
}
std::string id = std::to_string(file_schema.id_);
std::string table_id = file_schema.table_id_;
std::string engine_type = std::to_string(file_schema.engine_type_);
std::string file_id = file_schema.file_id_;
std::string file_type = std::to_string(file_schema.file_type_);
std::string size = std::to_string(file_schema.size_);
std::string updated_time = std::to_string(file_schema.updated_time_);
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
for (auto &file_schema : files) { updateTableFileQuery << "UPDATE TableFiles " <<
"SET table_id = " << quote << table_id << ", " <<
"engine_type = " << engine_type << ", " <<
"file_id = " << quote << file_id << ", " <<
"file_type = " << file_type << ", " <<
"size = " << size << ", " <<
"updated_time = " << updated_time << ", " <<
"created_on = " << created_on << ", " <<
"date = " << date << " " <<
"WHERE id = " << id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
if (!updateTableFileQuery.exec()) {
ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE";
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE",
updateTableFileQuery.error());
}
} //Scoped Connection
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what());
}
return Status::OK();
}
if (!has_tables[file_schema.table_id_]) { Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) {
file_schema.file_type_ = TableFileSchema::TO_DELETE; try {
} ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
std::string id = std::to_string(file_schema.id_);
std::string table_id = file_schema.table_id_;
std::string engine_type = std::to_string(file_schema.engine_type_);
std::string file_id = file_schema.file_id_;
std::string file_type = std::to_string(file_schema.file_type_);
std::string size = std::to_string(file_schema.size_);
std::string updated_time = std::to_string(file_schema.updated_time_);
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
updateTableFilesQuery << "UPDATE TableFiles " <<
"SET table_id = " << quote << table_id << ", " <<
"engine_type = " << engine_type << ", " <<
"file_id = " << quote << file_id << ", " <<
"file_type = " << file_type << ", " <<
"size = " << size << ", " <<
"updated_time = " << updated_time << ", " <<
"created_on = " << created_on << ", " <<
"date = " << date << " " <<
"WHERE id = " << id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
if (!updateTableFilesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES";
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES",
updateTableFilesQuery.error());
}
}
} //Scoped Connection
} catch (const BadQuery& er) { if (connectionPtr == nullptr) {
// Handle any query errors return Status::Error("Failed to connect to database server");
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what());
} }
return Status::OK();
}
Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { Query updateTableFilesToIndexQuery = connectionPtr->query();
// static int b_count = 0;
// b_count++;
// std::cout << "CleanUpFilesWithTTL: " << b_count << std::endl;
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
auto now = utils::GetMicroSecTimeStamp(); updateTableFilesToIndexQuery << "UPDATE TableFiles " <<
try { "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " <<
MetricCollector metric; "WHERE table_id = " << quote << table_id << " AND " <<
"file_type = " << std::to_string(TableFileSchema::RAW) << ";";
{ ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFilesToIndexQuery.str();
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use before creating ScopedConnection = " } catch (const BadQuery &er) {
// << mysql_connection_pool_->getConnectionsInUse(); // Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what());
}
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); return Status::OK();
}
if (connectionPtr == nullptr) { Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = "
// << mysql_connection_pool_->getConnectionsInUse();
// }
Query cleanUpFilesWithTTLQuery = connectionPtr->query(); try {
cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << MetricCollector metric;
"FROM TableFiles " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " <<
"updated_time < " << std::to_string(now - seconds * US_PS) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
TableFileSchema table_file; Query updateTableFilesQuery = connectionPtr->query();
std::vector<std::string> idsToDelete;
for (auto &resRow : res) { std::map<std::string, bool> has_tables;
for (auto &file_schema : files) {
table_file.id_ = resRow["id"]; //implicit conversion if (has_tables.find(file_schema.table_id_) != has_tables.end()) {
continue;
}
std::string table_id; updateTableFilesQuery << "SELECT EXISTS " <<
resRow["table_id"].to_string(table_id); "(SELECT 1 FROM Tables " <<
table_file.table_id_ = table_id; "WHERE table_id = " << quote << file_schema.table_id_ << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
std::string file_id; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
table_file.date_ = resRow["date"]; StoreQueryResult res = updateTableFilesQuery.store();
utils::DeleteTableFilePath(options_, table_file); int check = res[0]["check"];
has_tables[file_schema.table_id_] = (check == 1);
}
ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " for (auto &file_schema : files) {
<< table_file.location_ << std::endl;
idsToDelete.emplace_back(std::to_string(table_file.id_)); if (!has_tables[file_schema.table_id_]) {
file_schema.file_type_ = TableFileSchema::TO_DELETE;
} }
file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
if (!idsToDelete.empty()) { std::string id = std::to_string(file_schema.id_);
std::string table_id = file_schema.table_id_;
std::string engine_type = std::to_string(file_schema.engine_type_);
std::string file_id = file_schema.file_id_;
std::string file_type = std::to_string(file_schema.file_type_);
std::string size = std::to_string(file_schema.size_);
std::string updated_time = std::to_string(file_schema.updated_time_);
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
std::stringstream idsToDeleteSS; updateTableFilesQuery << "UPDATE TableFiles " <<
for (auto &id : idsToDelete) { "SET table_id = " << quote << table_id << ", " <<
idsToDeleteSS << "id = " << id << " OR "; "engine_type = " << engine_type << ", " <<
} "file_id = " << quote << file_id << ", " <<
"file_type = " << file_type << ", " <<
"size = " << size << ", " <<
"updated_time = " << updated_time << ", " <<
"created_on = " << created_on << ", " <<
"date = " << date << " " <<
"WHERE id = " << id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
if (!updateTableFilesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES";
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES",
updateTableFilesQuery.error());
}
}
} //Scoped Connection
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what());
}
return Status::OK();
}
std::string idsToDeleteStr = idsToDeleteSS.str(); Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR "
cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " <<
idsToDeleteStr << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
if (!cleanUpFilesWithTTLQuery.exec()) { auto now = utils::GetMicroSecTimeStamp();
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; try {
return Status::DBTransactionError("CleanUpFilesWithTTL Error", MetricCollector metric;
cleanUpFilesWithTTLQuery.error());
}
}
} //Scoped Connection
} catch (const BadQuery& er) { {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
}
try {
MetricCollector metric;
{ ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use before creating ScopedConnection = "
// << mysql_connection_pool_->getConnectionsInUse();
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { Query cleanUpFilesWithTTLQuery = connectionPtr->query();
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " <<
// << mysql_connection_pool_->getConnectionsInUse(); "FROM TableFiles " <<
// } "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " <<
"updated_time < " << std::to_string(now - seconds * US_PS) << ";";
Query cleanUpFilesWithTTLQuery = connectionPtr->query(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
cleanUpFilesWithTTLQuery << "SELECT id, table_id " <<
"FROM Tables " <<
"WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); TableFileSchema table_file;
// std::cout << res.num_rows() << std::endl; std::vector<std::string> idsToDelete;
if (!res.empty()) { for (auto &resRow : res) {
std::stringstream idsToDeleteSS; table_file.id_ = resRow["id"]; //implicit conversion
for (auto &resRow : res) {
size_t id = resRow["id"];
std::string table_id;
resRow["table_id"].to_string(table_id);
utils::DeleteTablePath(options_, table_id); std::string table_id;
resRow["table_id"].to_string(table_id);
table_file.table_id_ = table_id;
idsToDeleteSS << "id = " << std::to_string(id) << " OR "; std::string file_id;
} resRow["file_id"].to_string(file_id);
std::string idsToDeleteStr = idsToDeleteSS.str(); table_file.file_id_ = file_id;
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR "
cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " <<
idsToDeleteStr << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); table_file.date_ = resRow["date"];
if (!cleanUpFilesWithTTLQuery.exec()) { utils::DeleteTableFilePath(options_, table_file);
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL";
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = "
cleanUpFilesWithTTLQuery.error()); << table_file.location_ << std::endl;
}
idsToDelete.emplace_back(std::to_string(table_file.id_));
}
if (!idsToDelete.empty()) {
std::stringstream idsToDeleteSS;
for (auto &id : idsToDelete) {
idsToDeleteSS << "id = " << id << " OR ";
} }
} //Scoped Connection
} catch (const BadQuery& er) { std::string idsToDeleteStr = idsToDeleteSS.str();
// Handle any query errors idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR "
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " <<
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); idsToDeleteStr << ";";
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
}
return Status::OK(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
if (!cleanUpFilesWithTTLQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL";
return Status::DBTransactionError("CleanUpFilesWithTTL Error",
cleanUpFilesWithTTLQuery.error());
}
}
} //Scoped Connection
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
} }
Status MySQLMetaImpl::CleanUp() { try {
MetricCollector metric;
{
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
try {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server"); return Status::Error("Failed to connect to database server");
} }
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query cleanUpQuery = connectionPtr->query(); Query cleanUpFilesWithTTLQuery = connectionPtr->query();
cleanUpQuery << "SELECT table_name " << cleanUpFilesWithTTLQuery << "SELECT id, table_id " <<
"FROM information_schema.tables " << "FROM Tables " <<
"WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";";
"AND table_name = " << quote << "TableFiles" << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
StoreQueryResult res = cleanUpQuery.store();
if (!res.empty()) { if (!res.empty()) {
ENGINE_LOG_DEBUG << "Remove table file type as NEW";
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); std::stringstream idsToDeleteSS;
for (auto &resRow : res) {
size_t id = resRow["id"];
std::string table_id;
resRow["table_id"].to_string(table_id);
if (!cleanUpQuery.exec()) { utils::DeleteTablePath(options_, table_id);
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); idsToDeleteSS << "id = " << std::to_string(id) << " OR ";
} }
} std::string idsToDeleteStr = idsToDeleteSS.str();
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR "
cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " <<
idsToDeleteStr << ";";
} catch (const BadQuery& er) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what());
}
return Status::OK(); if (!cleanUpFilesWithTTLQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL";
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL",
cleanUpFilesWithTTLQuery.error());
}
}
} //Scoped Connection
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
} }
Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { return Status::OK();
}
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
try {
MetricCollector metric;
TableSchema table_schema; Status MySQLMetaImpl::CleanUp() {
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
StoreQueryResult res; try {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
{ if (connectionPtr == nullptr) {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); return Status::Error("Failed to connect to database server");
}
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { Query cleanUpQuery = connectionPtr->query();
// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); cleanUpQuery << "SELECT table_name " <<
// } "FROM information_schema.tables " <<
"WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " <<
"AND table_name = " << quote << "TableFiles" << ";";
Query countQuery = connectionPtr->query(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
countQuery << "SELECT size " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); StoreQueryResult res = cleanUpQuery.store();
res = countQuery.store(); if (!res.empty()) {
} //Scoped Connection ENGINE_LOG_DEBUG << "Remove table file type as NEW";
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
result = 0; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
for (auto &resRow : res) {
size_t size = resRow["size"];
result += size;
}
if (table_schema.dimension_ <= 0) { if (!cleanUpQuery.exec()) {
std::stringstream errorMsg; ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) << ", table_id = " << table_id; return Status::DBTransactionError("Clean up Error", cleanUpQuery.error());
ENGINE_LOG_ERROR << errorMsg.str();
return Status::Error(errorMsg.str());
} }
result /= table_schema.dimension_;
result /= sizeof(float);
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what());
} }
return Status::OK();
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what());
} }
Status MySQLMetaImpl::DropAll() { return Status::OK();
}
Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
if (boost::filesystem::is_directory(options_.path)) { try {
boost::filesystem::remove_all(options_.path); MetricCollector metric;
TableSchema table_schema;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
} }
try {
StoreQueryResult res;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server"); return Status::Error("Failed to connect to database server");
} }
// if (mysql_connection_pool_->getConnectionsInUse() <= 0) {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
Query dropTableQuery = connectionPtr->query(); Query countQuery = connectionPtr->query();
dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; countQuery << "SELECT size " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str();
if (dropTableQuery.exec()) { res = countQuery.store();
return Status::OK(); } //Scoped Connection
}
else { result = 0;
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; for (auto &resRow : res) {
return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); size_t size = resRow["size"];
} result += size;
} catch (const BadQuery& er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what());
} catch (const Exception& er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what());
} }
return Status::OK();
if (table_schema.dimension_ <= 0) {
std::stringstream errorMsg;
errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_)
<< ", table_id = " << table_id;
ENGINE_LOG_ERROR << errorMsg.str();
return Status::Error(errorMsg.str());
}
result /= table_schema.dimension_;
result /= sizeof(float);
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what());
} }
return Status::OK();
}
Status MySQLMetaImpl::DropAll() {
MySQLMetaImpl::~MySQLMetaImpl() {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex); if (boost::filesystem::is_directory(options_.path)) {
if (mode_ != Options::MODE::READ_ONLY) { boost::filesystem::remove_all(options_.path);
CleanUp(); }
try {
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
} }
Query dropTableQuery = connectionPtr->query();
dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str();
if (dropTableQuery.exec()) {
return Status::OK();
} else {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE";
return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error());
}
} catch (const BadQuery &er) {
// Handle any query errors
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what();
return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what());
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what());
}
return Status::OK();
}
MySQLMetaImpl::~MySQLMetaImpl() {
if (mode_ != Options::MODE::READ_ONLY) {
CleanUp();
} }
}
} // namespace meta } // namespace meta
} // namespace engine } // namespace engine
......
cpp/src/db/Options.h
浏览文件 @ e950fff6
...@@ -63,7 +63,7 @@ struct Options { ...@@ -63,7 +63,7 @@ struct Options {
size_t index_trigger_size = ONE_GB; //unit: byte size_t index_trigger_size = ONE_GB; //unit: byte
DBMetaOptions meta; DBMetaOptions meta;
int mode = MODE::SINGLE; int mode = MODE::SINGLE;
float maximum_memory = 4 * ONE_GB; size_t maximum_memory = 4 * ONE_GB;
}; // Options }; // Options
......
cpp/src/server/DBWrapper.cpp
浏览文件 @ e950fff6
...@@ -28,11 +28,11 @@ DBWrapper::DBWrapper() { ...@@ -28,11 +28,11 @@ DBWrapper::DBWrapper() {
if(index_size > 0) {//ensure larger than zero, unit is MB if(index_size > 0) {//ensure larger than zero, unit is MB
opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
} }
float maximum_memory = config.GetFloatValue(CONFIG_MAXMIMUM_MEMORY); int64_t maximum_memory = config.GetInt64Value(CONFIG_MAXMIMUM_MEMORY);
if (maximum_memory > 1.0) { if (maximum_memory > 1.0) {
opt.maximum_memory = maximum_memory * engine::ONE_GB; opt.maximum_memory = maximum_memory * engine::ONE_GB;
} }
else { else if (maximum_memory != 0) { //if maximum_memory = 0, set it to default
std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl;
kill(0, SIGUSR1); kill(0, SIGUSR1);
} }
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或