diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 6dc2369b47ec0d1f6f79fe0591bd3394823e5301..667399cc5d2b4267c72a23fb32e19d3f515a239b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-566 - Refactor cmake - MS-555 - Remove old scheduler +- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data ## New Feature diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 5796b781b2a4fecf525005d88214e811305ac0bf..fd52b851e2d4055836fe01d7fa17bc5e8024edef 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -17,7 +17,7 @@ #include "Options.h" #include "utils/Exception.h" -#include "utils/easylogging++.h" +#include "utils/Log.h" #include #include @@ -56,11 +56,11 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { - LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } if (kv[0] != "disk" && kv[0] != "days") { - LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } try { @@ -68,20 +68,22 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { criterias_[kv[0]] = value; } catch (std::out_of_range&){ - LOG(ERROR) << "Out of range: '" << kv[1] << "'"; - throw OutOfRangeException(); + std::string msg = "Out of range: '" + kv[1] + "'"; + ENGINE_LOG_ERROR << msg; + throw InvalidArgumentException(msg); } catch (...){ - LOG(ERROR) << "Invalid argument: '" << kv[1] << "'"; - throw InvalidArgumentException(); + std::string msg = "Invalid argument: '" + kv[1] + "'"; + ENGINE_LOG_ERROR << msg; + throw InvalidArgumentException(msg); } } } void ArchiveConf::ParseType(const std::string& type) { if (type != "delete" && type != "swap") { - LOG(ERROR) << "Invalid argument: type='" << type << "'"; - throw InvalidArgumentException(); + std::string msg = "Invalid argument: type='" + type + "'"; + throw InvalidArgumentException(msg); } type_ = type; } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 2fb868f548cc94ec1d51a9f4917f2061d6272244..41729d104f156cbec12d1f229fbd55cd1b359d59 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -45,14 +45,18 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); - if (!index_) throw Exception("Create Empty VecIndex"); + if (!index_) { + throw Exception(DB_ERROR, "Could not create VecIndex"); + } Config build_cfg; build_cfg["dim"] = dimension; build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; AutoGenParams(index_->GetType(), 0, build_cfg); auto ec = std::static_pointer_cast(index_)->Build(build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { + throw Exception(DB_ERROR, "Build index error"); + } } ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, @@ -273,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t auto to_index = CreatetVecIndex(engine_type); if (!to_index) { - throw Exception("Create Empty VecIndex"); + throw Exception(DB_ERROR, "Could not create VecIndex"); } Config build_cfg; @@ -287,7 +291,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t from_index->GetRawVectors(), from_index->GetRawIds(), build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index ee4972ae6be73c53bff381239eed90c1fa2e4ff7..58168e5fee3cabb8eba089bfdedf40714b967f8c 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -31,6 +31,9 @@ namespace milvus { namespace engine { namespace meta { +static const char* META_TABLES = "Tables"; +static const char* META_TABLEFILES = "TableFiles"; + class Meta { public: virtual ~Meta() = default; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index a07d4c82cf951303a159b08d1febc3dd853e35bd..282df32eb18f9a2c18b714bce28348c80cce34c7 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -153,8 +153,9 @@ Status MySQLMetaImpl::Initialize() { } Query InitializeQuery = connectionPtr->query(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + META_TABLES << " " << + "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) UNIQUE NOT NULL, " << "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << @@ -171,8 +172,9 @@ Status MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } - InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + META_TABLEFILES << " " << + "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "file_id VARCHAR(255) NOT NULL, " << @@ -233,7 +235,8 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE TableFiles " << + dropPartitionsByDatesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -266,7 +269,8 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { - createTableQuery << "SELECT state FROM Tables " << + createTableQuery << "SELECT state FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -297,8 +301,9 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string nlist = std::to_string(table_schema.nlist_); std::string metric_type = std::to_string(table_schema.metric_type_); - createTableQuery << "INSERT INTO Tables VALUES" << - "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + createTableQuery << "INSERT INTO " << + META_TABLES << " " << + "VALUES(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ");"; @@ -348,7 +353,8 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, Query hasNonIndexFilesQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type FROM TableFiles " << + hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type in (" << types << ");"; @@ -418,8 +424,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI } Query updateTableIndexParamQuery = connectionPtr->query(); - updateTableIndexParamQuery << "SELECT id, state, dimension, created_on " << - "FROM Tables " << + updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -435,7 +441,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI uint16_t dimension = resRow["dimension"]; int64_t created_on = resRow["created_on"]; - updateTableIndexParamQuery << "UPDATE Tables " << + updateTableIndexParamQuery << "UPDATE " << + META_TABLES << " " << "SET id = " << id << ", " << "state = " << state << ", " << "dimension = " << dimension << ", " << @@ -476,7 +483,8 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) } Query updateTableFlagQuery = connectionPtr->query(); - updateTableFlagQuery << "UPDATE Tables " << + updateTableFlagQuery << "UPDATE " << + META_TABLES << " " << "SET flag = " << flag << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -507,8 +515,8 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex } Query describeTableIndexQuery = connectionPtr->query(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type " << - "FROM Tables " << + describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -549,7 +557,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { Query dropTableIndexQuery = connectionPtr->query(); //soft delete index files - dropTableIndexQuery << "UPDATE TableFiles " << + dropTableIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -562,7 +571,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //set all backup file to raw - dropTableIndexQuery << "UPDATE TableFiles " << + dropTableIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -575,7 +585,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //set table index type to raw - dropTableIndexQuery << "UPDATE Tables " << + dropTableIndexQuery << "UPDATE " << + META_TABLES << " " << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << @@ -609,7 +620,8 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { //soft delete table Query deleteTableQuery = connectionPtr->query(); // - deleteTableQuery << "UPDATE Tables " << + deleteTableQuery << "UPDATE " << + META_TABLES << " " << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -645,7 +657,8 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { //soft delete table files Query deleteTableFilesQuery = connectionPtr->query(); // - deleteTableFilesQuery << "UPDATE TableFiles " << + deleteTableFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -676,9 +689,8 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, state, dimension, created_on, " << - "flag, index_file_size, engine_type, nlist, metric_type " << - "FROM Tables " << + describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -732,7 +744,8 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << + "(SELECT 1 FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; @@ -764,8 +777,8 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { } Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type " << - "FROM Tables " << + allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " << + META_TABLES << " " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); @@ -846,8 +859,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { Query createTableFileQuery = connectionPtr->query(); - createTableFileQuery << "INSERT INTO TableFiles VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + createTableFileQuery << "INSERT INTO " << + META_TABLEFILES << " " << + "VALUES(" << id << ", " << quote << table_id << ", " << engine_type << ", " << quote << file_id << ", " << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; @@ -883,8 +897,8 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { } Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << - "FROM TableFiles " << + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); @@ -967,8 +981,8 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, } Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " << - "FROM TableFiles " << + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id; if (!partition.empty()) { @@ -1086,8 +1100,8 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, } Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on " << - "FROM TableFiles " << + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY row_count DESC" << ";"; @@ -1177,8 +1191,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, } Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << - "FROM TableFiles " << + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ") AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1259,7 +1273,8 @@ Status MySQLMetaImpl::Archive() { } Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE TableFiles " << + archiveQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1299,8 +1314,8 @@ Status MySQLMetaImpl::Size(uint64_t &result) { } Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum " << - "FROM TableFiles " << + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << + META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); @@ -1339,8 +1354,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { } Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, file_size " << - "FROM TableFiles " << + discardFilesQuery << "SELECT id, file_size FROM " << + META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << "LIMIT 10;"; @@ -1369,7 +1384,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { std::string idsToDiscardStr = idsToDiscardSS.str(); idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " - discardFilesQuery << "UPDATE TableFiles " << + discardFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; @@ -1406,7 +1422,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM Tables " << + updateTableFileQuery << "SELECT state FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1433,7 +1450,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE TableFiles " << + updateTableFileQuery << "UPDATE " << + META_TABLEFILES << " " << "SET table_id = " << quote << table_id << ", " << "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << @@ -1470,7 +1488,8 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { Query updateTableFilesToIndexQuery = connectionPtr->query(); - updateTableFilesToIndexQuery << "UPDATE TableFiles " << + updateTableFilesToIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; @@ -1508,7 +1527,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << + "(SELECT 1 FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; @@ -1539,7 +1559,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFilesQuery << "UPDATE TableFiles " << + updateTableFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET table_id = " << quote << table_id << ", " << "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << @@ -1582,8 +1603,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << - "FROM TableFiles " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << + META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; @@ -1626,8 +1647,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; + cleanUpFilesWithTTLQuery << "DELETE FROM " << + META_TABLEFILES << " " << + "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1653,8 +1675,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id " << - "FROM Tables " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << + META_TABLES << " " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1675,8 +1697,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; + cleanUpFilesWithTTLQuery << "DELETE FROM " << + META_TABLES << " " << + "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1704,8 +1727,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { for(auto& table_id : table_ids) { Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id " << - "FROM TableFiles " << + cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1736,7 +1759,7 @@ Status MySQLMetaImpl::CleanUp() { cleanUpQuery << "SELECT table_name " << "FROM information_schema.tables " << "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << "TableFiles" << ";"; + "AND table_name = " << quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1744,7 +1767,7 @@ Status MySQLMetaImpl::CleanUp() { if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type IN (" + cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(TableFileSchema::NEW) << "," << std::to_string(TableFileSchema::NEW_MERGE) << "," << std::to_string(TableFileSchema::NEW_INDEX) << ");"; @@ -1785,8 +1808,8 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { Query countQuery = connectionPtr->query(); - countQuery << "SELECT row_count " << - "FROM TableFiles " << + countQuery << "SELECT row_count FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << @@ -1820,7 +1843,7 @@ Status MySQLMetaImpl::DropAll() { } Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + dropTableQuery << "DROP TABLE IF EXISTS " << META_TABLES << ", " << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 564a8eae4d0bd4f3dadca6f6fbac7ee0a7de43eb..a28038d019e2de4bb853b3f3a965b9738fc7f290 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -19,6 +19,7 @@ #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" +#include "utils/Exception.h" #include "MetaConsts.h" #include "metrics/Metrics.h" @@ -55,7 +56,7 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { inline auto StoragePrototype(const std::string &path) { return make_storage(path, - make_table("Tables", + make_table(META_TABLES, make_column("id", &TableSchema::id_, primary_key()), make_column("table_id", &TableSchema::table_id_, unique()), make_column("state", &TableSchema::state_), @@ -66,7 +67,7 @@ inline auto StoragePrototype(const std::string &path) { make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), make_column("metric_type", &TableSchema::metric_type_)), - make_table("TableFiles", + make_table(META_TABLEFILES, make_column("id", &TableFileSchema::id_, primary_key()), make_column("table_id", &TableFileSchema::table_id_), make_column("engine_type", &TableFileSchema::engine_type_), @@ -122,6 +123,17 @@ Status SqliteMetaImpl::Initialize() { ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); + //old meta could be recreated since schema changed, throw exception if meta schema is not compatible + auto ret = ConnectorPtr->sync_schema_simulate(); + if(ret.find(META_TABLES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); + } + if(ret.find(META_TABLEFILES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); + } + ConnectorPtr->sync_schema(); ConnectorPtr->open_forever(); // thread safe option ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log @@ -1246,8 +1258,8 @@ Status SqliteMetaImpl::DropAll() { ENGINE_LOG_DEBUG << "Drop all sqlite meta"; try { - ConnectorPtr->drop_table("Tables"); - ConnectorPtr->drop_table("TableFiles"); + ConnectorPtr->drop_table(META_TABLES); + ConnectorPtr->drop_table(META_TABLEFILES); } catch (std::exception &e) { return HandleException("Encounter exception when drop all meta", e.what()); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 54f9541effacd050f037393b1de02a3ea2966daf..5a61c5bfb4508d2d1ef2f560ff7e32f00a431c17 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -102,10 +102,6 @@ main(int argc, char *argv[]) { } } - server::Server &server = server::Server::Instance(); - server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - server.Start(); - /* Handle Signal */ signal(SIGHUP, server::SignalUtil::HandleSignal); signal(SIGINT, server::SignalUtil::HandleSignal); @@ -114,6 +110,10 @@ main(int argc, char *argv[]) { signal(SIGUSR2, server::SignalUtil::HandleSignal); signal(SIGTERM, server::SignalUtil::HandleSignal); + server::Server &server = server::Server::Instance(); + server.Init(start_daemonized, pid_filename, config_filename, log_config_file); + server.Start(); + /* wait signal */ pause(); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 115a636621c94b04376783712dd308a273b1610f..1409fff597ae18cc4967581714c2a61c8a878db0 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -109,15 +109,10 @@ Status DBWrapper::StartService() { } //create db instance - std::string msg = opt.meta.path; try { db_ = engine::DBFactory::Build(opt); } catch(std::exception& ex) { - msg = ex.what(); - } - - if(db_ == nullptr) { - std::cerr << "ERROR! Failed to open database: " << msg << std::endl; + std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 38b41a5331461386b7acd678f85720077489dcdd..2d0a4e5c99f7488959102a87a4fbef438aaef818 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -196,8 +196,8 @@ Server::Start() { server::Metrics::GetInstance().Init(); server::SystemInfo::GetInstance().Init(); - std::cout << "Milvus server start successfully." << std::endl; StartService(); + std::cout << "Milvus server start successfully." << std::endl; } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index d7008115f972b5f75814332a5e53965610f78ddf..672aa9284887d6226c8d1b01de64641c997532ae 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -86,6 +86,7 @@ constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); +constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); //knowhere error code constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index acb796c61ded06fbf368a46641b8e80ad0833ae3..6542f913af2f550d1bf452cacb5fe2cc09136360 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -17,6 +17,8 @@ #pragma once +#include "utils/Error.h" + #include #include @@ -25,13 +27,14 @@ namespace milvus { class Exception : public std::exception { public: - Exception(const std::string& message) - : message_(message) { + Exception(ErrorCode code, const std::string& message) + : code_(code_), + message_(message) { } - Exception() - : message_() { - } + ErrorCode code() const throw() { + return code_; + } virtual const char* what() const throw() { if (message_.empty()) { @@ -44,20 +47,21 @@ public: virtual ~Exception() throw() {}; protected: - + ErrorCode code_; std::string message_; }; class InvalidArgumentException : public Exception { public: - InvalidArgumentException() : Exception("Invalid Argument"){}; - InvalidArgumentException(const std::string& message) : Exception(message) {}; -}; + InvalidArgumentException() + : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + + }; + InvalidArgumentException(const std::string& message) + : Exception(SERVER_INVALID_ARGUMENT, message) { + + }; -class OutOfRangeException : public Exception { -public: - OutOfRangeException() : Exception("Out Of Range"){}; - OutOfRangeException(const std::string& message) : Exception(message) {}; }; } // namespace milvus diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 5bb28696e21facde872fbfe30c647bf23f1cd0f8..2610a3a8bf904d59fa8ebae71197fbe2fc20664c 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -24,7 +24,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define SERVER_DOMAIN_NAME "[SERVER] " -#define SERVER_ERROR_TEXT "SERVER Error:" #define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME #define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME @@ -35,7 +34,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define ENGINE_DOMAIN_NAME "[ENGINE] " -#define ENGINE_ERROR_TEXT "ENGINE Error:" #define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME #define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME @@ -46,7 +44,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define WRAPPER_DOMAIN_NAME "[WRAPPER] " -#define WRAPPER_ERROR_TEXT "WRAPPER Error:" #define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME #define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index e4af1104db3ccae24dd89f1236a2e3a674917733..cfdbbdc768e9ab56cdb45ffc2fb0180e8d93f5d0 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -17,10 +17,8 @@ #include "LogUtil.h" #include "server/ServerConfig.h" -#include "easylogging++.h" #include - #include #include diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index a1fd14a8f410e3a671674b34c1d69ab7b0f0a257..f2f81215388723850b21946babcdbcedbe6e69a3 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -24,10 +24,7 @@ #include "utils/CommonUtil.h" #include -#include "utils/easylogging++.h" - #include - #include #include diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 62271b9a3b51180b9c38d9fb32ad93eeafc5edbf..7a6ee31243b009bacd5f898524f33ecc4d27d896 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include - #include "utils.h" #include "db/meta/SqliteMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" +#include +#include +#include +#include + using namespace zilliz::milvus; using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index ff530b0a734c5a172aaee1bd9f0e3ed1913a816c..dd388dab7ab78cf1a155ef6d041f11446438d01d 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -21,7 +21,6 @@ #include "db/Utils.h" #include "utils/Status.h" #include "utils/Exception.h" -#include "utils/easylogging++.h" #include #include @@ -31,13 +30,13 @@ using namespace zilliz::milvus; TEST(DBMiscTest, EXCEPTION_TEST) { - Exception ex1(""); + Exception ex1(100, "error"); std::string what = ex1.what(); - ASSERT_FALSE(what.empty()); + ASSERT_EQ(what, "error"); + ASSERT_EQ(ex1.code(), 100); - OutOfRangeException ex2; - what = ex2.what(); - ASSERT_FALSE(what.empty()); + InvalidArgumentException ex2; + ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT); } TEST(DBMiscTest, OPTIONS_TEST) { diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 9d99b4095becd4528ab7f2a19e6a0114eaf77c8a..53ce51ca45f5434a7eabf39206bafa8fe5bf64f1 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -21,7 +21,6 @@ #include "db/meta/MetaConsts.h" #include -#include "utils/easylogging++.h" #include #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index e99565886d3cedcf1694f4337f1a4aa64021f311..a7c9837f146292c95c3785211cd61a1d8cd5df52 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -15,20 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include - #include "utils.h" #include "db/meta/MySQLMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" -#include "mysql++/mysql++.h" - #include +#include +#include +#include +#include +#include using namespace zilliz::milvus; using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index b0c2be6d5ae02c34ddd9191781c2af53161e566f..33a860945dfa035354d2e329cd76b7089c669aa5 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include -#include #include "utils/easylogging++.h" - #include "server/ServerConfig.h" #include "utils/CommonUtil.h" +#include +#include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus; diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index b582c9a17afb74e145189a348141629b54e6fd99..169b2e23ebaad8b91c0f2312800d45c4fb02c6f1 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -15,14 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include -#include -#include - +#include "utils/SignalUtil.h" #include "utils/CommonUtil.h" #include "utils/Error.h" #include "utils/StringHelpFunctions.h" @@ -32,6 +25,12 @@ #include "utils/ValidationUtil.h" #include "db/engine/ExecutionEngine.h" +#include +#include +#include +#include +#include + using namespace zilliz::milvus; namespace { diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index ac185efbb3f8137462a9b15917a1313a149874ed..4c0226f637b0eb57afc401bd7ab649385975de08 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -16,14 +16,13 @@ // under the License. -#include #include "utils/easylogging++.h" - -#include "src/wrapper/vec_index.h" +#include "wrapper/vec_index.h" #include "knowhere/index/vector_index/gpu_ivf.h" - #include "utils.h" +#include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus::engine;