From ccec483e1e9e19d79f421c0a0a38a0b0ab91e78f Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 14 Aug 2019 15:50:23 +0800 Subject: [PATCH] add preloadtable Former-commit-id: f052a555777fce2a1d27478f7b2bb51d1e536c7c --- cpp/src/db/DB.h | 1 + cpp/src/db/DBImpl.cpp | 31 +++++++++++++++++++++++++++ cpp/src/db/DBImpl.h | 3 +++ cpp/src/db/meta/Meta.h | 3 +++ cpp/src/db/meta/MySQLMetaImpl.cpp | 5 +++++ cpp/src/db/meta/MySQLMetaImpl.h | 1 + cpp/src/db/meta/SqliteMetaImpl.cpp | 34 ++++++++++++++++++++++++++++++ cpp/src/db/meta/SqliteMetaImpl.h | 3 +++ 8 files changed, 81 insertions(+) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index f1d3a77a..0a84ffd7 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -28,6 +28,7 @@ public: virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; virtual Status AllTables(std::vector& table_schema_array) = 0; virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; + virtual Status PreloadTable(const std::string& table_id) = 0; virtual Status InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 636efc51..81243854 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -32,6 +32,7 @@ namespace { constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; +constexpr int64_t unit = 1024 * 1024 * 1024; void CollectInsertMetrics(double total_time, size_t n, bool succeed) { double avg_time = total_time / n; @@ -127,6 +128,36 @@ Status DBImpl::AllTables(std::vector& table_schema_array) { return meta_ptr_->AllTables(table_schema_array); } +Status DBImpl::PreloadTable(const std::string &table_id) { + meta::TableFilesSchema files; + auto status = meta_ptr_->PreloadTable(table_id, files); + + int64_t size = 0; + + server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); + cap *= unit; + for(auto &file : files) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); + if(engine == nullptr) { + ENGINE_LOG_ERROR << "Invalid engine type"; + return Status::Error("Invalid engine type"); + } + + size += engine->PhysicalSize(); + if (size <= cap) { + try { + //step 1: load index + engine->Load(options_.insert_cache_immediately_); + } catch (std::exception &ex) { + std::string msg = "load to cache exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); + } + } + } +} + Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { return meta_ptr_->Count(table_id, row_count); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index ccd7f541..969b7b05 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -51,6 +51,9 @@ class DBImpl : public DB { Status AllTables(std::vector &table_schema_array) override; + Status + PreloadTable(const std::string &table_id) override; + Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index e88761b4..b6e867a8 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -38,6 +38,9 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status + PreloadTable(const std::string &table_id, meta::TableFilesSchema &files) = 0; + virtual Status DeleteTable(const std::string &table_id) = 0; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index bf458935..800faefc 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -690,6 +690,11 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { return Status::OK(); } +Status +MySQLMetaImpl::PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) { + +} + Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 30695423..f673678a 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -29,6 +29,7 @@ class MySQLMetaImpl : public Meta { Status DescribeTable(TableSchema &group_info_) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; Status AllTables(std::vector &table_schema_array) override; + Status PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; Status DeleteTable(const std::string &table_id) override; Status DeleteTableFiles(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 9411a5e8..738c3113 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -404,6 +404,40 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } +Status SqliteMetaImpl::PreloadTable(const std::string &table_id, TableFilesSchema &files_schema) { + try { + MetricCollector metric; + + auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::dimension_, + &TableFileSchema::location_, + &TableFileSchema::engine_type_), + where(c(&TableFileSchema::table_id_) == table_id)); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + int64_t size = 0; + for (auto &file : selected) { + TableFileSchema file_schema; + file_schema.size_ = std::get<0>(file); + file_schema.dimension_ = std::get<1>(file); + file_schema.location_ = std::get<2>(file); + file_schema.engine_type_ = std::get<3>(file); + files_schema.push_back(file_schema); + } + + + } catch (std::exception &e) { + return HandleException("Encounter exception when lookup all tables", e); + } + return Status::OK(); +} + Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 1525f27e..68e483b7 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -33,6 +33,9 @@ class SqliteMetaImpl : public Meta { Status AllTables(std::vector &table_schema_array) override; + Status + PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; + Status DeleteTable(const std::string &table_id) override; -- GitLab