diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index f1d3a77a8a7ad28315a87cc158d6dd11e1852f5c..0a84ffd7ff970b035f6304ce95044a6391577343 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -28,6 +28,7 @@ public: virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; virtual Status AllTables(std::vector& table_schema_array) = 0; virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; + virtual Status PreloadTable(const std::string& table_id) = 0; virtual Status InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 636efc51bdaa5a04850283b47acf443e916e195a..812438546c110abf445b47dcb8080a79ef51a5e1 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -32,6 +32,7 @@ namespace { constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; +constexpr int64_t unit = 1024 * 1024 * 1024; void CollectInsertMetrics(double total_time, size_t n, bool succeed) { double avg_time = total_time / n; @@ -127,6 +128,36 @@ Status DBImpl::AllTables(std::vector& table_schema_array) { return meta_ptr_->AllTables(table_schema_array); } +Status DBImpl::PreloadTable(const std::string &table_id) { + meta::TableFilesSchema files; + auto status = meta_ptr_->PreloadTable(table_id, files); + + int64_t size = 0; + + server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); + cap *= unit; + for(auto &file : files) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); + if(engine == nullptr) { + ENGINE_LOG_ERROR << "Invalid engine type"; + return Status::Error("Invalid engine type"); + } + + size += engine->PhysicalSize(); + if (size <= cap) { + try { + //step 1: load index + engine->Load(options_.insert_cache_immediately_); + } catch (std::exception &ex) { + std::string msg = "load to cache exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); + } + } + } +} + Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { return meta_ptr_->Count(table_id, row_count); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index ccd7f541bdb80111976e285dfc98846bf96d80c9..969b7b0509ff497ba310d8e7e47d69969721d81b 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -51,6 +51,9 @@ class DBImpl : public DB { Status AllTables(std::vector &table_schema_array) override; + Status + PreloadTable(const std::string &table_id) override; + Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index e88761b4460613cf8899453c19178c0fb9c04099..b6e867a81998c5a13dd5d03499176be28c59cdab 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -38,6 +38,9 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status + PreloadTable(const std::string &table_id, meta::TableFilesSchema &files) = 0; + virtual Status DeleteTable(const std::string &table_id) = 0; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index bf4589351ecaece83406d8989890ecd32c80ae8d..800faefc7cfc9f9eaad8e36a6e38d9659db321a0 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -690,6 +690,11 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { return Status::OK(); } +Status +MySQLMetaImpl::PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) { + +} + Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 30695423ddd39a1ab4f5d63edacf43e5a22ad319..f673678aa3a237fcc49e49fb660b2362a58f6d2d 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -29,6 +29,7 @@ class MySQLMetaImpl : public Meta { Status DescribeTable(TableSchema &group_info_) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; Status AllTables(std::vector &table_schema_array) override; + Status PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; Status DeleteTable(const std::string &table_id) override; Status DeleteTableFiles(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 9411a5e82b0a88eccbc0351d275e334dfa5ad8b3..738c31132b1866290a259d9f2f8afca2837eda14 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -404,6 +404,40 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } +Status SqliteMetaImpl::PreloadTable(const std::string &table_id, TableFilesSchema &files_schema) { + try { + MetricCollector metric; + + auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::dimension_, + &TableFileSchema::location_, + &TableFileSchema::engine_type_), + where(c(&TableFileSchema::table_id_) == table_id)); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + int64_t size = 0; + for (auto &file : selected) { + TableFileSchema file_schema; + file_schema.size_ = std::get<0>(file); + file_schema.dimension_ = std::get<1>(file); + file_schema.location_ = std::get<2>(file); + file_schema.engine_type_ = std::get<3>(file); + files_schema.push_back(file_schema); + } + + + } catch (std::exception &e) { + return HandleException("Encounter exception when lookup all tables", e); + } + return Status::OK(); +} + Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 1525f27e4b776afc7757c893199b7575e2f96fda..68e483b76ffa091995e38309cea8d41d7c8b9168 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -33,6 +33,9 @@ class SqliteMetaImpl : public Meta { Status AllTables(std::vector &table_schema_array) override; + Status + PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; + Status DeleteTable(const std::string &table_id) override;