add preloadtable

Former-commit-id: 61faf0724db583db9e2bea14b3ee6c9ad6e11eb8

cpp/src/db/DB.h

@@ -28,6 +28,7 @@ public:
     virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0;
     virtual Status AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
     virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
+    virtual Status PreloadTable(const std::string& table_id) = 0;
 
     virtual Status InsertVectors(const std::string& table_id_,
             uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;

cpp/src/db/DBImpl.cpp

@@ -32,6 +32,7 @@ namespace {
 constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
 constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
 constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
+constexpr int64_t unit = 1024 * 1024 * 1024;
 
 void CollectInsertMetrics(double total_time, size_t n, bool succeed) {
     double avg_time = total_time / n;
@@ -127,6 +128,36 @@ Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
     return meta_ptr_->AllTables(table_schema_array);
 }
 
+Status DBImpl::PreloadTable(const std::string &table_id) {
+    meta::TableFilesSchema files;
+    auto status = meta_ptr_->PreloadTable(table_id, files);
+
+    int64_t size = 0;
+
+    server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
+    int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16);
+    cap *= unit;
+    for(auto &file : files) {
+        ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
+        if(engine == nullptr) {
+            ENGINE_LOG_ERROR << "Invalid engine type";
+            return Status::Error("Invalid engine type");
+        }
+
+        size += engine->PhysicalSize();
+        if (size <= cap) {
+            try {
+                //step 1: load index
+                engine->Load(options_.insert_cache_immediately_);
+            } catch (std::exception &ex) {
+                std::string msg = "load to cache exception" + std::string(ex.what());
+                ENGINE_LOG_ERROR << msg;
+                return Status::Error(msg);
+            }
+        }
+    }
+}
+
 Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
     return meta_ptr_->Count(table_id, row_count);
 }

cpp/src/db/DBImpl.h

@@ -51,6 +51,9 @@ class DBImpl : public DB {
     Status
     AllTables(std::vector<meta::TableSchema> &table_schema_array) override;
 
+    Status
+    PreloadTable(const std::string &table_id) override;
+
     Status
     GetTableRowCount(const std::string &table_id, uint64_t &row_count) override;
 

cpp/src/db/meta/Meta.h

@@ -38,6 +38,9 @@ class Meta {
     virtual Status
     AllTables(std::vector<TableSchema> &table_schema_array) = 0;
 
+    virtual Status
+    PreloadTable(const std::string &table_id, meta::TableFilesSchema &files) = 0;
+
     virtual Status
     DeleteTable(const std::string &table_id) = 0;
 

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -690,6 +690,11 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
     return Status::OK();
 }
 
+Status
+MySQLMetaImpl::PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) {
+
+}
+
 Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
 
 

cpp/src/db/meta/MySQLMetaImpl.h

@@ -29,6 +29,7 @@ class MySQLMetaImpl : public Meta {
     Status DescribeTable(TableSchema &group_info_) override;
     Status HasTable(const std::string &table_id, bool &has_or_not) override;
     Status AllTables(std::vector<TableSchema> &table_schema_array) override;
+    Status PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override;
 
     Status DeleteTable(const std::string &table_id) override;
     Status DeleteTableFiles(const std::string &table_id) override;

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -404,6 +404,40 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
     return Status::OK();
 }
 
+Status SqliteMetaImpl::PreloadTable(const std::string &table_id, TableFilesSchema &files_schema) {
+    try {
+        MetricCollector metric;
+
+        auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_,
+                                                     &TableFileSchema::dimension_,
+                                                     &TableFileSchema::location_,
+                                                     &TableFileSchema::engine_type_),
+                                             where(c(&TableFileSchema::table_id_) == table_id));
+
+        TableSchema table_schema;
+        table_schema.table_id_ = table_id;
+        auto status = DescribeTable(table_schema);
+        if (!status.ok()) {
+            return status;
+        }
+
+        int64_t size = 0;
+        for (auto &file : selected) {
+            TableFileSchema file_schema;
+            file_schema.size_ = std::get<0>(file);
+            file_schema.dimension_ = std::get<1>(file);
+            file_schema.location_ = std::get<2>(file);
+            file_schema.engine_type_ = std::get<3>(file);
+            files_schema.push_back(file_schema);
+        }
+
+
+    } catch (std::exception &e) {
+        return HandleException("Encounter exception when lookup all tables", e);
+    }
+    return Status::OK();
+}
+
 Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
     if (file_schema.date_ == EmptyDate) {
         file_schema.date_ = Meta::GetDate();

cpp/src/db/meta/SqliteMetaImpl.h

@@ -33,6 +33,9 @@ class SqliteMetaImpl : public Meta {
     Status
     AllTables(std::vector<TableSchema> &table_schema_array) override;
 
+    Status
+    PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override;
+
     Status
     DeleteTable(const std::string &table_id) override;