提交 9fdfee96 编写于 作者: Y Yu Kun

add preloadtable


Former-commit-id: 61faf0724db583db9e2bea14b3ee6c9ad6e11eb8
上级 80fc6d3b
......@@ -28,6 +28,7 @@ public:
virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
virtual Status PreloadTable(const std::string& table_id) = 0;
virtual Status InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
......
......@@ -32,6 +32,7 @@ namespace {
constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
constexpr int64_t unit = 1024 * 1024 * 1024;
void CollectInsertMetrics(double total_time, size_t n, bool succeed) {
double avg_time = total_time / n;
......@@ -127,6 +128,36 @@ Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
return meta_ptr_->AllTables(table_schema_array);
}
Status DBImpl::PreloadTable(const std::string &table_id) {
meta::TableFilesSchema files;
auto status = meta_ptr_->PreloadTable(table_id, files);
int64_t size = 0;
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16);
cap *= unit;
for(auto &file : files) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
if(engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status::Error("Invalid engine type");
}
size += engine->PhysicalSize();
if (size <= cap) {
try {
//step 1: load index
engine->Load(options_.insert_cache_immediately_);
} catch (std::exception &ex) {
std::string msg = "load to cache exception" + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status::Error(msg);
}
}
}
}
Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
return meta_ptr_->Count(table_id, row_count);
}
......
......@@ -51,6 +51,9 @@ class DBImpl : public DB {
Status
AllTables(std::vector<meta::TableSchema> &table_schema_array) override;
Status
PreloadTable(const std::string &table_id) override;
Status
GetTableRowCount(const std::string &table_id, uint64_t &row_count) override;
......
......@@ -38,6 +38,9 @@ class Meta {
virtual Status
AllTables(std::vector<TableSchema> &table_schema_array) = 0;
virtual Status
PreloadTable(const std::string &table_id, meta::TableFilesSchema &files) = 0;
virtual Status
DeleteTable(const std::string &table_id) = 0;
......
......@@ -690,6 +690,11 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
return Status::OK();
}
Status
MySQLMetaImpl::PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) {
}
Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
......
......@@ -29,6 +29,7 @@ class MySQLMetaImpl : public Meta {
Status DescribeTable(TableSchema &group_info_) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status AllTables(std::vector<TableSchema> &table_schema_array) override;
Status PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override;
Status DeleteTable(const std::string &table_id) override;
Status DeleteTableFiles(const std::string &table_id) override;
......
......@@ -404,6 +404,40 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
return Status::OK();
}
Status SqliteMetaImpl::PreloadTable(const std::string &table_id, TableFilesSchema &files_schema) {
try {
MetricCollector metric;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_,
&TableFileSchema::dimension_,
&TableFileSchema::location_,
&TableFileSchema::engine_type_),
where(c(&TableFileSchema::table_id_) == table_id));
TableSchema table_schema;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
int64_t size = 0;
for (auto &file : selected) {
TableFileSchema file_schema;
file_schema.size_ = std::get<0>(file);
file_schema.dimension_ = std::get<1>(file);
file_schema.location_ = std::get<2>(file);
file_schema.engine_type_ = std::get<3>(file);
files_schema.push_back(file_schema);
}
} catch (std::exception &e) {
return HandleException("Encounter exception when lookup all tables", e);
}
return Status::OK();
}
Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
if (file_schema.date_ == EmptyDate) {
file_schema.date_ = Meta::GetDate();
......
......@@ -33,6 +33,9 @@ class SqliteMetaImpl : public Meta {
Status
AllTables(std::vector<TableSchema> &table_schema_array) override;
Status
PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override;
Status
DeleteTable(const std::string &table_id) override;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册