From 01e4e605e2c29558354454f8a6b97c4819aefe0a Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:12:56 +0800 Subject: [PATCH] refactor(db): CreateTableFile Former-commit-id: c29b8ccc97cc327ec9c278f2ea5456369365c4f3 --- cpp/src/db/DBImpl.cpp | 38 ++++++------ cpp/src/db/DBMetaImpl.cpp | 28 ++++----- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/MemManager.cpp | 8 +-- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 102 ++++++++++++++++----------------- 6 files changed, 90 insertions(+), 90 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 4d2d51fe..0a09386e 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -259,17 +259,17 @@ void DBImpl::background_call() { template Status DBImpl::merge_files(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { - meta::TableFileSchema group_file; - group_file.table_id = table_id; - group_file.date = date; - Status status = _pMeta->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = table_id; + table_file.date = date; + Status status = _pMeta->CreateTableFile(table_file); if (!status.ok()) { LOG(INFO) << status.ToString() << std::endl; return status; } - EngineT index(group_file.dimension, group_file.location); + EngineT index(table_file.dimension, table_file.location); meta::TableFilesSchema updated; long index_size = 0; @@ -288,14 +288,14 @@ Status DBImpl::merge_files(const std::string& table_id, const meta::Dat index.Serialize(); if (index_size >= _options.index_trigger_size) { - group_file.file_type = meta::TableFileSchema::TO_INDEX; + table_file.file_type = meta::TableFileSchema::TO_INDEX; } else { - group_file.file_type = meta::TableFileSchema::RAW; + table_file.file_type = meta::TableFileSchema::RAW; } - group_file.size = index_size; - updated.push_back(group_file); + table_file.size = index_size; + updated.push_back(table_file); status = _pMeta->update_files(updated); - LOG(DEBUG) << "New merged file " << group_file.file_id << + LOG(DEBUG) << "New merged file " << table_file.file_id << " of size=" << index.PhysicalSize()/(1024*1024) << " M"; index.Cache(); @@ -337,10 +337,10 @@ Status DBImpl::background_merge_files(const std::string& table_id) { template Status DBImpl::build_index(const meta::TableFileSchema& file) { - meta::TableFileSchema group_file; - group_file.table_id = file.table_id; - group_file.date = file.date; - Status status = _pMeta->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = file.table_id; + table_file.date = file.date; + Status status = _pMeta->CreateTableFile(table_file); if (!status.ok()) { return status; } @@ -348,18 +348,18 @@ Status DBImpl::build_index(const meta::TableFileSchema& file) { EngineT to_index(file.dimension, file.location); to_index.Load(); - auto index = to_index.BuildIndex(group_file.location); + auto index = to_index.BuildIndex(table_file.location); - group_file.file_type = meta::TableFileSchema::INDEX; - group_file.size = index->Size(); + table_file.file_type = meta::TableFileSchema::INDEX; + table_file.size = index->Size(); auto to_remove = file; to_remove.file_type = meta::TableFileSchema::TO_DELETE; - meta::TableFilesSchema update_files = {to_remove, group_file}; + meta::TableFilesSchema update_files = {to_remove, table_file}; _pMeta->update_files(update_files); - LOG(DEBUG) << "New index file " << group_file.file_id << " of size " + LOG(DEBUG) << "New index file " << table_file.file_id << " of size " << index->PhysicalSize()/(1024*1024) << " M" << " from file " << to_remove.file_id; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index ec2548c1..b04445cc 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -218,35 +218,35 @@ Status DBMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { return Status::OK(); } -Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { - if (group_file.date == EmptyDate) { - group_file.date = Meta::GetDate(); +Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) { + if (file_schema.date == EmptyDate) { + file_schema.date = Meta::GetDate(); } TableSchema table_schema; - table_schema.table_id = group_file.table_id; + table_schema.table_id = file_schema.table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - NextFileId(group_file.file_id); - group_file.file_type = TableFileSchema::NEW; - group_file.dimension = table_schema.dimension; - group_file.size = 0; - group_file.created_on = utils::GetMicroSecTimeStamp(); - group_file.updated_time = group_file.created_on; - GetGroupFilePath(group_file); + NextFileId(file_schema.file_id); + file_schema.file_type = TableFileSchema::NEW; + file_schema.dimension = table_schema.dimension; + file_schema.size = 0; + file_schema.created_on = utils::GetMicroSecTimeStamp(); + file_schema.updated_time = file_schema.created_on; + GetGroupFilePath(file_schema); { try { - auto id = ConnectorPtr->insert(group_file); - group_file.id = id; + auto id = ConnectorPtr->insert(file_schema); + file_schema.id = id; } catch (...) { return Status::DBTransactionError("Add file Error"); } } - auto partition_path = GetGroupDatePartitionPath(group_file.table_id, group_file.date); + auto partition_path = GetGroupDatePartitionPath(file_schema.table_id, file_schema.date); if (!boost::filesystem::is_directory(partition_path)) { auto ret = boost::filesystem::create_directory(partition_path); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 014cc3d6..f1350cce 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -23,7 +23,7 @@ public: virtual Status DescribeTable(TableSchema& group_info_) override; virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status add_group_file(TableFileSchema& group_file_info) override; + virtual Status CreateTableFile(TableFileSchema& file_schema) override; virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) override; diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index bd69bbc2..60553c78 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -85,14 +85,14 @@ typename MemManager::MemVectorsPtr MemManager::get_mem_by_grou return memIt->second; } - meta::TableFileSchema group_file; - group_file.table_id = table_id; - auto status = _pMeta->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = table_id; + auto status = _pMeta->CreateTableFile(table_file); if (!status.ok()) { return nullptr; } - _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, group_file, options_)); + _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, table_file, options_)); return _memMap[table_id]; } diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 9ad78a5b..6fd00a96 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -26,7 +26,7 @@ public: virtual Status DescribeTable(TableSchema& table_schema) = 0; virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; - virtual Status add_group_file(TableFileSchema& group_file_info) = 0; + virtual Status CreateTableFile(TableFileSchema& file_schema) = 0; virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index b74a65c9..0c8b3f34 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -41,53 +41,53 @@ TEST_F(MetaTest, GROUP_TEST) { ASSERT_TRUE(!status.ok()); } -TEST_F(MetaTest, GROUP_FILE_TEST) { +TEST_F(MetaTest, table_file_TEST) { auto table_id = "meta_test_group"; meta::TableSchema group; group.table_id = table_id; auto status = impl_->CreateTable(group); - meta::TableFileSchema group_file; - group_file.table_id = group.table_id; - status = impl_->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = group.table_id; + status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type, meta::TableFileSchema::NEW); - auto file_id = group_file.file_id; + auto file_id = table_file.file_id; auto new_file_type = meta::TableFileSchema::INDEX; - group_file.file_type = new_file_type; + table_file.file_type = new_file_type; - status = impl_->update_group_file(group_file); + status = impl_->update_group_file(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.file_type, new_file_type); + ASSERT_EQ(table_file.file_type, new_file_type); meta::DatesT dates; dates.push_back(meta::Meta::GetDate()); - status = impl_->delete_group_partitions(group_file.table_id, dates); + status = impl_->delete_group_partitions(table_file.table_id, dates); ASSERT_FALSE(status.ok()); dates.clear(); for (auto i=2; i < 10; ++i) { dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); } - status = impl_->delete_group_partitions(group_file.table_id, dates); + status = impl_->delete_group_partitions(table_file.table_id, dates); ASSERT_TRUE(status.ok()); - group_file.date = meta::Meta::GetDateWithDelta(-2); - status = impl_->update_group_file(group_file); + table_file.date = meta::Meta::GetDateWithDelta(-2); + status = impl_->update_group_file(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2)); - ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date, meta::Meta::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type == meta::TableFileSchema::TO_DELETE); dates.clear(); - dates.push_back(group_file.date); - status = impl_->delete_group_partitions(group_file.table_id, dates); + dates.push_back(table_file.date); + status = impl_->delete_group_partitions(table_file.table_id, dates); ASSERT_TRUE(status.ok()); - status = impl_->get_group_file(group_file.table_id, group_file.file_id, group_file); + status = impl_->get_group_file(table_file.table_id, table_file.file_id, table_file); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE); + ASSERT_TRUE(table_file.file_type == meta::TableFileSchema::TO_DELETE); } TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { @@ -107,19 +107,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { auto status = impl.CreateTable(group); meta::TableFilesSchema files; - meta::TableFileSchema group_file; - group_file.table_id = group.table_id; + meta::TableFileSchema table_file; + table_file.table_id = group.table_id; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); std::vector days; for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::NEW; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::NEW; + status = impl_->update_group_file(table_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::RAW; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::RAW; + status = impl_->update_group_file(table_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::TO_INDEX; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::TO_INDEX; + status = impl_->update_group_file(table_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::INDEX; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::INDEX; + status = impl_->update_group_file(table_file); } meta::TableFilesSchema files; @@ -231,15 +231,15 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { meta::DatePartionedTableFilesSchema dated_files; status = impl_->files_to_merge(group.table_id, dated_files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt); + ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt); status = impl_->files_to_index(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatesT dates = {group_file.date}; + meta::DatesT dates = {table_file.date}; status = impl_->files_to_search(table_id, dates, dated_files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(dated_files[group_file.date].size(), + ASSERT_EQ(dated_files[table_file.date].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); } -- GitLab