From 3d330e0aa47a602e14c8811309858bc7840792d0 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:45:39 +0800 Subject: [PATCH] refactor(db): change group_info to table_schema and add_group to CreateTable Former-commit-id: e8d48ec8e54429bf5d1080997ca329b42ddcab82 --- cpp/src/db/DB.h | 4 +- cpp/src/db/DBImpl.cpp | 8 ++-- cpp/src/db/DBImpl.h | 4 +- cpp/src/db/DBMetaImpl.cpp | 80 +++++++++++++++++----------------- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/LocalMetaImpl.cpp | 60 ++++++++++++------------- cpp/src/db/LocalMetaImpl.h | 8 ++-- cpp/src/db/Meta.h | 4 +- cpp/unittest/db/meta_tests.cpp | 12 ++--- 9 files changed, 91 insertions(+), 91 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 20e49240..1307d8ed 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -21,8 +21,8 @@ class DB { public: static void Open(const Options& options, DB** dbptr); - virtual Status add_group(meta::TableSchema& group_info_) = 0; - virtual Status get_group(meta::TableSchema& group_info_) = 0; + virtual Status add_group(meta::TableSchema& table_schema_) = 0; + virtual Status get_group(meta::TableSchema& table_schema_) = 0; virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 54a13800..3d202e65 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -35,13 +35,13 @@ DBImpl::DBImpl(const Options& options) } template -Status DBImpl::add_group(meta::TableSchema& group_info) { - return _pMeta->add_group(group_info); +Status DBImpl::add_group(meta::TableSchema& table_schema) { + return _pMeta->CreateTable(table_schema); } template -Status DBImpl::get_group(meta::TableSchema& group_info) { - return _pMeta->get_group(group_info); +Status DBImpl::get_group(meta::TableSchema& table_schema) { + return _pMeta->get_group(table_schema); } template diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 35a4a93e..19025cf2 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -33,8 +33,8 @@ public: DBImpl(const Options& options); - virtual Status add_group(meta::TableSchema& group_info) override; - virtual Status get_group(meta::TableSchema& group_info) override; + virtual Status add_group(meta::TableSchema& table_schema) override; + virtual Status get_group(meta::TableSchema& table_schema) override; virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 41fceb5c..54f65f57 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -117,9 +117,9 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id, return Status::OK(); } - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group(table_schema); if (!status.ok()) { return status; } @@ -148,24 +148,24 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id, return Status::OK(); } -Status DBMetaImpl::add_group(TableSchema& group_info) { - if (group_info.table_id == "") { - NextGroupId(group_info.table_id); +Status DBMetaImpl::CreateTable(TableSchema& table_schema) { + if (table_schema.table_id == "") { + NextGroupId(table_schema.table_id); } - group_info.files_cnt = 0; - group_info.id = -1; - group_info.created_on = utils::GetMicroSecTimeStamp(); + table_schema.files_cnt = 0; + table_schema.id = -1; + table_schema.created_on = utils::GetMicroSecTimeStamp(); { try { - auto id = ConnectorPtr->insert(group_info); - group_info.id = id; + auto id = ConnectorPtr->insert(table_schema); + table_schema.id = id; } catch (...) { return Status::DBTransactionError("Add Group Error"); } } - auto group_path = GetGroupPath(group_info.table_id); + auto group_path = GetGroupPath(table_schema.table_id); if (!boost::filesystem::is_directory(group_path)) { auto ret = boost::filesystem::create_directories(group_path); @@ -178,24 +178,24 @@ Status DBMetaImpl::add_group(TableSchema& group_info) { return Status::OK(); } -Status DBMetaImpl::get_group(TableSchema& group_info) { - return get_group_no_lock(group_info); +Status DBMetaImpl::get_group(TableSchema& table_schema) { + return get_group_no_lock(table_schema); } -Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { +Status DBMetaImpl::get_group_no_lock(TableSchema& table_schema) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id, &TableSchema::table_id, &TableSchema::files_cnt, &TableSchema::dimension), - where(c(&TableSchema::table_id) == group_info.table_id)); + where(c(&TableSchema::table_id) == table_schema.table_id)); assert(groups.size() <= 1); if (groups.size() == 1) { - group_info.id = std::get<0>(groups[0]); - group_info.files_cnt = std::get<2>(groups[0]); - group_info.dimension = std::get<3>(groups[0]); + table_schema.id = std::get<0>(groups[0]); + table_schema.files_cnt = std::get<2>(groups[0]); + table_schema.dimension = std::get<3>(groups[0]); } else { - return Status::NotFound("Group " + group_info.table_id + " not found"); + return Status::NotFound("Group " + table_schema.table_id + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -226,16 +226,16 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { if (group_file.date == EmptyDate) { group_file.date = Meta::GetDate(); } - TableSchema group_info; - group_info.table_id = group_file.table_id; - auto status = get_group(group_info); + TableSchema table_schema; + table_schema.table_id = group_file.table_id; + auto status = get_group(table_schema); if (!status.ok()) { return status; } NextFileId(group_file.file_id); group_file.file_type = TableFileSchema::NEW; - group_file.dimension = group_info.dimension; + group_file.dimension = table_schema.dimension; group_file.size = 0; group_file.created_on = utils::GetMicroSecTimeStamp(); group_file.updated_time = group_file.created_on; @@ -288,13 +288,13 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { GetGroupFilePath(group_file); auto groupItr = groups.find(group_file.table_id); if (groupItr == groups.end()) { - TableSchema group_info; - group_info.table_id = group_file.table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = group_file.table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } - groups[group_file.table_id] = group_info; + groups[group_file.table_id] = table_schema; } group_file.dimension = groups[group_file.table_id].dimension; files.push_back(group_file); @@ -327,9 +327,9 @@ Status DBMetaImpl::files_to_search(const std::string &table_id, c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX))); - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } @@ -342,7 +342,7 @@ Status DBMetaImpl::files_to_search(const std::string &table_id, group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); - group_file.dimension = group_info.dimension; + group_file.dimension = table_schema.dimension; GetGroupFilePath(group_file); auto dateItr = files.find(group_file.date); if (dateItr == files.end()) { @@ -372,9 +372,9 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and c(&TableFileSchema::table_id) == table_id)); - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } @@ -387,7 +387,7 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); - group_file.dimension = group_info.dimension; + group_file.dimension = table_schema.dimension; GetGroupFilePath(group_file); auto dateItr = files.find(group_file.date); if (dateItr == files.end()) { @@ -670,9 +670,9 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) { c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and c(&TableFileSchema::table_id) == table_id)); - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } @@ -682,7 +682,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) { result += std::get<0>(file); } - result /= group_info.dimension; + result /= table_schema.dimension; } catch (std::exception & e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index b278bee2..2706e43e 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -19,7 +19,7 @@ class DBMetaImpl : public Meta { public: DBMetaImpl(const DBMetaOptions& options_); - virtual Status add_group(TableSchema& group_info) override; + virtual Status CreateTable(TableSchema& table_schema) override; virtual Status get_group(TableSchema& group_info_) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index 70a07350..c8c320fc 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) { return GetGroupMetaPathByGroupPath(GetGroupPath(table_id)); } -Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) { +Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema) { boost::property_tree::ptree ptree; boost::property_tree::read_json(path, ptree); auto files_cnt = ptree.get_child("files_cnt").data(); @@ -62,18 +62,18 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchem /* std::cout << dimension << std::endl; */ /* std::cout << files_cnt << std::endl; */ - group_info.id = std::stoi(group_info.table_id); - group_info.files_cnt = std::stoi(files_cnt); - group_info.dimension = std::stoi(dimension); - group_info.location = GetGroupPath(group_info.table_id); + table_schema.id = std::stoi(table_schema.table_id); + table_schema.files_cnt = std::stoi(files_cnt); + table_schema.dimension = std::stoi(dimension); + table_schema.location = GetGroupPath(table_schema.table_id); return Status::OK(); } -Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info) { - group_info.table_id = table_id; - return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), group_info); +Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema) { + table_schema.table_id = table_id; + return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), table_schema); } LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_) @@ -90,15 +90,15 @@ Status LocalMetaImpl::initialize() { return Status::OK(); } -Status LocalMetaImpl::add_group(TableSchema& group_info) { +Status LocalMetaImpl::CreateTable(TableSchema& table_schema) { std::string real_gid; size_t id = SimpleIDGenerator().getNextIDNumber(); - if (group_info.table_id == "") { + if (table_schema.table_id == "") { std::stringstream ss; ss << id; real_gid = ss.str(); } else { - real_gid = group_info.table_id; + real_gid = table_schema.table_id; } bool group_exist; @@ -110,27 +110,27 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) { return Status::GroupError("Cannot Create Group " + real_gid); } - group_info.table_id = real_gid; - group_info.files_cnt = 0; - group_info.id = 0; - group_info.location = GetGroupPath(real_gid); + table_schema.table_id = real_gid; + table_schema.files_cnt = 0; + table_schema.id = 0; + table_schema.location = GetGroupPath(real_gid); boost::property_tree::ptree out; - out.put("files_cnt", group_info.files_cnt); - out.put("dimension", group_info.dimension); + out.put("files_cnt", table_schema.files_cnt); + out.put("dimension", table_schema.dimension); boost::property_tree::write_json(GetGroupMetaPath(real_gid), out); return Status::OK(); } -Status LocalMetaImpl::get_group(TableSchema& group_info) { +Status LocalMetaImpl::get_group(TableSchema& table_schema) { bool group_exist; - has_group(group_info.table_id, group_exist); + has_group(table_schema.table_id, group_exist); if (!group_exist) { - return Status::NotFound("Group " + group_info.table_id + " Not Found"); + return Status::NotFound("Group " + table_schema.table_id + " Not Found"); } - return GetGroupMetaInfo(group_info.table_id, group_info); + return GetGroupMetaInfo(table_schema.table_id, table_schema); } Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { @@ -139,14 +139,14 @@ Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { } Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { - TableSchema group_info; - /* auto status = get_group(group_info); */ + TableSchema table_schema; + /* auto status = get_group(table_schema); */ /* if (!status.ok()) { */ /* return status; */ /* } */ /* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */ /* group_file_info.table_id = table_id; */ - /* group_file_info.dimension = group_info.dimension; */ + /* group_file_info.dimension = table_schema.dimension; */ /* group_file_info.location = location; */ /* group_file_info.date = date; */ return Status::OK(); @@ -159,8 +159,8 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { boost::filesystem::directory_iterator end_itr; for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { auto group_path = itr->path().string(); - TableSchema group_info; - GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); + TableSchema table_schema; + GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { auto partition_path = innerItr->path().string(); for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { @@ -172,7 +172,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { TableFileSchema f; f.location = location; /* f.table_id = table_id; */ - f.dimension = group_info.dimension; + f.dimension = table_schema.dimension; files.push_back(f); } } @@ -188,8 +188,8 @@ Status LocalMetaImpl::files_to_merge(const std::string& table_id, /* boost::filesystem::directory_iterator end_itr; */ /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */ /* auto group_path = itr->path().string(); */ - /* TableSchema group_info; */ - /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */ + /* TableSchema table_schema; */ + /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); */ /* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */ /* auto partition_path = innerItr->path().string(); */ /* for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */ @@ -201,7 +201,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& table_id, /* TableFileSchema f; */ /* f.location = location; */ /* f.table_id = table_id; */ - /* f.dimension = group_info.dimension; */ + /* f.dimension = table_schema.dimension; */ /* files.push_back(f); */ /* } */ /* } */ diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index e57f0a97..46ecdd41 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -18,8 +18,8 @@ public: const size_t INDEX_TRIGGER_SIZE = 1024*1024*500; LocalMetaImpl(const DBMetaOptions& options_); - virtual Status add_group(TableSchema& group_info_) override; - virtual Status get_group(TableSchema& group_info_) override; + virtual Status CreateTable(TableSchema& table_schema) override; + virtual Status get_group(TableSchema& table_schema_) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; @@ -59,9 +59,9 @@ public: private: - Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info); + Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema); std::string GetGroupMetaPathByGroupPath(const std::string& group_path); - Status GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info); + Status GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema); std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, TableFileSchema::FILE_TYPE file_type); std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 90b3384c..014b04c9 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -22,8 +22,8 @@ class Meta { public: using Ptr = std::shared_ptr; - virtual Status add_group(TableSchema& group_info) = 0; - virtual Status get_group(TableSchema& group_info) = 0; + virtual Status CreateTable(TableSchema& table_schema) = 0; + virtual Status get_group(TableSchema& table_schema) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 2f1bd4fb..0c8d52f7 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -22,7 +22,7 @@ TEST_F(MetaTest, GROUP_TEST) { meta::TableSchema group; group.table_id = table_id; - auto status = impl_->add_group(group); + auto status = impl_->CreateTable(group); ASSERT_TRUE(status.ok()); auto gid = group.id; @@ -37,7 +37,7 @@ TEST_F(MetaTest, GROUP_TEST) { ASSERT_TRUE(!status.ok()); group.table_id = table_id; - status = impl_->add_group(group); + status = impl_->CreateTable(group); ASSERT_TRUE(!status.ok()); } @@ -46,7 +46,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { meta::TableSchema group; group.table_id = table_id; - auto status = impl_->add_group(group); + auto status = impl_->CreateTable(group); meta::TableFileSchema group_file; group_file.table_id = group.table_id; @@ -104,7 +104,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { meta::TableSchema group; group.table_id = table_id; - auto status = impl.add_group(group); + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; @@ -150,7 +150,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { meta::TableSchema group; group.table_id = table_id; - auto status = impl.add_group(group); + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; @@ -188,7 +188,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { meta::TableSchema group; group.table_id = table_id; - auto status = impl_->add_group(group); + auto status = impl_->CreateTable(group); int new_files_cnt = 4; int raw_files_cnt = 5; -- GitLab