From 4eea03af5cf9cb91375e325b707192d23035f9a7 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Sun, 19 May 2019 21:50:24 +0800 Subject: [PATCH] feat(db): add simple delete partitions Former-commit-id: ed39bb26d68f4ea576e824c081e72bfb17d15b8a --- cpp/src/db/DB.h | 2 ++ cpp/src/db/DBImpl.cpp | 6 ++++++ cpp/src/db/DBImpl.h | 1 + cpp/src/db/DBMetaImpl.cpp | 38 ++++++++++++++++++++++++++++++++++ cpp/src/db/DBMetaImpl.h | 2 ++ cpp/src/db/LocalMetaImpl.h | 4 +++- cpp/src/db/Meta.cpp | 21 ++++++++++++++++--- cpp/src/db/Meta.h | 6 ++++-- cpp/unittest/db/meta_tests.cpp | 5 +++++ 9 files changed, 79 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 450a980c..6f922475 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -23,6 +23,8 @@ public: virtual Status add_group(meta::GroupSchema& group_info_) = 0; virtual Status get_group(meta::GroupSchema& group_info_) = 0; + virtual Status delete_vectors(const std::string& group_id, + const meta::DatesT& dates) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status get_group_files(const std::string& group_id_, const int date_delta_, diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 216a9b35..72b05323 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -44,6 +44,12 @@ Status DBImpl::get_group(meta::GroupSchema& group_info) { return _pMeta->get_group(group_info); } +template +Status DBImpl::delete_vectors(const std::string& group_id, + const meta::DatesT& dates) { + return _pMeta->delete_group_partitions(group_id, dates); +} + template Status DBImpl::has_group(const std::string& group_id_, bool& has_or_not_) { return _pMeta->has_group(group_id_, has_or_not_); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index d2aed0af..45f249bd 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -35,6 +35,7 @@ public: virtual Status add_group(meta::GroupSchema& group_info) override; virtual Status get_group(meta::GroupSchema& group_info) override; + virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status get_group_files(const std::string& group_id_, diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index aaaaf21c..b1518081 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -104,6 +104,44 @@ Status DBMetaImpl::initialize() { return Status::OK(); } +// PXU TODO: Temp solution. Will fix later +Status DBMetaImpl::delete_group_partitions(const std::string& group_id, + const meta::DatesT& dates) { + if (dates.size() == 0) { + return Status::OK(); + } + + GroupSchema group_info; + group_info.group_id = group_id; + auto status = get_group(group_info); + if (!status.ok()) { + return status; + } + + auto yesterday = GetDate(-2); + + for (auto& date : dates) { + if (date >= yesterday) { + return Status::Error("Could not delete partitions with 2 days"); + } + } + + try { + ConnectorPtr->update_all( + set( + c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE + ), + where( + c(&GroupFileSchema::group_id) == group_id and + in(&GroupFileSchema::date, dates) + )); + } catch (std::exception & e) { + LOG(DEBUG) << e.what(); + throw e; + } + return Status::OK(); +} + Status DBMetaImpl::add_group(GroupSchema& group_info) { if (group_info.group_id == "") { std::stringstream ss; diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index aca0ec31..fab2e156 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -24,6 +24,8 @@ public: virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status add_group_file(GroupFileSchema& group_file_info) override; + virtual Status delete_group_partitions(const std::string& group_id, + const meta::DatesT& dates) override; virtual Status has_group_file(const std::string& group_id_, const std::string& file_id_, diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index 4c324c57..116c9e46 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -22,7 +22,9 @@ public: virtual Status get_group(GroupSchema& group_info_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; - virtual Status add_group_file(GroupFileSchema& group_file_info) = 0; + virtual Status add_group_file(GroupFileSchema& group_file_info) override; + /* virtual Status delete_group_partitions(const std::string& group_id, */ + /* const meta::DatesT& dates) override; */ virtual Status has_group_file(const std::string& group_id_, const std::string& file_id_, diff --git a/cpp/src/db/Meta.cpp b/cpp/src/db/Meta.cpp index 1b97c06c..1bd3a206 100644 --- a/cpp/src/db/Meta.cpp +++ b/cpp/src/db/Meta.cpp @@ -11,13 +11,28 @@ namespace vecwise { namespace engine { namespace meta { -DateT Meta::GetDate(const std::time_t& t) { +DateT Meta::GetDate(const std::time_t& t, int day_delta) { tm *ltm = std::localtime(&t); + if (day_delta > 0) { + do { + ++ltm->tm_mday; + --day_delta; + } while(day_delta > 0); + mktime(ltm); + } else if (day_delta < 0) { + do { + --ltm->tm_mday; + ++day_delta; + } while(day_delta < 0); + mktime(ltm); + } else { + ltm->tm_mday; + } return ltm->tm_year*10000 + ltm->tm_mon*100 + ltm->tm_mday; } -DateT Meta::GetDate() { - return GetDate(std::time(nullptr)); +DateT Meta::GetDate(int day_delta) { + return GetDate(std::time(nullptr), day_delta); } } // namespace meta diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index e0c1a84c..ffc35b06 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -67,6 +67,8 @@ public: virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status add_group_file(GroupFileSchema& group_file_info) = 0; + virtual Status delete_group_partitions(const std::string& group_id, + const meta::DatesT& dates) = 0; virtual Status has_group_file(const std::string& group_id_, const std::string& file_id_, @@ -98,8 +100,8 @@ public: virtual Status count(const std::string& group_id, long& result) = 0; - static DateT GetDate(const std::time_t& t); - static DateT GetDate(); + static DateT GetDate(const std::time_t& t, int day_delta); + static DateT GetDate(int day_delta = 0); }; // MetaData diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 2ede5398..dc11feaf 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -59,6 +59,11 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(group_file.file_type, new_file_type); + meta::DatesT dates; + dates.push_back(meta::Meta::GetDate()); + status = impl_->delete_group_partitions(group_file.group_id, dates); + ASSERT_FALSE(status.ok()); + /* group_file.file_type = meta::GroupFileSchema::NEW; */ /* status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); */ /* ASSERT_TRUE(status.ok()); */ -- GitLab