feat(db): add ttl files cleanup api

Former-commit-id: bb84f6f2baa8c20c9f0e745c67213af236609507

cpp/src/db/DBImpl.cpp

@@ -242,6 +242,8 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
         try_build_index();
     }
 
+    _pMeta->cleanup_ttl_files(1);
+
     return Status::OK();
 }
 

cpp/src/db/DBMetaImpl.cpp

@@ -366,6 +366,38 @@ Status DBMetaImpl::update_files(GroupFilesSchema& files) {
     return Status::OK();
 }
 
+Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
+    auto now = GetMicroSecTimeStamp();
+    auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
+                                               &GroupFileSchema::group_id,
+                                               &GroupFileSchema::file_id,
+                                               &GroupFileSchema::file_type,
+                                               &GroupFileSchema::rows,
+                                               &GroupFileSchema::date),
+                                      where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE and
+                                            c(&GroupFileSchema::updated_time) > now - 1000000*seconds));
+
+    GroupFilesSchema updated;
+
+    for (auto& file : selected) {
+        GroupFileSchema group_file;
+        group_file.id = std::get<0>(file);
+        group_file.group_id = std::get<1>(file);
+        group_file.file_id = std::get<2>(file);
+        group_file.file_type = std::get<3>(file);
+        group_file.rows = std::get<4>(file);
+        group_file.date = std::get<5>(file);
+        GetGroupFilePath(group_file);
+        if (group_file.file_type == GroupFileSchema::TO_DELETE) {
+            boost::filesystem::remove(group_file.location);
+        }
+        ConnectorPtr->remove<GroupFileSchema>(group_file.id);
+        std::cout << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl;
+    }
+
+    return Status::OK();
+}
+
 Status DBMetaImpl::cleanup() {
     auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
                                                &GroupFileSchema::group_id,

cpp/src/db/DBMetaImpl.h

@@ -46,6 +46,8 @@ public:
 
     virtual Status cleanup() override;
 
+    virtual Status cleanup_ttl_files(uint16_t seconds) override;
+
     virtual ~DBMetaImpl();
 
 private:

cpp/src/db/LocalMetaImpl.cpp

@@ -241,6 +241,11 @@ Status LocalMetaImpl::cleanup() {
     return Status::OK();
 }
 
+Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) {
+    // PXU TODO
+    return Status::OK();
+}
+
 } // namespace meta
 } // namespace engine
 } // namespace vecwise

cpp/src/db/LocalMetaImpl.h

@@ -41,6 +41,8 @@ public:
 
     virtual Status files_to_index(GroupFilesSchema&) override;
 
+    virtual Status cleanup_ttl_files(uint16_t seconds) override;
+
 private:
 
     Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info);

cpp/src/db/Meta.h

@@ -81,6 +81,7 @@ public:
     virtual Status files_to_index(GroupFilesSchema&) = 0;
 
     virtual Status cleanup() = 0;
+    virtual Status cleanup_ttl_files(uint16_t) = 0;
 
     static DateT GetDate(const std::time_t& t);
     static DateT GetDate();