Merge branch 'branch-0.3.0' into 'branch-0.3.0'

MS-122 Archive configuration

See merge request megasearch/vecwise_engine!123

Former-commit-id: f62a8253e114635cd13d2859a0ca9500f49c1c10

cpp/conf/server_config.yaml

@@ -8,6 +8,8 @@ db_config:
   db_path: /tmp/milvus             # milvus data storage path
   db_backend_url: http://127.0.0.1 # meta database uri
   index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
+  archive_disk_threshold: 512      # triger archive action if storage size exceed this value, unit: GB
+  archive_days_threshold: 30       # files older than x days will be archived, unit: day
 
 metric_config:
   is_startup: off      # if monitoring start: on, off

cpp/src/db/DBMetaImpl.cpp

@@ -655,7 +655,7 @@ Status DBMetaImpl::Archive() {
     for (auto kv : criterias) {
         auto &criteria = kv.first;
         auto &limit = kv.second;
-        if (criteria == "days") {
+        if (criteria == engine::ARCHIVE_CONF_DAYS) {
             long usecs = limit * D_SEC * US_PS;
             long now = utils::GetMicroSecTimeStamp();
             try {
@@ -671,11 +671,11 @@ Status DBMetaImpl::Archive() {
                 return HandleException("Encounter exception when update table files", e);
             }
         }
-        if (criteria == "disk") {
+        if (criteria == engine::ARCHIVE_CONF_DISK) {
             uint64_t sum = 0;
             Size(sum);
 
-            auto to_delete = (sum - limit * G);
+            int64_t to_delete = (int64_t)sum - limit * G;
             DiscardFiles(to_delete);
         }
     }

cpp/src/db/Options.cpp

@@ -24,6 +24,12 @@ ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias)
     ParseCritirias(criterias);
 }
 
+void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) {
+    for(auto& pair : criterial) {
+        criterias_[pair.first] = pair.second;
+    }
+}
+
 void ArchiveConf::ParseCritirias(const std::string& criterias) {
     std::stringstream ss(criterias);
     std::vector<std::string> tokens;

cpp/src/db/Options.h

@@ -19,14 +19,20 @@ static constexpr uint64_t ONE_KB = 1024;
 static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB;
 static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB;
 
+static const std::string ARCHIVE_CONF_DISK = "disk";
+static const std::string ARCHIVE_CONF_DAYS = "days";
+static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512";
+
 struct ArchiveConf {
     using CriteriaT = std::map<std::string, int>;
 
-    ArchiveConf(const std::string& type, const std::string& criterias = "disk:512");
+    ArchiveConf(const std::string& type, const std::string& criterias = ARCHIVE_CONF_DEFAULT);
 
     const std::string& GetType() const { return type_; }
     const CriteriaT GetCriterias() const { return criterias_; }
 
+    void SetCriterias(const ArchiveConf::CriteriaT& criterial);
+
 private:
     void ParseCritirias(const std::string& type);
     void ParseType(const std::string& criterias);

cpp/src/server/DBWrapper.cpp

@@ -24,6 +24,19 @@ DBWrapper::DBWrapper() {
         opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
     }
 
+    //set archive config
+    engine::ArchiveConf::CriteriaT criterial;
+    int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0);
+    int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0);
+    if(disk > 0) {
+        criterial[engine::ARCHIVE_CONF_DISK] = disk;
+    }
+    if(days > 0) {
+        criterial[engine::ARCHIVE_CONF_DAYS] = days;
+    }
+    opt.meta.archive_conf.SetCriterias(criterial);
+
+    //create db root folder
     CommonUtil::CreateDirectory(opt.meta.path);
 
     zilliz::milvus::engine::DB::Open(opt, &db_);

cpp/src/server/ServerConfig.h

@@ -24,6 +24,8 @@ static const std::string CONFIG_DB = "db_config";
 static const std::string CONFIG_DB_URL = "db_backend_url";
 static const std::string CONFIG_DB_PATH = "db_path";
 static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
+static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
+static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
 
 static const std::string CONFIG_LOG = "log_config";