Merge branch 'branch-0.3.0' into 'branch-0.3.0'

MS-22 Enhancement for MemVector size control

See merge request megasearch/vecwise_engine!95

Former-commit-id: 941450e120d8b3d60513d0801540d789e3550bbd

cpp/conf/server_config.yaml

@@ -8,8 +8,8 @@ server_config:
 db_config:
   db_path: /tmp/milvus
   db_backend_url: http://127.0.0.1
-  db_flush_interval: 5              #unit: second
-  idmapper_max_open_file: 128
+  db_flush_interval: 5                    #flush cache data into disk at intervals, unit: second
+  index_building_threshold: 1024          #build index file when raw data file size larger than this value, unit: MB
 
 metric_config:
   is_startup: true # true is on, false is off
@@ -24,5 +24,4 @@ license_config:
   license_path: "/tmp/system.license"
 
 cache_config:
-    cpu_cache_capacity: 16   # unit: GB
-    gpu_cache_capacity:  2   # unit: GB
\ No newline at end of file
+    cpu_cache_capacity: 16   # memory pool to hold index data, unit: GB
\ No newline at end of file

cpp/conf/server_config_template.yaml

-server_config:
-  address: 0.0.0.0
-  port: 33001
-  transfer_protocol: binary   #optional: binary, compact, json
-  server_mode: thread_pool    #optional: simple, thread_pool
-  gpu_index: 0                #which gpu to be used
-
-db_config:
-  db_path: /tmp/milvus
-  db_backend_url: http://127.0.0.1
-  db_flush_interval: 5              #unit: second
-  idmapper_max_open_file: 128
-
-metric_config:
-  is_startup: true # true is on, false is off
-  collector: prometheus # prometheus, now we only have prometheus
-  prometheus_config:
-    collect_type: pull # pull means prometheus pull the message from server, push means server push metric to push gateway
-    port: 8080
-    push_gateway_ip_address: 127.0.0.1
-    push_gateway_port: 9091
-
-license_config:
-  license_path: "/tmp/system.license"
-
-cache_config:
-  cpu_cache_capacity: 16   # unit: GB
-  gpu_cache_capacity:  2   # unit: GB
\ No newline at end of file

cpp/src/db/DBMetaImpl.cpp

@@ -515,7 +515,8 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
                                                      &TableFileSchema::size_,
                                                      &TableFileSchema::date_),
                                              where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and
-                                                 c(&TableFileSchema::table_id_) == table_id));
+                                                 c(&TableFileSchema::table_id_) == table_id),
+                                             order_by(&TableFileSchema::size_).desc());
         auto end_time = METRICS_NOW_TIME;
         auto total_time = METRICS_MICROSECONDS(start_time, end_time);
         server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);

cpp/src/db/Options.h

@@ -15,6 +15,10 @@ namespace engine {
 
 class Env;
 
+static constexpr uint64_t ONE_KB = 1024;
+static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB;
+static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB;
+
 struct ArchiveConf {
     using CriteriaT = std::map<std::string, int>;
 
@@ -40,9 +44,9 @@ struct DBMetaOptions {
 
 struct Options {
     Options();
-    uint16_t  memory_sync_interval = 1;
+    uint16_t  memory_sync_interval = 1;             //unit: second
     uint16_t  merge_trigger_number = 2;
-    size_t  index_trigger_size = 1024*1024*1024;
+    size_t  index_trigger_size = ONE_GB;            //unit: byte
     Env* env;
     DBMetaOptions meta;
 }; // Options

cpp/src/sdk/examples/simple/src/ClientTest.cpp

@@ -20,6 +20,7 @@ namespace {
     static constexpr int64_t TOTAL_ROW_COUNT = 100000;
     static constexpr int64_t TOP_K = 10;
     static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
+    static constexpr int64_t ADD_VECTOR_LOOP = 1;
 
 #define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
 
@@ -156,9 +157,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
 
     {//create table
         TableSchema tb_schema = BuildTableSchema();
-        PrintTableSchema(tb_schema);
         Status stat = conn->CreateTable(tb_schema);
         std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
+        PrintTableSchema(tb_schema);
     }
 
     {//describe table
@@ -168,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
         PrintTableSchema(tb_schema);
     }
 
-    {//add vectors
+    for(int i = 0; i < ADD_VECTOR_LOOP; i++){//add vectors
         std::vector<RowRecord> record_array;
-        BuildVectors(0, TOTAL_ROW_COUNT, record_array);
+        BuildVectors(i*TOTAL_ROW_COUNT, (i+1)*TOTAL_ROW_COUNT, record_array);
         std::vector<int64_t> record_ids;
         Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids);
         std::cout << "AddVector function call status: " << stat.ToString() << std::endl;

cpp/src/server/RequestTask.cpp

@@ -36,6 +36,10 @@ namespace {
             std::string db_path = config.GetValue(CONFIG_DB_PATH);
             opt.memory_sync_interval = (uint16_t)config.GetInt32Value(CONFIG_DB_FLUSH_INTERVAL, 10);
             opt.meta.path = db_path + "/db";
+            int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
+            if(index_size > 0) {//ensure larger than zero, unit is MB
+                opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
+            }
 
             CommonUtil::CreateDirectory(opt.meta.path);
 

cpp/src/server/ServerConfig.h

@@ -24,7 +24,7 @@ static const std::string CONFIG_DB = "db_config";
 static const std::string CONFIG_DB_URL = "db_backend_url";
 static const std::string CONFIG_DB_PATH = "db_path";
 static const std::string CONFIG_DB_FLUSH_INTERVAL = "db_flush_interval";
-static const std::string CONFIG_DB_IDMAPPER_MAX_FILE = "idmapper_max_open_file";
+static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
 
 static const std::string CONFIG_LOG = "log_config";