Merge branch 'branch-0.4.0' into 'branch-0.4.0'

refine code

See merge request megasearch/milvus!416

Former-commit-id: d6336613e508359cc770cc0ee2609ddcc7ced329

cpp/conf/server_config.template

 server_config:
-  address: 0.0.0.0
+  address: 0.0.0.0            # milvus server ip address
   port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
   gpu_index: 0                # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
   mode: single                # milvus deployment type: single, cluster, read_only
@@ -7,43 +7,35 @@ server_config:
 db_config:
   db_path: @MILVUS_DB_PATH@             # milvus data storage path
   db_slave_path:                        # secondry data storage path, split by semicolon
-
-  parallel_reduce: false                 # use multi-threads to reduce topk result
+  parallel_reduce: false                # use multi-threads to reduce topk result
 
   # URI format: dialect://username:password@host:port/database
   # All parts except dialect are optional, but you MUST include the delimiters
   # Currently dialect supports mysql or sqlite
   db_backend_url: sqlite://:@:/
 
-  index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
   archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
   archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day
   insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
                                    # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
 
 metric_config:
-  is_startup: off      # if monitoring start: on, off
-  collector: prometheus # metrics collector: prometheus
-  prometheus_config:    # following are prometheus configure
-    port: 8080          # the port prometheus use to fetch metrics
-    push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address
-    push_gateway_port: 9091            # push method configure: push gateway port
-
-
-license_config:         # license configure
-  license_path: "@MILVUS_DB_PATH@/system.license" # license file path
-
-cache_config:           # cache configure
-  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
-  cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-  insert_cache_immediately: false  # insert data will be load into cache immediately for hot query
-  gpu_cache_capacity: 5    # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
-  gpu_cache_free_percent: 0.85     # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-  gpu_ids: 0,1             # gpu id
+  is_startup: off                       # if monitoring start: on, off
+  collector: prometheus                 # metrics collector: prometheus
+  prometheus_config:                    # following are prometheus configure
+    port: 8080                          # the port prometheus use to fetch metrics
+    push_gateway_ip_address: 127.0.0.1  # push method configure: push gateway ip address
+    push_gateway_port: 9091             # push method configure: push gateway port
+
+cache_config:
+  cpu_cache_capacity: 16            # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+  cpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
+  insert_cache_immediately: false   # insert data will be load into cache immediately for hot query
+  gpu_cache_capacity: 5             # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
+  gpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
+  gpu_ids: 0,1                      # gpu id
 
 engine_config:
-  nprobe: 10
-  nlist: 16384
   use_blas_threshold: 20
-  metric_type: L2               # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP
-  omp_thread_num: 0             # how many compute threads be used by engine, 0 means use all cpu core to compute
+  metric_type: L2                   # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP
+  omp_thread_num: 0                 # how many cpu cores be used by engine, 0 means use all cpu cores used

cpp/src/db/DBImpl.cpp

@@ -396,7 +396,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
         ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_;
         index_size = index->Size();
 
-        if (index_size >= options_.index_trigger_size) break;
+        if (index_size >= file_schema.index_file_size_) break;
     }
 
     //step 3: serialize to disk
@@ -546,6 +546,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
         //step 2: drop old index files
         DropIndex(table_id);
 
+        if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
+            ENGINE_LOG_DEBUG << "index type = IDMAP, no need to build index";
+            return Status::OK();
+        }
+
         //step 3: update index info
 
         status = meta_ptr_->UpdateTableIndexParam(table_id, index);
@@ -607,7 +612,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
 
         try {
             server::CollectBuildIndexMetrics metrics;
-            index = to_index->BuildIndex(table_file.location_);
+            index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
         } catch (std::exception& ex) {
             //typical error: out of gpu memory
             std::string msg = "BuildIndex encounter exception" + std::string(ex.what());

cpp/src/db/Options.h

@@ -55,9 +55,8 @@ struct Options {
     } MODE;
 
     Options();
-    uint16_t  memory_sync_interval = 1;             //unit: second
+
     uint16_t  merge_trigger_number = 2;
-    size_t  index_trigger_size = ONE_GB;            //unit: byte
     DBMetaOptions meta;
     int mode = MODE::SINGLE;
 

cpp/src/db/engine/ExecutionEngine.h

@@ -60,7 +60,7 @@ public:
                   float *distances,
                   long *labels) const = 0;
 
-    virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string&) = 0;
+    virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string& location, EngineType engine_type) = 0;
 
     virtual Status Cache() = 0;
 

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -214,11 +214,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
 }
 
 ExecutionEnginePtr
-ExecutionEngineImpl::BuildIndex(const std::string &location) {
+ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) {
     ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
 
     auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
-    auto to_index = CreatetVecIndex(index_type_);
+    auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
         throw Exception("Create Empty VecIndex");
     }
@@ -236,7 +236,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location) {
                                  build_cfg);
     if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
 
-    return std::make_shared<ExecutionEngineImpl>(to_index, location, index_type_, metric_type_, nlist_);
+    return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
 }
 
 Status ExecutionEngineImpl::Search(long n,

cpp/src/db/engine/ExecutionEngineImpl.h

@@ -59,7 +59,7 @@ public:
                   float *distances,
                   long *labels) const override;
 
-    ExecutionEnginePtr BuildIndex(const std::string &) override;
+    ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
 
     Status Cache() override;
 

cpp/src/db/insert/MemTableFile.cpp

@@ -87,7 +87,7 @@ Status MemTableFile::Serialize() {
     table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
     table_file_schema_.row_count_ = execution_engine_->Count();
 
-    table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ?
+    table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
                                     meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
 
     auto status = meta_->UpdateTableFile(table_file_schema_);

cpp/src/db/meta/MetaTypes.h

@@ -19,7 +19,7 @@ namespace meta {
 
 constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
 constexpr int32_t DEFAULT_NLIST = 16384;
-constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB;
+constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
 constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
 
 constexpr int64_t FLAG_MASK_USERID = 1;
@@ -71,6 +71,7 @@ struct TableFileSchema {
     int64_t created_on_ = 0;
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
+    int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
     int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta
 }; // TableFileSchema
 

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -835,6 +835,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.updated_time_ = file_schema.created_on_;
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
+        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.metric_type_ = table_schema.metric_type_;
         utils::GetTableFilePath(options_, file_schema);
 
@@ -949,8 +950,9 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
                 groups[table_file.table_id_] = table_schema;
 
             }
-            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
             table_file.nlist_ = groups[table_file.table_id_].nlist_;
+            table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
+            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
             table_file.dimension_ = groups[table_file.table_id_].dimension_;
 
             utils::GetTableFilePath(options_, table_file);
@@ -1043,10 +1045,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
 
             table_file.engine_type_ = resRow["engine_type"];
 
-            table_file.metric_type_ = table_schema.metric_type_;
-
             table_file.nlist_ = table_schema.nlist_;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
+            table_file.metric_type_ = table_schema.metric_type_;
+
             std::string file_id;
             resRow["file_id"].to_string(file_id);
             table_file.file_id_ = file_id;
@@ -1155,10 +1159,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
 
             table_file.engine_type_ = resRow["engine_type"];
 
-            table_file.metric_type_ = table_schema.metric_type_;
-
             table_file.nlist_ = table_schema.nlist_;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
+            table_file.metric_type_ = table_schema.metric_type_;
+
             std::string file_id;
             resRow["file_id"].to_string(file_id);
             table_file.file_id_ = file_id;
@@ -1255,10 +1261,12 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
 
             table_file.engine_type_ = resRow["engine_type"];
 
-            table_file.metric_type_ = table_schema.metric_type_;
-
             table_file.nlist_ = table_schema.nlist_;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
+            table_file.metric_type_ = table_schema.metric_type_;
+
             table_file.created_on_ = resRow["created_on"];
 
             table_file.dimension_ = table_schema.dimension_;
@@ -1338,10 +1346,12 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
 
             file_schema.engine_type_ = resRow["engine_type"];
 
-            file_schema.metric_type_ = table_schema.metric_type_;
-
             file_schema.nlist_ = table_schema.nlist_;
 
+            file_schema.index_file_size_ = table_schema.index_file_size_;
+
+            file_schema.metric_type_ = table_schema.metric_type_;
+
             std::string file_id;
             resRow["file_id"].to_string(file_id);
             file_schema.file_id_ = file_id;

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -548,6 +548,7 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.updated_time_ = file_schema.created_on_;
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
+        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.metric_type_ = table_schema.metric_type_;
 
         //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@@ -608,8 +609,9 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
                 }
                 groups[table_file.table_id_] = table_schema;
             }
-            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
             table_file.nlist_ = groups[table_file.table_id_].nlist_;
+            table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
+            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
             table_file.dimension_ = groups[table_file.table_id_].dimension_;
             files.push_back(table_file);
         }
@@ -703,9 +705,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
                 table_file.row_count_ = std::get<5>(file);
                 table_file.date_ = std::get<6>(file);
                 table_file.engine_type_ = std::get<7>(file);
-                table_file.metric_type_ = table_schema.metric_type_;
                 table_file.nlist_ = table_schema.nlist_;
+                table_file.index_file_size_ = table_schema.index_file_size_;
+                table_file.metric_type_ = table_schema.metric_type_;
                 table_file.dimension_ = table_schema.dimension_;
+
                 utils::GetTableFilePath(options_, table_file);
                 auto dateItr = files.find(table_file.date_);
                 if (dateItr == files.end()) {
@@ -782,8 +786,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
             table_file.date_ = std::get<6>(file);
             table_file.engine_type_ = std::get<7>(file);
             table_file.dimension_ = table_schema.dimension_;
-            table_file.metric_type_ = table_schema.metric_type_;
             table_file.nlist_ = table_schema.nlist_;
+            table_file.index_file_size_ = table_schema.index_file_size_;
+            table_file.metric_type_ = table_schema.metric_type_;
+
             utils::GetTableFilePath(options_, table_file);
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
@@ -842,8 +848,10 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
             table_file.date_ = std::get<6>(file);
             table_file.created_on_ = std::get<7>(file);
             table_file.dimension_ = table_schema.dimension_;
-            table_file.metric_type_ = table_schema.metric_type_;
             table_file.nlist_ = table_schema.nlist_;
+            table_file.index_file_size_ = table_schema.index_file_size_;
+            table_file.metric_type_ = table_schema.metric_type_;
+
             utils::GetTableFilePath(options_, table_file);
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
@@ -892,8 +900,9 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
             file_schema.row_count_ = std::get<4>(file);
             file_schema.date_ = std::get<5>(file);
             file_schema.engine_type_ = std::get<6>(file);
-            file_schema.metric_type_ = table_schema.metric_type_;
             file_schema.nlist_ = table_schema.nlist_;
+            file_schema.index_file_size_ = table_schema.index_file_size_;
+            file_schema.metric_type_ = table_schema.metric_type_;
             file_schema.created_on_ = std::get<7>(file);
             file_schema.dimension_ = table_schema.dimension_;
 

cpp/src/server/DBWrapper.cpp

@@ -27,19 +27,6 @@ DBWrapper::DBWrapper() {
     std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH);
     StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths);
 
-    int64_t index_size = db_config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
-    if(index_size > 0) {//ensure larger than zero, unit is MB
-        opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
-    }
-    int64_t insert_buffer_size = db_config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
-    if (insert_buffer_size >= 1) {
-        opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB;
-    }
-    else {
-        std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl;
-        kill(0, SIGUSR1);
-    }
-
     // cache config
     ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE);
     opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false);

cpp/src/server/ServerConfig.cpp

@@ -79,19 +79,6 @@ ServerError ServerConfig::ValidateConfig() const {
         return SERVER_INVALID_ARGUMENT;
     }
 
-    uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE, 1024);
-    index_building_threshold *= MB;
-
-    size_t gpu_mem = 0;
-    ValidationUtil::GetGpuMemory(gpu_index, gpu_mem);
-    if(index_building_threshold >= gpu_mem) {
-        std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl;
-        return SERVER_INVALID_ARGUMENT;
-    } else if(index_building_threshold >= gpu_mem/3) {
-        std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, "
-            << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl;
-    }
-
     //cache config validation
     ConfigNode cache_config = GetConfig(CONFIG_CACHE);
     uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY, 16);

cpp/src/server/ServerConfig.h

@@ -24,7 +24,6 @@ static const char* CONFIG_DB = "db_config";
 static const char* CONFIG_DB_URL = "db_backend_url";
 static const char* CONFIG_DB_PATH = "db_path";
 static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path";
-static const char* CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
 static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
 static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
 static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";

cpp/unittest/server/config_test.cpp

@@ -135,12 +135,6 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
     err = config.ValidateConfig();
     ASSERT_NE(err, server::SERVER_SUCCESS);
 
-    size_t index_building_threshold = (gpu_mem + 1*MB)/MB;
-    db_config.SetValue(server::CONFIG_DB_INDEX_TRIGGER_SIZE,
-            std::to_string(index_building_threshold));
-    err = config.ValidateConfig();
-    ASSERT_NE(err, server::SERVER_SUCCESS);
-
     insert_buffer_size = total_mem/GB + 2;
     db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size));
     err = config.ValidateConfig();