diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 107b4a10605f4c530e1d7232c4e39d7dfb1de33a..53f2a02cff64daa858fd1de55ae390a5843d6758 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,5 +1,5 @@ server_config: - address: 0.0.0.0 + address: 0.0.0.0 # milvus server ip address port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 mode: single # milvus deployment type: single, cluster, read_only @@ -7,44 +7,34 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path db_slave_path: # secondry data storage path, split by semicolon - - parallel_reduce: false # use multi-threads to reduce topk result + parallel_reduce: false # use multi-threads to reduce topk result # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite db_backend_url: sqlite://:@:/ - index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - collect_type: pull # prometheus collect data method - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port - - -license_config: # license configure - license_path: "@MILVUS_DB_PATH@/system.license" # license file path - -cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - gpu_ids: 0,1 # gpu id + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port + +cache_config: + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 + insert_cache_immediately: false # insert data will be load into cache immediately for hot query + gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory + gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 + gpu_ids: 0,1 # gpu id engine_config: - nprobe: 10 - nlist: 16384 use_blas_threshold: 20 - metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP - omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute + metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP + omp_thread_num: 0 # how many cpu cores be used by engine, 0 means use all cpu cores used diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 6b0b87889ce208cf7b57208d2be729c23f2925ef..d3c7d771813f00d0e4e4849b5306ab3cb6462a22 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -435,7 +435,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; index_size = index->Size(); - if (index_size >= options_.index_trigger_size) break; + if (index_size >= file_schema.index_file_size_) break; } //step 3: serialize to disk @@ -585,6 +585,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //step 2: drop old index files DropIndex(table_id); + if(index.metric_type_ == (int)EngineType::FAISS_IDMAP) { + ENGINE_LOG_DEBUG << "index type = IDMAP, no need to build index"; + return Status::OK(); + } + //step 3: update index info status = meta_ptr_->UpdateTableIndexParam(table_id, index); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 80815312366911a402e72c89bedc3f0bf24dc17e..4cd262ccfad6ef046fab1c11a7f7fea31e5e37e5 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -55,9 +55,8 @@ struct Options { } MODE; Options(); - uint16_t memory_sync_interval = 1; //unit: second + uint16_t merge_trigger_number = 2; - size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index f8f79c8618bcf1afafbc7258da24b720e89c13d1..e18e82973219b8e4317748e78b27aadfac53d214 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -94,7 +94,7 @@ Status MemTableFile::Serialize() { server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size / total_time); - table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? + table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = meta_->UpdateTableFile(table_file_schema_); diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 5fa4c0bcfce506b5f76542a6fcd99b6d7d3a416b..61af7927469211d964596fab62c63625cd77dcc2 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -19,7 +19,7 @@ namespace meta { constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB; +constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; constexpr int64_t FLAG_MASK_USERID = 1; @@ -71,6 +71,7 @@ struct TableFileSchema { int64_t created_on_ = 0; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; //not persist to meta + int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta }; // TableFileSchema diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 3f85b3503f4600db2c32f27011dbc555fcd8efd3..adf6901efc5925ff55763df4f83151ea93b741d6 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -853,6 +853,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, file_schema); @@ -967,8 +968,9 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { groups[table_file.table_id_] = table_schema; } - table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.dimension_ = groups[table_file.table_id_].dimension_; utils::GetTableFilePath(options_, table_file); @@ -1061,10 +1063,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.engine_type_ = resRow["engine_type"]; - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + + table_file.metric_type_ = table_schema.metric_type_; + std::string file_id; resRow["file_id"].to_string(file_id); table_file.file_id_ = file_id; @@ -1173,10 +1177,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.engine_type_ = resRow["engine_type"]; - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + + table_file.metric_type_ = table_schema.metric_type_; + std::string file_id; resRow["file_id"].to_string(file_id); table_file.file_id_ = file_id; @@ -1273,10 +1279,12 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, table_file.engine_type_ = resRow["engine_type"]; - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + + table_file.metric_type_ = table_schema.metric_type_; + table_file.created_on_ = resRow["created_on"]; table_file.dimension_ = table_schema.dimension_; @@ -1356,10 +1364,12 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, file_schema.engine_type_ = resRow["engine_type"]; - file_schema.metric_type_ = table_schema.metric_type_; - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; + + file_schema.metric_type_ = table_schema.metric_type_; + std::string file_id; resRow["file_id"].to_string(file_id); file_schema.file_id_ = file_id; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 0132fa87ceb73e471d5a8a7da178cb9b8a05d439..b3001554dc69711e0044a4b53d534bab96484dab 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -566,6 +566,7 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.metric_type_ = table_schema.metric_type_; //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here @@ -626,8 +627,9 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { } groups[table_file.table_id_] = table_schema; } - table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.dimension_ = groups[table_file.table_id_].dimension_; files.push_back(table_file); } @@ -721,9 +723,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.row_count_ = std::get<5>(file); table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); - table_file.metric_type_ = table_schema.metric_type_; table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.metric_type_ = table_schema.metric_type_; table_file.dimension_ = table_schema.dimension_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -800,8 +804,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); table_file.dimension_ = table_schema.dimension_; - table_file.metric_type_ = table_schema.metric_type_; table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.metric_type_ = table_schema.metric_type_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -860,8 +866,10 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, table_file.date_ = std::get<6>(file); table_file.created_on_ = std::get<7>(file); table_file.dimension_ = table_schema.dimension_; - table_file.metric_type_ = table_schema.metric_type_; table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.metric_type_ = table_schema.metric_type_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -910,8 +918,9 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, file_schema.row_count_ = std::get<4>(file); file_schema.date_ = std::get<5>(file); file_schema.engine_type_ = std::get<6>(file); - file_schema.metric_type_ = table_schema.metric_type_; file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.metric_type_ = table_schema.metric_type_; file_schema.created_on_ = std::get<7>(file); file_schema.dimension_ = table_schema.dimension_; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 923f8f2861d51c251e628e64f0968f7b42acf296..d5ca579773ee86875235096e4c3d034a01c2d7fb 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -27,19 +27,6 @@ DBWrapper::DBWrapper() { std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); - int64_t index_size = db_config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); - if(index_size > 0) {//ensure larger than zero, unit is MB - opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; - } - int64_t insert_buffer_size = db_config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); - if (insert_buffer_size >= 1) { - opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB; - } - else { - std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl; - kill(0, SIGUSR1); - } - // cache config ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 71d32452ac5bfbf6fe4d03d8e2772f21b23c5258..6e740002cf87adb855e44b36525be1f5820e5e38 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -79,19 +79,6 @@ ServerError ServerConfig::ValidateConfig() const { return SERVER_INVALID_ARGUMENT; } - uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE, 1024); - index_building_threshold *= MB; - - size_t gpu_mem = 0; - ValidationUtil::GetGpuMemory(gpu_index, gpu_mem); - if(index_building_threshold >= gpu_mem) { - std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl; - return SERVER_INVALID_ARGUMENT; - } else if(index_building_threshold >= gpu_mem/3) { - std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " - << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; - } - //cache config validation ConfigNode cache_config = GetConfig(CONFIG_CACHE); uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY, 16); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index a800664ebd38b83f38a02a8011113db35e306ce3..2979e35c15568473ff6504fd6c5317f7d95b76f1 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -24,7 +24,6 @@ static const char* CONFIG_DB = "db_config"; static const char* CONFIG_DB_URL = "db_backend_url"; static const char* CONFIG_DB_PATH = "db_path"; static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path"; -static const char* CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml index 0f09d570f165f9e5701252c6245b308d20dc9b40..7faabee2240e353f2cfcfb3efa0fc7419c93edb0 100644 --- a/cpp/unittest/server/appendix/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -18,7 +18,6 @@ metric_config: is_startup: off # if monitoring start: on, off collector: prometheus # metrics collector: prometheus prometheus_config: # following are prometheus configure - collect_type: pull # prometheus collect data method port: 8080 # the port prometheus use to fetch metrics push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address push_gateway_port: 9091 # push method configure: push gateway port diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 462b813f26c0443cc0e01c344974f0f89bb02f9c..59669710ba58188bb434b16a9a67328f5424f19e 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -135,12 +135,6 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { err = config.ValidateConfig(); ASSERT_NE(err, server::SERVER_SUCCESS); - size_t index_building_threshold = (gpu_mem + 1*MB)/MB; - db_config.SetValue(server::CONFIG_DB_INDEX_TRIGGER_SIZE, - std::to_string(index_building_threshold)); - err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); - insert_buffer_size = total_mem/GB + 2; db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); err = config.ValidateConfig();