From a909f95c6cdf173825a09705f749cb83c58e82a1 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Sat, 18 Jan 2020 10:08:46 +0800 Subject: [PATCH] #1078 Move 'insert_buffer_size' to Cache Config section (#1085) Co-authored-by: Jin Hai --- CHANGELOG.md | 1 + core/conf/demo/server_config.yaml | 10 +-- core/conf/server_cpu_config.template | 10 +-- core/conf/server_gpu_config.template | 10 +-- core/src/server/Config.cpp | 93 ++++++++++++++-------------- core/src/server/Config.h | 16 ++--- core/src/utils/ValidationUtil.cpp | 6 ++ core/unittest/db/utils.cpp | 3 +- core/unittest/server/test_config.cpp | 39 ++++++++---- core/unittest/server/utils.cpp | 3 +- core/unittest/wrapper/utils.cpp | 4 +- 11 files changed, 108 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bea2a15a..43d4ad51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ Please mark all change in change log and use the issue from GitHub - \#931 - Remove "collector" from config - \#966 - Update NOTICE.md - \#1002 - Rename minio to s3 in Storage Config section +- \#1078 - Move 'insert_buffer_size' to Cache Config section ## Task diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index 44704226..523760af 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -48,17 +48,12 @@ server_config: # | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | # | 'mysql', replace other texts with real values. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | -# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # preload_table | A comma-separated list of table names that need to be pre- | StringList | | # | loaded when Milvus server starts up. | | | # | '*' means preload all existing tables. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ db_config: backend_url: sqlite://:@:/ - insert_buffer_size: 1 preload_table: #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -113,12 +108,17 @@ metric_config: # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ # cache_insert_data | Whether to load inserted data into cache immediately for | Boolean | false | # | hot query. If want to simultaneously insert and query | | | # | vectors, it's recommended to enable this config. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: cpu_cache_capacity: 4 + insert_buffer_size: 1 cache_insert_data: false #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 1e986651..8b8f4bf7 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -48,17 +48,12 @@ server_config: # | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | # | 'mysql', replace other texts with real values. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | -# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # preload_table | A comma-separated list of table names that need to be pre- | StringList | | # | loaded when Milvus server starts up. | | | # | '*' means preload all existing tables. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ db_config: backend_url: sqlite://:@:/ - insert_buffer_size: 1 preload_table: #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -113,12 +108,17 @@ metric_config: # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ # cache_insert_data | Whether to load inserted data into cache immediately for | Boolean | false | # | hot query. If want to simultaneously insert and query | | | # | vectors, it's recommended to enable this config. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: cpu_cache_capacity: 4 + insert_buffer_size: 1 cache_insert_data: false #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 93d4f112..ae872f58 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -48,17 +48,12 @@ server_config: # | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | # | 'mysql', replace other texts with real values. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | -# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # preload_table | A comma-separated list of table names that need to be pre- | StringList | | # | loaded when Milvus server starts up. | | | # | '*' means preload all existing tables. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ db_config: backend_url: sqlite://:@:/ - insert_buffer_size: 1 preload_table: #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -113,12 +108,17 @@ metric_config: # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ # cache_insert_data | Whether to load inserted data into cache immediately for | Boolean | false | # | hot query. If want to simultaneously insert and query | | | # | vectors, it's recommended to enable this config. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: cpu_cache_capacity: 4 + insert_buffer_size: 1 cache_insert_data: false #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index ca2a9442..26d519e3 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -101,9 +101,6 @@ Config::ValidateConfig() { int64_t db_archive_days_threshold; CONFIG_CHECK(GetDBConfigArchiveDaysThreshold(db_archive_days_threshold)); - int64_t db_insert_buffer_size; - CONFIG_CHECK(GetDBConfigInsertBufferSize(db_insert_buffer_size)); - /* storage config */ std::string storage_primary_path; CONFIG_CHECK(GetStorageConfigPrimaryPath(storage_primary_path)); @@ -147,6 +144,9 @@ Config::ValidateConfig() { float cache_cpu_cache_threshold; CONFIG_CHECK(GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold)); + int64_t cache_insert_buffer_size; + CONFIG_CHECK(GetCacheConfigInsertBufferSize(cache_insert_buffer_size)); + bool cache_insert_data; CONFIG_CHECK(GetCacheConfigCacheInsertData(cache_insert_data)); @@ -203,7 +203,6 @@ Config::ResetDefaultConfig() { CONFIG_CHECK(SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT)); CONFIG_CHECK(SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); CONFIG_CHECK(SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); - CONFIG_CHECK(SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT)); /* storage config */ CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT)); @@ -223,6 +222,7 @@ Config::ResetDefaultConfig() { /* cache config */ CONFIG_CHECK(SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT)); CONFIG_CHECK(SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT)); + CONFIG_CHECK(SetCacheConfigInsertBufferSize(CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT)); CONFIG_CHECK(SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); /* engine config */ @@ -280,6 +280,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key return SetCacheConfigCpuCacheThreshold(value); } else if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA) { return SetCacheConfigCacheInsertData(value); + } else if (child_key == CONFIG_CACHE_INSERT_BUFFER_SIZE) { + return SetCacheConfigInsertBufferSize(value); } } else if (parent_key == CONFIG_ENGINE) { if (child_key == CONFIG_ENGINE_USE_BLAS_THRESHOLD) { @@ -457,31 +459,6 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckDBConfigInsertBufferSize(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid insert buffer size: " + value + - ". Possible reason: db_config.insert_buffer_size is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - int64_t buffer_size = std::stoll(value) * GB; - if (buffer_size <= 0) { - std::string msg = "Invalid insert buffer size: " + value + - ". Possible reason: db_config.insert_buffer_size is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - uint64_t total_mem = 0, free_mem = 0; - CommonUtil::GetSystemMemInfo(total_mem, free_mem); - if (buffer_size >= total_mem) { - std::string msg = "Invalid insert buffer size: " + value + - ". Possible reason: db_config.insert_buffer_size exceeds system memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - return Status::OK(); -} - /* storage config */ Status Config::CheckStorageConfigPrimaryPath(const std::string& value) { @@ -617,13 +594,13 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { } int64_t buffer_value; - CONFIG_CHECK(GetDBConfigInsertBufferSize(buffer_value)); + CONFIG_CHECK(GetCacheConfigInsertBufferSize(buffer_value)); int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: sum of cache_config.cpu_cache_capacity and " - "db_config.insert_buffer_size exceeds system memory."; + "cache_config.insert_buffer_size exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -647,6 +624,31 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { return Status::OK(); } +Status +Config::CheckCacheConfigInsertBufferSize(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid insert buffer size: " + value + + ". Possible reason: cache_config.insert_buffer_size is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int64_t buffer_size = std::stoll(value) * GB; + if (buffer_size <= 0) { + std::string msg = "Invalid insert buffer size: " + value + + ". Possible reason: cache_config.insert_buffer_size is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + uint64_t total_mem = 0, free_mem = 0; + CommonUtil::GetSystemMemInfo(total_mem, free_mem); + if (buffer_size >= total_mem) { + std::string msg = "Invalid insert buffer size: " + value + + ". Possible reason: cache_config.insert_buffer_size exceeds system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -943,14 +945,6 @@ Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { return Status::OK(); } -Status -Config::GetDBConfigInsertBufferSize(int64_t& value) { - std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); - CONFIG_CHECK(CheckDBConfigInsertBufferSize(str)); - value = std::stoll(str); - return Status::OK(); -} - Status Config::GetDBConfigPreloadTable(std::string& value) { value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE); @@ -1050,6 +1044,15 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { return Status::OK(); } +Status +Config::GetCacheConfigInsertBufferSize(int64_t& value) { + std::string str = + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT); + CONFIG_CHECK(CheckCacheConfigInsertBufferSize(str)); + value = std::stoll(str); + return Status::OK(); +} + Status Config::GetCacheConfigCacheInsertData(bool& value) { std::string str = @@ -1238,12 +1241,6 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } -Status -Config::SetDBConfigInsertBufferSize(const std::string& value) { - CONFIG_CHECK(CheckDBConfigInsertBufferSize(value)); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); -} - /* storage config */ Status Config::SetStorageConfigPrimaryPath(const std::string& value) { @@ -1325,6 +1322,12 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); } +Status +Config::SetCacheConfigInsertBufferSize(const std::string& value) { + CONFIG_CHECK(CheckCacheConfigInsertBufferSize(value)); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, value); +} + Status Config::SetCacheConfigCacheInsertData(const std::string& value) { CONFIG_CHECK(CheckCacheConfigCacheInsertData(value)); diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 17286570..e65c501c 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -60,8 +60,6 @@ static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; -static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "1"; static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_DB_PRELOAD_TABLE_DEFAULT = ""; @@ -90,6 +88,8 @@ static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "4"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT = "1"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -188,8 +188,6 @@ class Config { CheckDBConfigArchiveDiskThreshold(const std::string& value); Status CheckDBConfigArchiveDaysThreshold(const std::string& value); - Status - CheckDBConfigInsertBufferSize(const std::string& value); /* storage config */ Status @@ -223,6 +221,8 @@ class Config { Status CheckCacheConfigCpuCacheThreshold(const std::string& value); Status + CheckCacheConfigInsertBufferSize(const std::string& value); + Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -277,8 +277,6 @@ class Config { Status GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int64_t& value); - Status GetDBConfigPreloadTable(std::string& value); /* storage config */ @@ -313,6 +311,8 @@ class Config { Status GetCacheConfigCpuCacheThreshold(float& value); Status + GetCacheConfigInsertBufferSize(int64_t& value); + Status GetCacheConfigCacheInsertData(bool& value); /* engine config */ @@ -362,8 +362,6 @@ class Config { SetDBConfigArchiveDiskThreshold(const std::string& value); Status SetDBConfigArchiveDaysThreshold(const std::string& value); - Status - SetDBConfigInsertBufferSize(const std::string& value); /* storage config */ Status @@ -397,6 +395,8 @@ class Config { Status SetCacheConfigCpuCacheThreshold(const std::string& value); Status + SetCacheConfigInsertBufferSize(const std::string& value); + Status SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 02ba9c70..728c5f08 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -309,6 +309,9 @@ ValidationUtil::ValidateStringIsNumber(const std::string& str) { } try { int32_t value = std::stoi(str); + if (value < 0) { + return Status(SERVER_INVALID_ARGUMENT, "Negative number"); + } } catch (...) { return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } @@ -330,6 +333,9 @@ Status ValidationUtil::ValidateStringIsFloat(const std::string& str) { try { float val = std::stof(str); + if (val < 0.0) { + return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str); + } } catch (...) { return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); } diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index 97c8250c..b8f9a4d2 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -53,8 +53,6 @@ static const char* CONFIG_STR = " \n" " # Replace 'dialect' with 'mysql' or 'sqlite'\n" "\n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - "\n" "storage_config:\n" " primary_path: /tmp/milvus # path used to store data and meta\n" " secondary_path: # path used to store data only, split by semicolon\n" @@ -67,6 +65,7 @@ static const char* CONFIG_STR = "cache_config:\n" " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" " cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" + " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" " cache_insert_data: false # whether load inserted data into cache\n" "\n" "engine_config:\n" diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 8785f4e3..6da7eff2 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -166,11 +166,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetDBConfigArchiveDaysThreshold(int64_val).ok()); ASSERT_TRUE(int64_val == db_archive_days_threshold); - int64_t db_insert_buffer_size = 2; - ASSERT_TRUE(config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)).ok()); - ASSERT_TRUE(config.GetDBConfigInsertBufferSize(int64_val).ok()); - ASSERT_TRUE(int64_val == db_insert_buffer_size); - /* storage config */ std::string storage_primary_path = "/home/zilliz"; ASSERT_TRUE(config.SetStorageConfigPrimaryPath(storage_primary_path).ok()); @@ -239,6 +234,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetCacheConfigCpuCacheThreshold(float_val).ok()); ASSERT_TRUE(float_val == cache_cpu_cache_threshold); + int64_t cache_insert_buffer_size = 2; + ASSERT_TRUE(config.SetCacheConfigInsertBufferSize(std::to_string(cache_insert_buffer_size)).ok()); + ASSERT_TRUE(config.GetCacheConfigInsertBufferSize(int64_val).ok()); + ASSERT_TRUE(int64_val == cache_insert_buffer_size); + bool cache_insert_data = true; ASSERT_TRUE(config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)).ok()); ASSERT_TRUE(config.GetCacheConfigCacheInsertData(bool_val).ok()); @@ -336,9 +336,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { ASSERT_TRUE(s.ok()); /* db config */ - std::string db_insert_buffer_size = "4"; - get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_INSERT_BUFFER_SIZE); - set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_INSERT_BUFFER_SIZE, db_insert_buffer_size); + std::string db_backend_url = "bad_url"; + get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL); + set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url); s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_FALSE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); @@ -378,8 +378,17 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == cache_cpu_cache_threshold); + std::string cache_insert_buffer_size = "1"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + std::string cache_insert_data = "true"; get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); @@ -503,10 +512,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetDBConfigArchiveDaysThreshold("0x10").ok()); - ASSERT_FALSE(config.SetDBConfigInsertBufferSize("a").ok()); - ASSERT_FALSE(config.SetDBConfigInsertBufferSize("0").ok()); - ASSERT_FALSE(config.SetDBConfigInsertBufferSize("2048").ok()); - /* storage config */ ASSERT_FALSE(config.SetStorageConfigPrimaryPath("").ok()); @@ -536,9 +541,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("a").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("0").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("2048").ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("-1").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("a").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("1.0").ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("-0.1").ok()); + + ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("a").ok()); + ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("0").ok()); + ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("2048").ok()); + ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("-1").ok()); ASSERT_FALSE(config.SetCacheConfigCacheInsertData("N").ok()); @@ -547,6 +559,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetEngineConfigOmpThreadNum("a").ok()); ASSERT_FALSE(config.SetEngineConfigOmpThreadNum("10000").ok()); + ASSERT_FALSE(config.SetEngineConfigOmpThreadNum("-10").ok()); #ifdef MILVUS_GPU_VERSION ASSERT_FALSE(config.SetEngineConfigGpuSearchThreshold("-1").ok()); @@ -558,9 +571,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("a").ok()); ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("128").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("-1").ok()); ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("a").ok()); ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("1.0").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("-0.1").ok()); ASSERT_FALSE(config.SetGpuResourceConfigSearchResources("gpu10").ok()); diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index a1d8c4d3..a00fde9a 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -38,8 +38,6 @@ static const char* VALID_CONFIG_STR = "\n" "db_config:\n" " backend_url: sqlite://:@:/ \n" - "\n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" " preload_table: \n" "\n" "storage_config:\n" @@ -54,6 +52,7 @@ static const char* VALID_CONFIG_STR = "cache_config:\n" " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" " cpu_cache_threshold: 0.85 \n" + " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" " cache_insert_data: false # whether to load inserted data into cache\n" "\n" "engine_config:\n" diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index fad0d5d4..1ff942c9 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -34,11 +34,8 @@ static const char* CONFIG_STR = "\n" "db_config:\n" " backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n" - " \n" " # Replace 'dialect' with 'mysql' or 'sqlite'\n" "\n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - "\n" "storage_config:\n" " primary_path: /tmp/milvus # path used to store data and meta\n" " secondary_path: # path used to store data only, split by semicolon\n" @@ -51,6 +48,7 @@ static const char* CONFIG_STR = "cache_config:\n" " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" " cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" + " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" " cache_insert_data: false # whether load inserted data into cache\n" "\n" "engine_config:\n" -- GitLab