From 9d9a453a5b8a2000131be1fc0c22e73108bda069 Mon Sep 17 00:00:00 2001 From: Wang XiangYu Date: Fri, 12 Jun 2020 09:22:26 +0800 Subject: [PATCH] Upgrade config (#2511) * enable cron test on 0.10.0 Signed-off-by: zw * fix merge result Signed-off-by: shengjun.li * import test_during_creating_index_restart Signed-off-by: zw * config cluster Signed-off-by: wxyu * config general Signed-off-by: wxyu * config network Signed-off-by: wxyu * config storage Signed-off-by: wxyu * config wal Signed-off-by: wxyu * config cache Signed-off-by: wxyu * config gpu and metrics Signed-off-by: wxyu * config logs Signed-off-by: wxyu * update server_config.template Signed-off-by: wxyu * update changelog Signed-off-by: wxyu * config with unit Signed-off-by: wxyu * fix clang-format Signed-off-by: wxyu * update unittests Signed-off-by: wxyu * disable restart case Signed-off-by: zw * fix gpu compile failed Signed-off-by: wxyu * Fix lint and comments Signed-off-by: JinHai-CN * fix wal unittest failed Signed-off-by: wxyu * fix some unittests Signed-off-by: wxyu * update config test cases Signed-off-by: zw * Fix config error Signed-off-by: yhz <413554850@qq.com> * remove unused code Signed-off-by: wxyu * Update CHANGELOG Signed-off-by: JinHai-CN * [skip-ci] fix test cases Signed-off-by: zw * fix config storage Signed-off-by: wxyu * make -j3 on ci Signed-off-by: wxyu * update python test Signed-off-by: wxyu * remove unused code Signed-off-by: wxyu * fix ut Signed-off-by: wxyu * fix ut Signed-off-by: wxyu * update status message Signed-off-by: wxyu * update Signed-off-by: wxyu * fix set config bug Signed-off-by: wxyu Co-authored-by: zw Co-authored-by: shengjun.li Co-authored-by: JinHai-CN Co-authored-by: yhz <413554850@qq.com> --- CHANGELOG.md | 3 +- ci/jenkins/Jenkinsfile | 2 +- core/conf/demo/server_config.yaml | 231 ++- core/conf/server_config.template | 240 ++- core/src/cache/CpuCacheMgr.cpp | 7 +- core/src/config/Config.cpp | 1382 ++++++++++------- core/src/config/Config.h | 278 ++-- core/src/config/Utils.cpp | 85 + core/src/config/Utils.h | 23 + .../handler/GpuResourceConfigHandler.cpp | 9 +- core/src/db/meta/MetaFactory.cpp | 1 + core/src/db/wal/WalDefinations.h | 1 + .../metrics/prometheus/PrometheusMetrics.cpp | 2 +- core/src/scheduler/SchedInst.h | 2 +- core/src/scheduler/selector/FaissFlatPass.cpp | 2 +- .../scheduler/selector/FaissIVFFlatPass.cpp | 2 +- .../src/scheduler/selector/FaissIVFPQPass.cpp | 2 +- .../scheduler/selector/FaissIVFSQ8HPass.cpp | 2 +- .../scheduler/selector/FaissIVFSQ8Pass.cpp | 2 +- core/src/server/DBWrapper.cpp | 43 +- core/src/server/Server.cpp | 59 +- .../request/ReLoadSegmentsRequest.cpp | 12 + core/src/server/grpc_impl/GrpcServer.cpp | 4 +- core/src/server/init/StorageChecker.cpp | 37 +- core/src/server/web_impl/WebServer.cpp | 4 +- .../web_impl/handler/WebRequestHandler.cpp | 9 +- core/src/utils/LogUtil.cpp | 1 - core/src/utils/ValidationUtil.cpp | 10 + core/src/utils/ValidationUtil.h | 3 + core/unittest/db/utils.cpp | 55 + core/unittest/server/test_check.cpp | 15 +- core/unittest/server/test_config.cpp | 351 ++--- core/unittest/server/test_rpc.cpp | 7 +- core/unittest/server/test_util.cpp | 12 +- core/unittest/server/test_web.cpp | 152 +- core/unittest/server/utils.cpp | 59 + tests/milvus_python_test/test_compact.py | 1 + tests/milvus_python_test/test_config.py | 1065 +++++-------- tests/milvus_python_test/utils.py | 8 +- 39 files changed, 2032 insertions(+), 2151 deletions(-) create mode 100644 core/src/config/Utils.cpp create mode 100644 core/src/config/Utils.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 56c81154..4a461c46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,12 +24,13 @@ Please mark all change in change log and use the issue from GitHub ## Feature - \#2363 Update branch version +- \#2510 Upgrade Milvus config ## Improvement - \#2381 Upgrade FAISS to 1.6.3 - \#2429 Fix Milvus 0.9.1 performance degrade issue - \#2441 Improve Knowhere code coverage -- \#2466 optimize k-selection implemention of faiss gpu version +- \#2466 Optimize k-selection implemention of faiss gpu version - \#2489 Add exception throw on mysql meta error - \#2495 Add creating lock file failure reason. - \#2516 Improve unit test coverage diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 943a8634..b5b83520 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -30,7 +30,7 @@ pipeline { LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase() SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" PIPELINE_NAME = "milvus-ci" - HELM_BRANCH = "master" + HELM_BRANCH = "0.10.0" } stages { diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index ced4fc60..8a537037 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -9,91 +9,99 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License. -version: 0.4 +version: 0.5 #----------------------+------------------------------------------------------------+------------+-----------------+ -# Server Config | Description | Type | Default | +# Cluster Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +# enable | If runinng with Mishards, set true, otherwise false. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +# role | Milvus deployment role: rw / ro | role | rw | #----------------------+------------------------------------------------------------+------------+-----------------+ -# deploy_mode | Milvus deployment type: | DeployMode | single | -# | single, cluster_readonly, cluster_writable | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# web_enable | Enable web server or not. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# web_port | Port that Milvus web server monitors. | Integer | 19121 | -# | Port range (1024, 65535) | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -server_config: - address: 0.0.0.0 - port: 19530 - deploy_mode: single - time_zone: UTC+8 - web_enable: true - web_port: 19121 +cluster: + enable: false + role: rw #----------------------+------------------------------------------------------------+------------+-----------------+ -# DataBase Config | Description | Type | Default | +# General Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | +# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# meta_uri | URI for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | # | Milvus) or MySQL (for distributed cluster Milvus). | | | # | Format: dialect://username:password@host:port/database | | | # | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | # | 'mysql', replace other texts with real values. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# preload_collection | A comma-separated list of collection names that need to | StringList | | -# | be pre-loaded when Milvus server starts up. | | | -# | '*' means preload all existing tables (single-quote or | | | -# | double-quote required). | | | +general: + timezone: UTC+8 + meta_uri: sqlite://:@:/ + #----------------------+------------------------------------------------------------+------------+-----------------+ -# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | -# | flushes data to disk. | | | -# | 0 means disable the regular flush. | | | +# Network Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -db_config: - backend_url: sqlite://:@:/ - preload_collection: - auto_flush_interval: 1 +# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# http.enable | Enable web server or not. | Boolean | true | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# http.port | Port that Milvus web server monitors. | Integer | 19121 | +# | Port range (1024, 65535) | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +network: + bind.address: 0.0.0.0 + bind.port: 19530 + http.enable: true + http.port: 19121 #----------------------+------------------------------------------------------------+------------+-----------------+ # Storage Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | -# | index data. | | | +# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus | #----------------------+------------------------------------------------------------+------------+-----------------+ -# secondary_path | A semicolon-separated list of secondary directories used | Path | | -# | to save vector data and index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# file_cleanup_timeout | The time gap between marking a file as 'deleted' and | Integer | 10 (s) | -# | physically deleting this file from disk, range [0, 3600] | | | +# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | +# | flushes data to disk. | | | +# | 0 means disable the regular flush. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -storage_config: - primary_path: /var/lib/milvus - secondary_path: - file_cleanup_timeout: 10 +storage: + path: /var/lib/milvus + auto_flush_interval: 1 #----------------------+------------------------------------------------------------+------------+-----------------+ -# Metric Config | Description | Type | Default | +# WAL Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# enable_monitor | Enable monitoring function or not. | Boolean | false | +# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | +# | If WAL is enabled, Milvus writes all data changes to log | | | +# | files in advance before implementing data changes. WAL | | | +# | ensures the atomicity and durability for Milvus operations.| | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | Pushgateway address | IP | 127.0.0.1 + +# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false | +# | recovery. If true, when Milvus restarts for recovery and | | | +# | there are errors in WAL log files, log files with errors | | | +# | are ignored. If false, Milvus does not restart when there | | | +# | are errors in WAL log files. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | +# | buffer_size must be in range [64, 4096] (MB). | | | +# | If the value you specified is out of range, Milvus | | | +# | automatically uses the boundary value closest to the | | | +# | specified value. It is recommended you set buffer_size to | | | +# | a value greater than the inserted data size of a single | | | +# | insert operation for better performance. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -metric_config: - enable_monitor: false - address: 127.0.0.1 - port: 9091 +# path | Location of WAL log files. | String | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +wal: + enable: true + recovery_error_ignore: false + buffer_size: 256 + path: /var/lib/milvus/wal #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | +# cache_size | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -101,23 +109,22 @@ metric_config: # | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | # | must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_insert_data | Whether to load data to cache for hot query | Boolean | false | +# preload_collection | A comma-separated list of collection names that need to | StringList | | +# | be pre-loaded when Milvus server starts up. | | | +# | '*' means preload all existing tables (single-quote or | | | +# | double-quote required). | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -cache_config: - cpu_cache_capacity: 4 +cache: + cache_size: 4 insert_buffer_size: 1 - cache_insert_data: false + preload_collection: #----------------------+------------------------------------------------------------+------------+-----------------+ -# Engine Config | Description | Type | Default | +# GPU Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable GPU resources or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | -# | compared with 'nq' to decide if OpenBLAS should be used. | | | -# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response time will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | -# | used, search speed will be faster but search response time | | | -# | will fluctuate. | | | +# cache_size | The size of GPU memory per card used for cache. | Integer | 1 (GB) | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | @@ -127,85 +134,29 @@ cache_config: # | if nq < gpu_search_threshold, the search computation will | | | # | be executed on both CPUs and GPUs. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -engine_config: - use_blas_threshold: 1100 - gpu_search_threshold: 1000 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# GPU Resource Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Enable GPU resources or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) | -#----------------------+------------------------------------------------------------+------------+-----------------+ # search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | # | Must be in format gpux. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | # | Must be in format gpux. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -gpu_resource_config: +gpu: enable: false - cache_capacity: 1 - search_resources: + cache_size: 1 + gpu_search_threshold: 1000 + search_devices: - gpu0 - build_index_resources: + build_index_devices: - gpu0 #----------------------+------------------------------------------------------------+------------+-----------------+ -# Tracing Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# json_config_path | Absolute path for tracing config file. | Path | | -# | Leave it empty, a no-op tracer will be created. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -tracing_config: - json_config_path: - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# WAL Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | -# | If WAL is enabled, Milvus writes all data changes to log | | | -# | files in advance before implementing data changes. WAL | | | -# | ensures the atomicity and durability for Milvus operations.| | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true | -# | recovery. If true, when Milvus restarts for recovery and | | | -# | there are errors in WAL log files, log files with errors | | | -# | are ignored. If false, Milvus does not restart when there | | | -# | are errors in WAL log files. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | -# | buffer_size must be in range [64, 4096] (MB). | | | -# | If the value you specified is out of range, Milvus | | | -# | automatically uses the boundary value closest to the | | | -# | specified value. It is recommended you set buffer_size to | | | -# | a value greater than the inserted data size of a single | | | -# | insert operation for better performance. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# wal_path | Location of WAL log files. | String | | +# Logs Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -wal_config: - enable: true - recovery_error_ignore: true - buffer_size: 256 - wal_path: /var/lib/milvus/wal - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Logs | Description | Type | Default | +# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug | +# | error, fatal | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true | #----------------------+------------------------------------------------------------+------------+-----------------+ -# debug.enable | Whether to enable debug level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# info.enable | Whether to enable info level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# warning.enable | Whether to enable warning level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# error.enable | Whether to enable error level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# fatal.enable | Whether to enable fatal level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ # path | Absolute path to the folder holding the log files. | String | | #----------------------+------------------------------------------------------------+------------+-----------------+ # max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) | @@ -214,13 +165,23 @@ wal_config: # | logging level, num range [0, 1024], 0 means unlimited. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ logs: + level: debug trace.enable: true - debug.enable: true - info.enable: true - warning.enable: true - error.enable: true - fatal.enable: true path: /var/lib/milvus/logs max_log_file_size: 1024 log_rotate_num: 0 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Metric Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable monitoring function or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +metric: + enable: false + address: 127.0.0.1 + port: 9091 + diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 215553eb..e8b383ed 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -9,115 +9,122 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License. -version: 0.4 +version: 0.5 #----------------------+------------------------------------------------------------+------------+-----------------+ -# Server Config | Description | Type | Default | +# Cluster Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +# enable | If runinng with Mishards, set true, otherwise false. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +# role | Milvus deployment role: rw / ro | role | rw | #----------------------+------------------------------------------------------------+------------+-----------------+ -# deploy_mode | Milvus deployment type: | DeployMode | single | -# | single, cluster_readonly, cluster_writable | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# web_enable | Enable web server or not. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# web_port | Port that Milvus web server monitors. | Integer | 19121 | -# | Port range (1024, 65535) | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -server_config: - address: 0.0.0.0 - port: 19530 - deploy_mode: single - time_zone: UTC+8 - web_enable: true - web_port: 19121 +cluster: + enable: false + role: rw #----------------------+------------------------------------------------------------+------------+-----------------+ -# DataBase Config | Description | Type | Default | +# General Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | +# meta_uri | URI for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | # | Milvus) or MySQL (for distributed cluster Milvus). | | | # | Format: dialect://username:password@host:port/database | | | # | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | # | 'mysql', replace other texts with real values. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# preload_collection | A comma-separated list of collection names that need to | StringList | | -# | be pre-loaded when Milvus server starts up. | | | -# | '*' means preload all existing tables (single-quote or | | | -# | double-quote required). | | | +general: + timezone: UTC+8 + meta_uri: sqlite://:@:/ + #----------------------+------------------------------------------------------------+------------+-----------------+ -# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | -# | flushes data to disk. | | | -# | 0 means disable the regular flush. | | | +# Network Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -db_config: - backend_url: sqlite://:@:/ - preload_collection: - auto_flush_interval: 1 +# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# http.enable | Enable web server or not. | Boolean | true | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# http.port | Port that Milvus web server monitors. | Integer | 19121 | +# | Port range (1024, 65535) | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +network: + bind.address: 0.0.0.0 + bind.port: 19530 + http.enable: true + http.port: 19121 #----------------------+------------------------------------------------------------+------------+-----------------+ # Storage Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | -# | index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# secondary_path | A semicolon-separated list of secondary directories used | Path | | -# | to save vector data and index data. | | | +# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus | #----------------------+------------------------------------------------------------+------------+-----------------+ -# file_cleanup_timeout | The time gap between marking a file as 'deleted' and | Integer | 10 (s) | -# | physically deleting this file from disk, range [0, 3600] | | | +# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | +# | flushes data to disk. | | | +# | 0 means disable the regular flush. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -storage_config: - primary_path: @MILVUS_DB_PATH@ - secondary_path: - file_cleanup_timeout: 10 +storage: + path: @MILVUS_DB_PATH@ + auto_flush_interval: 1 #----------------------+------------------------------------------------------------+------------+-----------------+ -# Metric Config | Description | Type | Default | +# WAL Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# enable_monitor | Enable monitoring function or not. | Boolean | false | +# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | +# | If WAL is enabled, Milvus writes all data changes to log | | | +# | files in advance before implementing data changes. WAL | | | +# | ensures the atomicity and durability for Milvus operations.| | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | Pushgateway address | IP | 127.0.0.1 + +# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false | +# | recovery. If true, when Milvus restarts for recovery and | | | +# | there are errors in WAL log files, log files with errors | | | +# | are ignored. If false, Milvus does not restart when there | | | +# | are errors in WAL log files. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB | +# | buffer_size must be in range [64MB, 4096MB]. | | | +# | If the value you specified is out of range, Milvus | | | +# | automatically uses the boundary value closest to the | | | +# | specified value. It is recommended you set buffer_size to | | | +# | a value greater than the inserted data size of a single | | | +# | insert operation for better performance. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -metric_config: - enable_monitor: false - address: 127.0.0.1 - port: 9091 +# path | Location of WAL log files. | String | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +wal: + enable: true + recovery_error_ignore: false + buffer_size: 256MB + path: @MILVUS_DB_PATH@/wal #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | +# cache_size | The size of CPU memory used for caching data for faster | String | 4GB | # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# insert_buffer_size | Buffer size used for data insertion. | String | 1GB | # | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | # | must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_insert_data | Whether to load data to cache for hot query | Boolean | false | +# preload_collection | A comma-separated list of collection names that need to | StringList | | +# | be pre-loaded when Milvus server starts up. | | | +# | '*' means preload all existing tables (single-quote or | | | +# | double-quote required). | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -cache_config: - cpu_cache_capacity: 4 - insert_buffer_size: 1 - cache_insert_data: false +cache: + cache_size: 4GB + insert_buffer_size: 1GB + preload_collection: #----------------------+------------------------------------------------------------+------------+-----------------+ -# Engine Config | Description | Type | Default | +# GPU Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | -# | compared with 'nq' to decide if OpenBLAS should be used. | | | -# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response time will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | -# | used, search speed will be faster but search response time | | | -# | will fluctuate. | | | +# enable | Enable GPU resources or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_size | The size of GPU memory per card used for cache. | String | 1GB | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | @@ -127,100 +134,55 @@ cache_config: # | if nq < gpu_search_threshold, the search computation will | | | # | be executed on both CPUs and GPUs. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -engine_config: - use_blas_threshold: 1100 - gpu_search_threshold: 1000 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# GPU Resource Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Enable GPU resources or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) | -#----------------------+------------------------------------------------------------+------------+-----------------+ # search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | # | Must be in format gpux. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | # | Must be in format gpux. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -gpu_resource_config: +gpu: enable: @GPU_ENABLE@ - cache_capacity: 1 - search_resources: + cache_size: 1GB + gpu_search_threshold: 1000 + search_devices: - gpu0 - build_index_resources: + build_index_devices: - gpu0 #----------------------+------------------------------------------------------------+------------+-----------------+ -# Tracing Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# json_config_path | Absolute path for tracing config file. | Path | | -# | Leave it empty, a no-op tracer will be created. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -tracing_config: - json_config_path: - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# WAL Config | Description | Type | Default | +# Logs Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | -# | If WAL is enabled, Milvus writes all data changes to log | | | -# | files in advance before implementing data changes. WAL | | | -# | ensures the atomicity and durability for Milvus operations.| | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true | -# | recovery. If true, when Milvus restarts for recovery and | | | -# | there are errors in WAL log files, log files with errors | | | -# | are ignored. If false, Milvus does not restart when there | | | -# | are errors in WAL log files. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | -# | buffer_size must be in range [64, 4096] (MB). | | | -# | If the value you specified is out of range, Milvus | | | -# | automatically uses the boundary value closest to the | | | -# | specified value. It is recommended you set buffer_size to | | | -# | a value greater than the inserted data size of a single | | | -# | insert operation for better performance. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# wal_path | Location of WAL log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -wal_config: - enable: true - recovery_error_ignore: true - buffer_size: 256 - wal_path: @MILVUS_DB_PATH@/wal - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Logs | Description | Type | Default | +# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug | +# | error, fatal | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true | #----------------------+------------------------------------------------------------+------------+-----------------+ -# debug.enable | Whether to enable debug level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# info.enable | Whether to enable info level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# warning.enable | Whether to enable warning level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# error.enable | Whether to enable error level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# fatal.enable | Whether to enable fatal level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ # path | Absolute path to the folder holding the log files. | String | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) | +# max_log_file_size | The maximum size of each log file, size range | String | 1024MB | +# | [512MB, 4096MB]. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | # | logging level, num range [0, 1024], 0 means unlimited. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ logs: + level: debug trace.enable: true - debug.enable: true - info.enable: true - warning.enable: true - error.enable: true - fatal.enable: true path: @MILVUS_DB_PATH@/logs - max_log_file_size: 1024 + max_log_file_size: 1024MB log_rotate_num: 0 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Metric Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable monitoring function or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +metric: + enable: false + address: 127.0.0.1 + port: 9091 + diff --git a/core/src/cache/CpuCacheMgr.cpp b/core/src/cache/CpuCacheMgr.cpp index ff864931..7059bc77 100644 --- a/core/src/cache/CpuCacheMgr.cpp +++ b/core/src/cache/CpuCacheMgr.cpp @@ -22,8 +22,9 @@ namespace milvus { namespace cache { namespace { -constexpr int64_t unit = 1024 * 1024 * 1024; -} +// constexpr int64_t unit = 1024 * 1024 * 1024; +constexpr int64_t unit = 1; +} // namespace CpuCacheMgr::CpuCacheMgr() { // All config values have been checked in Config::ValidateConfig() @@ -32,6 +33,8 @@ CpuCacheMgr::CpuCacheMgr() { int64_t cpu_cache_cap; config.GetCacheConfigCpuCacheCapacity(cpu_cache_cap); int64_t cap = cpu_cache_cap * unit; + LOG_SERVER_DEBUG_ << "cpu cache.size: " << cap; + LOG_SERVER_INFO_ << "cpu cache.size: " << cap; cache_ = std::make_shared>(cap, 1UL << 32, "[CACHE CPU]"); float cpu_cache_threshold; diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index 1aa3488a..2197bbd7 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -26,6 +26,7 @@ #include #include "config/Config.h" +#include "config/Utils.h" #include "config/YamlConfigMgr.h" #include "server/DBWrapper.h" #include "thirdparty/nlohmann/json.hpp" @@ -40,71 +41,65 @@ namespace server { const char* CONFIG_NODE_DELIMITER = "."; const char* CONFIG_VERSION = "version"; -/* server config */ -const char* CONFIG_SERVER = "server_config"; -const char* CONFIG_SERVER_ADDRESS = "address"; -const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; -const char* CONFIG_SERVER_PORT = "port"; -const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; -const char* CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; -const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; -const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; -const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; -const char* CONFIG_SERVER_WEB_ENABLE = "web_enable"; -const char* CONFIG_SERVER_WEB_ENABLE_DEFAULT = "true"; -const char* CONFIG_SERVER_WEB_PORT = "web_port"; -const char* CONFIG_SERVER_WEB_PORT_DEFAULT = "19121"; +/* cluster config */ +const char* CONFIG_CLUSTER = "cluster"; +const char* CONFIG_CLUSTER_ENABLE = "enable"; +const char* CONFIG_CLUSTER_ENABLE_DEFAULT = "true"; +const char* CONFIG_CLUSTER_ROLE = "role"; +const char* CONFIG_CLUSTER_ROLE_DEFAULT = "rw"; + +/* general config */ +const char* CONFIG_GENERAL = "general"; +const char* CONFIG_GENERAL_TIMEZONE = "timezone"; +const char* CONFIG_GENERAL_TIMEZONE_DEFAULT = "UTC+8"; +const char* CONFIG_GENERAL_METAURI = "meta_uri"; +const char* CONFIG_GENERAL_METAURI_DEFAULT = "sqlite://:@:/"; + +/* network config */ +const char* CONFIG_NETWORK = "network"; +const char* CONFIG_NETWORK_BIND_ADDRESS = "bind.address"; +const char* CONFIG_NETWORK_BIND_ADDRESS_DEFAULT = "127.0.0.1"; +const char* CONFIG_NETWORK_BIND_PORT = "bind.port"; +const char* CONFIG_NETWORK_BIND_PORT_DEFAULT = "19530"; +const char* CONFIG_NETWORK_HTTP_ENABLE = "http.enable"; +const char* CONFIG_NETWORK_HTTP_ENABLE_DEFAULT = "true"; +const char* CONFIG_NETWORK_HTTP_PORT = "http.port"; +const char* CONFIG_NETWORK_HTTP_PORT_DEFAULT = "19121"; /* db config */ const char* CONFIG_DB = "db_config"; -const char* CONFIG_DB_BACKEND_URL = "backend_url"; -const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -const char* CONFIG_DB_PRELOAD_COLLECTION = "preload_collection"; -const char* CONFIG_DB_PRELOAD_COLLECTION_DEFAULT = ""; -const char* CONFIG_DB_AUTO_FLUSH_INTERVAL = "auto_flush_interval"; -const char* CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT = "1"; /* storage config */ -const char* CONFIG_STORAGE = "storage_config"; -const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path"; -const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; -const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path"; -const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = ""; +const char* CONFIG_STORAGE = "storage"; +const char* CONFIG_STORAGE_PATH = "path"; +const char* CONFIG_STORAGE_PATH_DEFAULT = "/tmp/milvus"; +const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL = "auto_flush_interval"; +const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL_DEFAULT = "1"; const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT = "file_cleanup_timeout"; const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10"; const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN = 0; const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX = 3600; -// const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable"; -// const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false"; -// const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address"; -// const char* CONFIG_STORAGE_S3_ADDRESS_DEFAULT = "127.0.0.1"; -// const char* CONFIG_STORAGE_S3_PORT = "s3_port"; -// const char* CONFIG_STORAGE_S3_PORT_DEFAULT = "9000"; -// const char* CONFIG_STORAGE_S3_ACCESS_KEY = "s3_access_key"; -// const char* CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT = "minioadmin"; -// const char* CONFIG_STORAGE_S3_SECRET_KEY = "s3_secret_key"; -// const char* CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT = "minioadmin"; -// const char* CONFIG_STORAGE_S3_BUCKET = "s3_bucket"; -// const char* CONFIG_STORAGE_S3_BUCKET_DEFAULT = "milvus-bucket"; /* cache config */ -const char* CONFIG_CACHE = "cache_config"; -const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; -const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "4"; +const char* CONFIG_CACHE = "cache"; +const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cache_size"; +const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "4294967296"; const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.7"; const char* CONFIG_CACHE_INSERT_BUFFER_SIZE = "insert_buffer_size"; -const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT = "1"; +const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT = "1073741824"; // 1024 * 1024 * 1024 const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; +const char* CONFIG_CACHE_PRELOAD_COLLECTION = "preload_collection"; +const char* CONFIG_CACHE_PRELOAD_COLLECTION_DEFAULT = ""; /* metric config */ -const char* CONFIG_METRIC = "metric_config"; -const char* CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +const char* CONFIG_METRIC = "metric"; +const char* CONFIG_METRIC_ENABLE_MONITOR = "enable"; const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; const char* CONFIG_METRIC_ADDRESS = "address"; const char* CONFIG_METRIC_ADDRESS_DEFAULT = "127.0.0.1"; @@ -119,25 +114,25 @@ const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; const char* CONFIG_ENGINE_SIMD_TYPE = "simd_type"; const char* CONFIG_ENGINE_SIMD_TYPE_DEFAULT = "auto"; -const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; -const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; /* gpu resource config */ -const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; +const char* CONFIG_GPU_RESOURCE = "gpu"; const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif -const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; -const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "1"; +const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_size"; +const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "1073741824"; // 1024 * 1024 * 1024 const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.7"; +const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; +const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; const char* CONFIG_GPU_RESOURCE_DELIMITER = ","; -const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_resources"; +const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_devices"; const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT = "gpu0"; -const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_resources"; +const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_devices"; const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT = "gpu0"; /* tracing config */ @@ -145,38 +140,30 @@ const char* CONFIG_TRACING = "tracing_config"; const char* CONFIG_TRACING_JSON_CONFIG_PATH = "json_config_path"; /* wal config */ -const char* CONFIG_WAL = "wal_config"; +const char* CONFIG_WAL = "wal"; const char* CONFIG_WAL_ENABLE = "enable"; const char* CONFIG_WAL_ENABLE_DEFAULT = "true"; const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE = "recovery_error_ignore"; const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT = "true"; const char* CONFIG_WAL_BUFFER_SIZE = "buffer_size"; -const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT = "256"; -const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 64; -const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4096; -const char* CONFIG_WAL_WAL_PATH = "wal_path"; +const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT = "268435456"; +const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 67108864; +const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4294967296; +const char* CONFIG_WAL_WAL_PATH = "path"; const char* CONFIG_WAL_WAL_PATH_DEFAULT = "/tmp/milvus/wal"; /* logs config */ const char* CONFIG_LOGS = "logs"; +const char* CONFIG_LOGS_LEVEL = "level"; +const char* CONFIG_LOGS_LEVEL_DEFAULT = "debug"; const char* CONFIG_LOGS_TRACE_ENABLE = "trace.enable"; const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT = "true"; -const char* CONFIG_LOGS_DEBUG_ENABLE = "debug.enable"; -const char* CONFIG_LOGS_DEBUG_ENABLE_DEFAULT = "true"; -const char* CONFIG_LOGS_INFO_ENABLE = "info.enable"; -const char* CONFIG_LOGS_INFO_ENABLE_DEFAULT = "true"; -const char* CONFIG_LOGS_WARNING_ENABLE = "warning.enable"; -const char* CONFIG_LOGS_WARNING_ENABLE_DEFAULT = "true"; -const char* CONFIG_LOGS_ERROR_ENABLE = "error.enable"; -const char* CONFIG_LOGS_ERROR_ENABLE_DEFAULT = "true"; -const char* CONFIG_LOGS_FATAL_ENABLE = "fatal.enable"; -const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT = "true"; const char* CONFIG_LOGS_PATH = "path"; const char* CONFIG_LOGS_PATH_DEFAULT = "/tmp/milvus/logs"; const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE = "max_log_file_size"; -const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "1024"; -const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 512; -const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4096; +const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "1073741824"; +const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 536870912; +const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4294967296; const char* CONFIG_LOGS_LOG_ROTATE_NUM = "log_rotate_num"; const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT = "0"; const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0; @@ -186,8 +173,13 @@ constexpr int64_t GB = 1UL << 30; constexpr int32_t PORT_NUMBER_MIN = 1024; constexpr int32_t PORT_NUMBER_MAX = 65535; -static const std::unordered_map milvus_config_version_map( - {{"0.6.0", "0.1"}, {"0.7.0", "0.2"}, {"0.7.1", "0.2"}, {"0.8.0", "0.3"}, {"0.9.0", "0.4"}}); +static const std::unordered_map milvus_config_version_map({{"0.6.0", "0.1"}, + {"0.7.0", "0.2"}, + {"0.7.1", "0.2"}, + {"0.8.0", "0.3"}, + {"0.9.0", "0.4"}, + {"0.9.1", "0.4"}, + {"0.10.0", "0.5"}}); ///////////////////////////////////////////////////////////// Config::Config() { @@ -208,15 +200,16 @@ Config::Config() { config_callback_[node_blas_threshold] = empty_map; // gpu resources config - std::string node_gpu_search_threshold = std::string(CONFIG_ENGINE) + "." + CONFIG_ENGINE_GPU_SEARCH_THRESHOLD; - config_callback_[node_gpu_search_threshold] = empty_map; - std::string node_gpu_enable = std::string(CONFIG_GPU_RESOURCE) + "." + CONFIG_GPU_RESOURCE_ENABLE; config_callback_[node_gpu_enable] = empty_map; std::string node_gpu_cache_capacity = std::string(CONFIG_GPU_RESOURCE) + "." + CONFIG_GPU_RESOURCE_CACHE_CAPACITY; config_callback_[node_gpu_cache_capacity] = empty_map; + std::string node_gpu_search_threshold = + std::string(CONFIG_GPU_RESOURCE) + "." + CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD; + config_callback_[node_gpu_search_threshold] = empty_map; + std::string node_gpu_search_res = std::string(CONFIG_GPU_RESOURCE) + "." + CONFIG_GPU_RESOURCE_SEARCH_RESOURCES; config_callback_[node_gpu_search_res] = empty_map; @@ -256,31 +249,48 @@ Config::ValidateConfig() { std::string config_version; STATUS_CHECK(GetConfigVersion(config_version)); - /* server config */ - std::string server_addr; - STATUS_CHECK(GetServerConfigAddress(server_addr)); + /* cluster config */ + bool cluster_enable; + STATUS_CHECK(GetClusterConfigEnable(cluster_enable)); - std::string server_port; - STATUS_CHECK(GetServerConfigPort(server_port)); + std::string cluster_role; + STATUS_CHECK(GetClusterConfigRole(cluster_role)); - std::string server_mode; - STATUS_CHECK(GetServerConfigDeployMode(server_mode)); + /* general config */ + std::string general_timezone; + STATUS_CHECK(GetGeneralConfigTimezone(general_timezone)); - std::string server_time_zone; - STATUS_CHECK(GetServerConfigTimeZone(server_time_zone)); + std::string general_metauri; + STATUS_CHECK(GetGeneralConfigMetaURI(general_metauri)); - bool server_web_enable; - STATUS_CHECK(GetServerConfigWebEnable(server_web_enable)); + /* network config */ + std::string bind_address; + STATUS_CHECK(GetNetworkConfigBindAddress(bind_address)); - std::string server_web_port; - STATUS_CHECK(GetServerConfigWebPort(server_web_port)); + std::string bind_port; + STATUS_CHECK(GetNetworkConfigBindPort(bind_port)); - /* db config */ - std::string db_backend_url; - STATUS_CHECK(GetDBConfigBackendUrl(db_backend_url)); + bool http_enable = false; + STATUS_CHECK(GetNetworkConfigHTTPEnable(http_enable)); + + std::string http_port; + STATUS_CHECK(GetNetworkConfigHTTPPort(http_port)); - std::string db_preload_collection; - STATUS_CHECK(GetDBConfigPreloadCollection(db_preload_collection)); + // std::string server_mode; + // STATUS_CHECK(GetServerConfigDeployMode(server_mode)); + + // std::string server_time_zone; + // STATUS_CHECK(GetServerConfigTimeZone(server_time_zone)); + + // bool server_web_enable; + // STATUS_CHECK(GetServerConfigWebEnable(server_web_enable)); + + // std::string server_web_port; + // STATUS_CHECK(GetServerConfigWebPort(server_web_port)); + + /* db config */ + // std::string db_backend_url; + // STATUS_CHECK(GetDBConfigBackendUrl(db_backend_url)); int64_t db_archive_disk_threshold; STATUS_CHECK(GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold)); @@ -288,15 +298,12 @@ Config::ValidateConfig() { int64_t db_archive_days_threshold; STATUS_CHECK(GetDBConfigArchiveDaysThreshold(db_archive_days_threshold)); - int64_t auto_flush_interval; - STATUS_CHECK(GetDBConfigAutoFlushInterval(auto_flush_interval)); - /* storage config */ - std::string storage_primary_path; - STATUS_CHECK(GetStorageConfigPrimaryPath(storage_primary_path)); + std::string storage_path; + STATUS_CHECK(GetStorageConfigPath(storage_path)); - std::string storage_secondary_path; - STATUS_CHECK(GetStorageConfigSecondaryPath(storage_secondary_path)); + int64_t auto_flush_interval; + STATUS_CHECK(GetStorageConfigAutoFlushInterval(auto_flush_interval)); // bool storage_s3_enable; // STATUS_CHECK(GetStorageConfigS3Enable(storage_s3_enable)); @@ -340,6 +347,9 @@ Config::ValidateConfig() { bool cache_insert_data; STATUS_CHECK(GetCacheConfigCacheInsertData(cache_insert_data)); + std::string cache_preload_collection; + STATUS_CHECK(GetCacheConfigPreloadCollection(cache_preload_collection)); + /* engine config */ int64_t engine_use_blas_threshold; STATUS_CHECK(GetEngineConfigUseBlasThreshold(engine_use_blas_threshold)); @@ -350,11 +360,6 @@ Config::ValidateConfig() { std::string engine_simd_type; STATUS_CHECK(GetEngineConfigSimdType(engine_simd_type)); -#ifdef MILVUS_GPU_VERSION - int64_t engine_gpu_search_threshold; - STATUS_CHECK(GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold)); -#endif - /* gpu resource config */ #ifdef MILVUS_GPU_VERSION bool gpu_resource_enable; @@ -368,6 +373,9 @@ Config::ValidateConfig() { float resource_cache_threshold; STATUS_CHECK(GetGpuResourceConfigCacheThreshold(resource_cache_threshold)); + int64_t engine_gpu_search_threshold; + STATUS_CHECK(GetGpuResourceConfigGpuSearchThreshold(engine_gpu_search_threshold)); + std::vector search_resources; STATUS_CHECK(GetGpuResourceConfigSearchResources(search_resources)); @@ -394,24 +402,12 @@ Config::ValidateConfig() { STATUS_CHECK(GetWalConfigWalPath(wal_path)); /* logs config */ + std::string logs_level; + STATUS_CHECK(GetLogsLevel(logs_level)); + bool trace_enable; STATUS_CHECK(GetLogsTraceEnable(trace_enable)); - bool debug_enable; - STATUS_CHECK(GetLogsDebugEnable(debug_enable)); - - bool info_enable; - STATUS_CHECK(GetLogsInfoEnable(info_enable)); - - bool warning_enable; - STATUS_CHECK(GetLogsWarningEnable(warning_enable)); - - bool error_enable; - STATUS_CHECK(GetLogsErrorEnable(error_enable)); - - bool fatal_enable; - STATUS_CHECK(GetLogsFatalEnable(fatal_enable)); - std::string logs_path; STATUS_CHECK(GetLogsPath(logs_path)); @@ -426,24 +422,36 @@ Config::ValidateConfig() { Status Config::ResetDefaultConfig() { + /* cluster config */ + STATUS_CHECK(SetClusterConfigEnable(CONFIG_CLUSTER_ENABLE_DEFAULT)); + STATUS_CHECK(SetClusterConfigRole(CONFIG_CLUSTER_ROLE_DEFAULT)); + + /* general config */ + STATUS_CHECK(SetGeneralConfigTimezone(CONFIG_GENERAL_TIMEZONE_DEFAULT)); + STATUS_CHECK(SetGeneralConfigMetaURI(CONFIG_GENERAL_METAURI_DEFAULT)); + + /* network config */ + STATUS_CHECK(SetNetworkConfigBindAddress(CONFIG_NETWORK_BIND_ADDRESS_DEFAULT)); + STATUS_CHECK(SetNetworkConfigBindPort(CONFIG_NETWORK_BIND_PORT_DEFAULT)); + STATUS_CHECK(SetNetworkConfigHTTPEnable(CONFIG_NETWORK_HTTP_ENABLE_DEFAULT)); + STATUS_CHECK(SetNetworkConfigHTTPPort(CONFIG_NETWORK_HTTP_PORT_DEFAULT)); + /* server config */ - STATUS_CHECK(SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT)); - STATUS_CHECK(SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT)); - STATUS_CHECK(SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT)); - STATUS_CHECK(SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT)); - STATUS_CHECK(SetServerConfigWebEnable(CONFIG_SERVER_WEB_ENABLE_DEFAULT)); - STATUS_CHECK(SetServerConfigWebPort(CONFIG_SERVER_WEB_PORT_DEFAULT)); + // STATUS_CHECK(SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT)); + // STATUS_CHECK(SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT)); + // STATUS_CHECK(SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT)); + // STATUS_CHECK(SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT)); + // STATUS_CHECK(SetServerConfigWebEnable(CONFIG_SERVER_WEB_ENABLE_DEFAULT)); + // STATUS_CHECK(SetServerConfigWebPort(CONFIG_SERVER_WEB_PORT_DEFAULT)); /* db config */ - STATUS_CHECK(SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT)); - STATUS_CHECK(SetDBConfigPreloadCollection(CONFIG_DB_PRELOAD_COLLECTION_DEFAULT)); + // STATUS_CHECK(SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT)); STATUS_CHECK(SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); STATUS_CHECK(SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); - STATUS_CHECK(SetDBConfigAutoFlushInterval(CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT)); /* storage config */ - STATUS_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT)); - STATUS_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT)); + STATUS_CHECK(SetStorageConfigPath(CONFIG_STORAGE_PATH_DEFAULT)); + STATUS_CHECK(SetStorageConfigAutoFlushInterval(CONFIG_STORAGE_AUTO_FLUSH_INTERVAL_DEFAULT)); STATUS_CHECK(SetStorageConfigFileCleanupTimeout(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT)); // STATUS_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT)); // STATUS_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT)); @@ -462,20 +470,19 @@ Config::ResetDefaultConfig() { STATUS_CHECK(SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT)); STATUS_CHECK(SetCacheConfigInsertBufferSize(CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT)); STATUS_CHECK(SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); + STATUS_CHECK(SetCacheConfigPreloadCollection(CONFIG_CACHE_PRELOAD_COLLECTION_DEFAULT)); /* engine config */ STATUS_CHECK(SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT)); STATUS_CHECK(SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); STATUS_CHECK(SetEngineConfigSimdType(CONFIG_ENGINE_SIMD_TYPE_DEFAULT)); -#ifdef MILVUS_GPU_VERSION - STATUS_CHECK(SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT)); -#endif /* gpu resource config */ #ifdef MILVUS_GPU_VERSION STATUS_CHECK(SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT)); STATUS_CHECK(SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT)); STATUS_CHECK(SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT)); + STATUS_CHECK(SetGpuResourceConfigGpuSearchThreshold(CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD_DEFAULT)); STATUS_CHECK(SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT)); STATUS_CHECK(SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT)); #endif @@ -487,12 +494,8 @@ Config::ResetDefaultConfig() { STATUS_CHECK(SetWalConfigWalPath(CONFIG_WAL_WAL_PATH_DEFAULT)); /* logs config */ + STATUS_CHECK(SetLogsLevel(CONFIG_LOGS_LEVEL_DEFAULT)); STATUS_CHECK(SetLogsTraceEnable(CONFIG_LOGS_TRACE_ENABLE_DEFAULT)); - STATUS_CHECK(SetLogsDebugEnable(CONFIG_LOGS_DEBUG_ENABLE_DEFAULT)); - STATUS_CHECK(SetLogsInfoEnable(CONFIG_LOGS_INFO_ENABLE_DEFAULT)); - STATUS_CHECK(SetLogsWarningEnable(CONFIG_LOGS_WARNING_ENABLE_DEFAULT)); - STATUS_CHECK(SetLogsErrorEnable(CONFIG_LOGS_ERROR_ENABLE_DEFAULT)); - STATUS_CHECK(SetLogsFatalEnable(CONFIG_LOGS_FATAL_ENABLE_DEFAULT)); STATUS_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT)); STATUS_CHECK(SetLogsMaxLogFileSize(CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT)); STATUS_CHECK(SetLogsLogRotateNum(CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT)); @@ -523,35 +526,46 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key return Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } auto status = Status::OK(); - if (parent_key == CONFIG_SERVER) { - if (child_key == CONFIG_SERVER_ADDRESS) { - status = SetServerConfigAddress(value); - } else if (child_key == CONFIG_SERVER_DEPLOY_MODE) { - status = SetServerConfigDeployMode(value); - } else if (child_key == CONFIG_SERVER_PORT) { - status = SetServerConfigPort(value); - } else if (child_key == CONFIG_SERVER_TIME_ZONE) { - status = SetServerConfigTimeZone(value); - } else if (child_key == CONFIG_SERVER_WEB_PORT) { - status = SetServerConfigWebPort(value); + if (parent_key == CONFIG_CLUSTER) { + if (child_key == CONFIG_CLUSTER_ENABLE) { + status = SetClusterConfigEnable(value); + } else if (child_key == CONFIG_CLUSTER_ROLE) { + status = SetClusterConfigRole(value); } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } - } else if (parent_key == CONFIG_DB) { - if (child_key == CONFIG_DB_BACKEND_URL) { - status = SetDBConfigBackendUrl(value); - } else if (child_key == CONFIG_DB_PRELOAD_COLLECTION) { - status = SetDBConfigPreloadCollection(value); - } else if (child_key == CONFIG_DB_AUTO_FLUSH_INTERVAL) { - status = SetDBConfigAutoFlushInterval(value); + } else if (parent_key == CONFIG_GENERAL) { + if (child_key == CONFIG_GENERAL_TIMEZONE) { + status = SetGeneralConfigTimezone(value); + } else if (child_key == CONFIG_GENERAL_METAURI) { + status = SetGeneralConfigMetaURI(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); + } + } else if (parent_key == CONFIG_NETWORK) { + if (child_key == CONFIG_NETWORK_BIND_ADDRESS) { + status = SetNetworkConfigBindAddress(value); + } else if (child_key == CONFIG_NETWORK_BIND_PORT) { + status = SetNetworkConfigBindPort(value); + } else if (child_key == CONFIG_NETWORK_HTTP_PORT) { + status = SetNetworkConfigHTTPPort(value); } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } + } else if (parent_key == CONFIG_DB) { + // if (child_key == CONFIG_DB_BACKEND_URL) { + // status = SetDBConfigBackendUrl(value); + // } else if (child_key == CONFIG_DB_PRELOAD_COLLECTION) { + // if (child_key == CONFIG_DB_PRELOAD_COLLECTION) { + // status = SetDBConfigPreloadCollection(value); + // } else { + // status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); + // } } else if (parent_key == CONFIG_STORAGE) { - if (child_key == CONFIG_STORAGE_PRIMARY_PATH) { - status = SetStorageConfigPrimaryPath(value); - } else if (child_key == CONFIG_STORAGE_SECONDARY_PATH) { - status = SetStorageConfigSecondaryPath(value); + if (child_key == CONFIG_STORAGE_PATH) { + status = SetStorageConfigPath(value); + } else if (child_key == CONFIG_STORAGE_AUTO_FLUSH_INTERVAL) { + status = SetStorageConfigAutoFlushInterval(value); // } else if (child_key == CONFIG_STORAGE_S3_ENABLE) { // status = SetStorageConfigS3Enable(value); // } else if (child_key == CONFIG_STORAGE_S3_ADDRESS) { @@ -586,6 +600,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetCacheConfigCacheInsertData(value); } else if (child_key == CONFIG_CACHE_INSERT_BUFFER_SIZE) { status = SetCacheConfigInsertBufferSize(value); + } else if (child_key == CONFIG_CACHE_PRELOAD_COLLECTION) { + status = SetCacheConfigPreloadCollection(value); } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } @@ -596,10 +612,6 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetEngineConfigOmpThreadNum(value); } else if (child_key == CONFIG_ENGINE_SIMD_TYPE) { status = SetEngineConfigSimdType(value); -#ifdef MILVUS_GPU_VERSION - } else if (child_key == CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) { - status = SetEngineConfigGpuSearchThreshold(value); -#endif } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } @@ -611,6 +623,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetGpuResourceConfigCacheCapacity(value); } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_THRESHOLD) { status = SetGpuResourceConfigCacheThreshold(value); + } else if (child_key == CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD) { + status = SetGpuResourceConfigGpuSearchThreshold(value); } else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES) { status = SetGpuResourceConfigSearchResources(value); } else if (child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) { @@ -638,18 +652,10 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } else if (parent_key == CONFIG_LOGS) { - if (child_key == CONFIG_LOGS_TRACE_ENABLE) { + if (child_key == CONFIG_LOGS_LEVEL) { + status = SetLogsLevel(value); + } else if (child_key == CONFIG_LOGS_TRACE_ENABLE) { status = SetLogsTraceEnable(value); - } else if (child_key == CONFIG_LOGS_DEBUG_ENABLE) { - status = SetLogsDebugEnable(value); - } else if (child_key == CONFIG_LOGS_INFO_ENABLE) { - status = SetLogsInfoEnable(value); - } else if (child_key == CONFIG_LOGS_WARNING_ENABLE) { - status = SetLogsWarningEnable(value); - } else if (child_key == CONFIG_LOGS_ERROR_ENABLE) { - status = SetLogsErrorEnable(value); - } else if (child_key == CONFIG_LOGS_FATAL_ENABLE) { - status = SetLogsFatalEnable(value); } else if (child_key == CONFIG_LOGS_PATH) { status = SetLogsPath(value); } else if (child_key == CONFIG_LOGS_MAX_LOG_FILE_SIZE) { @@ -687,9 +693,18 @@ Config::ProcessConfigCli(std::string& result, const std::string& cmd) { return Status::OK(); } else { server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); - if (nodes.size() != 2) { + if (nodes.size() < 2) { return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } else if (nodes.size() > 2) { + // to support case likes network.bind.address + std::string result; + std::vector nodes_s(nodes.begin() + 1, nodes.end()); + StringHelpFunctions::MergeStringWithDelimeter(nodes_s, CONFIG_NODE_DELIMITER, result); + nodes[1] = result; } + // if (nodes.size() != 2) { + // return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + // } return GetConfigCli(result, nodes[0], nodes[1]); } } else if (tokens[0] == "set_config") { @@ -697,8 +712,14 @@ Config::ProcessConfigCli(std::string& result, const std::string& cmd) { return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); } server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); - if (nodes.size() != 2) { + if (nodes.size() < 2) { return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } else if (nodes.size() > 2) { + // to support case likes network.bind.address + std::string result; + std::vector nodes_s(nodes.begin() + 1, nodes.end()); + StringHelpFunctions::MergeStringWithDelimeter(nodes_s, CONFIG_NODE_DELIMITER, result); + nodes[1] = result; } return SetConfigCli(nodes[0], nodes[1], tokens[2]); } else { @@ -788,13 +809,15 @@ Config::UpdateFileConfigFromMem(const std::string& parent_key, const std::string while (getline(conf_fin, line)) { if (!parent_key_read) { conf_str += line + "\n"; - if (!(line.empty() || line.find_first_of('#') == 0 || line.find(parent_key) == std::string::npos)) - parent_key_read = true; + // TODO: danger + if (not(line.empty() || line.find_first_of('#') == 0)) + if (line.find(parent_key) == 0) + parent_key_read = true; continue; } if (line.find_first_of('#') == 0) { - status = Status(SERVER_UNEXPECTED_ERROR, "Cannot find child key: " + child_key); + status = Status(SERVER_UNEXPECTED_ERROR, "Cannot find child key: " + child_key + ", line is " + line); break; } @@ -882,150 +905,226 @@ Config::CheckConfigVersion(const std::string& value) { return Status::OK(); } -/* server config */ +/* cluster config */ Status -Config::CheckServerConfigAddress(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok(); - fiu_do_on("check_config_address_fail", exist_error = true); +Config::CheckClusterConfigEnable(const std::string& value) { + return ValidationUtil::ValidateStringIsBool(value); +} - if (exist_error) { - std::string msg = - "Invalid server IP address: " + value + ". Possible reason: server_config.address is invalid."; - return Status(SERVER_INVALID_ARGUMENT, msg); +Status +Config::CheckClusterConfigRole(const std::string& value) { + fiu_return_on("check_config_cluster_role_fail", + Status(SERVER_INVALID_ARGUMENT, "cluster.role is not one of rw and ro.")); + + if (value != "rw" && value != "ro") { + return Status(SERVER_INVALID_ARGUMENT, "cluster.role is not one of rw and ro."); } return Status::OK(); } +/* general config */ Status -Config::CheckServerConfigPort(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); - fiu_do_on("check_config_port_fail", exist_error = true); +Config::CheckGeneralConfigTimezone(const std::string& value) { + fiu_return_on("check_config_timezone_fail", Status(SERVER_INVALID_ARGUMENT, "Invalid general.timezone: " + value)); - if (exist_error) { - std::string msg = "Invalid server port: " + value + ". Possible reason: server_config.port is not a number."; - return Status(SERVER_INVALID_ARGUMENT, msg); + if (value.length() <= 3) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid general.timezone: " + value); } else { - try { - int32_t port = std::stoi(value); - if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { - std::string msg = "Invalid server port: " + value + - ". Possible reason: server_config.port is not in range (1024, 65535)."; - return Status(SERVER_INVALID_ARGUMENT, msg); + if (value.substr(0, 3) != "UTC") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid general.timezone: " + value); + } else { + if (!ValidationUtil::IsNumber(value.substr(4))) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid general.timezone: " + value); } - } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.port: " + value); } } return Status::OK(); } Status -Config::CheckServerConfigDeployMode(const std::string& value) { - fiu_return_on("check_config_deploy_mode_fail", - Status(SERVER_INVALID_ARGUMENT, - "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.")); +Config::CheckGeneralConfigMetaURI(const std::string& value) { + auto exist_error = !ValidationUtil::ValidateDbURI(value).ok(); + fiu_do_on("check_config_meta_uri_fail", exist_error = true); - if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { - return Status(SERVER_INVALID_ARGUMENT, - "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."); + if (exist_error) { + std::string msg = + "Invalid meta uri: " + value + ". Possible reason: general.meta_uri is invalid. " + + "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } +/* network config */ Status -Config::CheckServerConfigTimeZone(const std::string& value) { - fiu_return_on("check_config_time_zone_fail", - Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value)); +Config::CheckNetworkConfigBindAddress(const std::string& value) { + auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok(); + fiu_do_on("check_config_bind_address_fail", exist_error = true); - if (value.length() <= 3) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); + if (exist_error) { + std::string msg = "Invalid server IP address: " + value + ". Possible reason: network.bind.address is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + +Status +Config::CheckNetworkConfigBindPort(const std::string& value) { + auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); + fiu_do_on("check_config_bind_port_fail", exist_error = true); + + if (exist_error) { + std::string msg = "Invalid server port: " + value + ". Possible reason: network.bind.port is not a number."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { - if (value.substr(0, 3) != "UTC") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); - } else { - if (!ValidationUtil::IsNumber(value.substr(4))) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); + try { + int32_t port = std::stoi(value); + if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { + std::string msg = "Invalid server port: " + value + + ". Possible reason: network.bind.port is not in range (1024, 65535)."; + return Status(SERVER_INVALID_ARGUMENT, msg); } + } catch (...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid network.bind.port: " + value); } } return Status::OK(); } Status -Config::CheckServerConfigWebEnable(const std::string& value) { +Config::CheckNetworkConfigHTTPEnable(const std::string& value) { return ValidationUtil::ValidateStringIsBool(value); } Status -Config::CheckServerConfigWebPort(const std::string& value) { +Config::CheckNetworkConfigHTTPPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = - "Invalid web server port: " + value + ". Possible reason: server_config.web_port is not a number."; + std::string msg = "Invalid web server port: " + value + ". Possible reason: network.http.port is not a number."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { try { int32_t port = std::stoi(value); if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { std::string msg = "Invalid web server port: " + value + - ". Possible reason: server_config.web_port is not in range (1024, 65535)."; + ". Possible reason: network.http.port is not in range (1024, 65535)."; return Status(SERVER_INVALID_ARGUMENT, msg); } } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.web_port: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid network.http.port: " + value); } } return Status::OK(); } -/* DB config */ -Status -Config::CheckDBConfigBackendUrl(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateDbURI(value).ok(); - fiu_do_on("check_config_backend_url_fail", exist_error = true); - - if (exist_error) { - std::string msg = - "Invalid backend url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " + - "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckDBConfigPreloadCollection(const std::string& value) { - fiu_return_on("check_config_preload_collection_fail", Status(SERVER_INVALID_ARGUMENT, "")); +/* server config */ +// Status +// Config::CheckServerConfigAddress(const std::string& value) { +// auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok(); +// fiu_do_on("check_config_address_fail", exist_error = true); +// +// if (exist_error) { +// std::string msg = +// "Invalid server IP address: " + value + ". Possible reason: server_config.address is invalid."; +// return Status(SERVER_INVALID_ARGUMENT, msg); +// } +// return Status::OK(); +// } - if (value.empty() || value == "*") { - return Status::OK(); - } +// Status +// Config::CheckServerConfigPort(const std::string& value) { +// auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); +// fiu_do_on("check_config_port_fail", exist_error = true); +// +// if (exist_error) { +// std::string msg = "Invalid server port: " + value + ". Possible reason: server_config.port is not a number."; +// return Status(SERVER_INVALID_ARGUMENT, msg); +// } else { +// try { +// int32_t port = std::stoi(value); +// if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { +// std::string msg = "Invalid server port: " + value + +// ". Possible reason: server_config.port is not in range (1024, 65535)."; +// return Status(SERVER_INVALID_ARGUMENT, msg); +// } +// } catch (...) { +// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.port: " + value); +// } +// } +// return Status::OK(); +// } - std::vector tables; - StringHelpFunctions::SplitStringByDelimeter(value, ",", tables); +// Status +// Config::CheckServerConfigDeployMode(const std::string& value) { +// fiu_return_on("check_config_deploy_mode_fail", +// Status(SERVER_INVALID_ARGUMENT, +// "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.")); +// +// if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { +// return Status(SERVER_INVALID_ARGUMENT, +// "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."); +// } +// return Status::OK(); +//} - std::unordered_set table_set; +// Status +// Config::CheckServerConfigTimeZone(const std::string& value) { +// fiu_return_on("check_config_time_zone_fail", +// Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value)); +// +// if (value.length() <= 3) { +// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); +// } else { +// if (value.substr(0, 3) != "UTC") { +// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); +// } else { +// if (!ValidationUtil::IsNumber(value.substr(4))) { +// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); +// } +// } +// } +// return Status::OK(); +// } - for (auto& collection : tables) { - if (!ValidationUtil::ValidateCollectionName(collection).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid collection name: " + collection); - } - bool exist = false; - auto status = DBWrapper::DB()->HasNativeCollection(collection, exist); - if (!(status.ok() && exist)) { - return Status(SERVER_COLLECTION_NOT_EXIST, "Collection " + collection + " not exist"); - } - table_set.insert(collection); - } +// Status +// Config::CheckServerConfigWebEnable(const std::string& value) { +// return ValidationUtil::ValidateStringIsBool(value); +// } - if (table_set.size() != tables.size()) { - std::string msg = - "Invalid preload tables. " - "Possible reason: db_config.preload_collection contains duplicate collection."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } +// Status +// Config::CheckServerConfigWebPort(const std::string& value) { +// if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { +// std::string msg = +// "Invalid web server port: " + value + ". Possible reason: server_config.web_port is not a number."; +// return Status(SERVER_INVALID_ARGUMENT, msg); +// } else { +// try { +// int32_t port = std::stoi(value); +// if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { +// std::string msg = "Invalid web server port: " + value + +// ". Possible reason: server_config.web_port is not in range (1024, 65535)."; +// return Status(SERVER_INVALID_ARGUMENT, msg); +// } +// } catch (...) { +// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.web_port: " + value); +// } +// } +// return Status::OK(); +// } - return Status::OK(); -} +/* DB config */ +// Status +// Config::CheckDBConfigBackendUrl(const std::string& value) { +// auto exist_error = !ValidationUtil::ValidateDbURI(value).ok(); +// fiu_do_on("check_config_backend_url_fail", exist_error = true); +// +// if (exist_error) { +// std::string msg = +// "Invalid backend url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " + +// "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; +// return Status(SERVER_INVALID_ARGUMENT, msg); +// } +// return Status::OK(); +// } Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { @@ -1053,56 +1152,26 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckDBConfigAutoFlushInterval(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); - fiu_do_on("check_config_auto_flush_interval_fail", exist_error = true); - - if (exist_error) { - std::string msg = "Invalid db configuration auto_flush_interval: " + value + - ". Possible reason: db.auto_flush_interval is not a natural number."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - return Status::OK(); -} - /* storage config */ Status -Config::CheckStorageConfigPrimaryPath(const std::string& value) { - fiu_return_on("check_config_primary_path_fail", Status(SERVER_INVALID_ARGUMENT, "")); +Config::CheckStorageConfigPath(const std::string& value) { + fiu_return_on("check_config_path_fail", Status(SERVER_INVALID_ARGUMENT, "")); if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "storage_config.db_path is empty."); + return Status(SERVER_INVALID_ARGUMENT, "storage.path is empty."); } return ValidationUtil::ValidateStoragePath(value); } Status -Config::CheckStorageConfigSecondaryPath(const std::string& value) { - fiu_return_on("check_config_secondary_path_fail", Status(SERVER_INVALID_ARGUMENT, "")); - - auto status = Status::OK(); - - if (value.empty()) { - return status; - } - - std::vector vec; - StringHelpFunctions::SplitStringByDelimeter(value, ",", vec); - std::unordered_set path_set; - for (auto& path : vec) { - StringHelpFunctions::TrimStringBlank(path); - status = ValidationUtil::ValidateStoragePath(path); - if (!status.ok()) { - return status; - } - - path_set.insert(path); - } +Config::CheckStorageConfigAutoFlushInterval(const std::string& value) { + auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); + fiu_do_on("check_config_auto_flush_interval_fail", exist_error = true); - if (path_set.size() != vec.size()) { - return Status(SERVER_INVALID_ARGUMENT, "Path value is duplicated"); + if (exist_error) { + std::string msg = "Invalid storage configuration auto_flush_interval: " + value + + ". Possible reason: storage.auto_flush_interval is not a natural number."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1112,14 +1181,14 @@ Status Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid file_cleanup_timeout: " + value + - ". Possible reason: storage_config.file_cleanup_timeout is not a positive integer."; + ". Possible reason: storage.file_cleanup_timeout is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t file_cleanup_timeout = std::stoll(value); if (file_cleanup_timeout < CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN || file_cleanup_timeout > CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX) { std::string msg = "Invalid file_cleanup_timeout: " + value + - ". Possible reason: storage_config.file_cleanup_timeout is not in range [" + + ". Possible reason: storage.file_cleanup_timeout is not in range [" + std::to_string(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN) + ", " + std::to_string(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN) + "]."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -1199,8 +1268,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) { fiu_do_on("check_config_enable_monitor_fail", exist_error = true); if (exist_error) { - std::string msg = - "Invalid metric config: " + value + ". Possible reason: metric_config.enable_monitor is not a boolean."; + std::string msg = "Invalid metric config: " + value + ". Possible reason: metric.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1209,7 +1277,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) { Status Config::CheckMetricConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { - std::string msg = "Invalid metric ip: " + value + ". Possible reason: metric_config.ip is invalid."; + std::string msg = "Invalid metric ip: " + value + ". Possible reason: metric.ip is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1218,18 +1286,18 @@ Config::CheckMetricConfigAddress(const std::string& value) { Status Config::CheckMetricConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid metric port: " + value + ". Possible reason: metric_config.port is not a number."; + std::string msg = "Invalid metric port: " + value + ". Possible reason: metric.port is not a number."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { try { int32_t port = std::stoi(value); if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { - std::string msg = "Invalid metric port: " + value + - ". Possible reason: metric_config.port is not in range (1024, 65535)."; + std::string msg = + "Invalid metric port: " + value + ". Possible reason: metric.port is not in range (1024, 65535)."; return Status(SERVER_INVALID_ARGUMENT, msg); } } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid metric_config.port: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric.port: " + value); } } return Status::OK(); @@ -1238,8 +1306,42 @@ Config::CheckMetricConfigPort(const std::string& value) { /* cache config */ Status Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { - fiu_return_on("check_config_cpu_cache_capacity_fail", Status(SERVER_INVALID_ARGUMENT, "")); + fiu_return_on("check_config_cache_size_fail", Status(SERVER_INVALID_ARGUMENT, "")); +#if 1 + std::string err; + int64_t cache_size = parse_bytes(value, err); + if (not err.empty()) { + return Status(SERVER_INVALID_ARGUMENT, err); + } else { + if (cache_size <= 0) { + std::string msg = "Invalid cpu cache capacity: " + value + + ". Possible reason: cache.cache_size is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + int64_t total_mem = 0, free_mem = 0; + CommonUtil::GetSystemMemInfo(total_mem, free_mem); + if (cache_size >= total_mem) { + std::string msg = + "Invalid cpu cache size: " + value + ". Possible reason: cache.cache_size exceeds system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (static_cast(cache_size) > static_cast(total_mem * 0.9)) { + std::cerr << "WARNING: cpu cache size value is too big" << std::endl; + } + + std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, "0"); + + int64_t insert_buffer_size = parse_bytes(value, err); + fiu_do_on("Config.CheckCacheConfigCpuCacheCapacity.large_insert_buffer", insert_buffer_size = total_mem + 1); + if (insert_buffer_size + cache_size >= total_mem) { + std::string msg = "Invalid cpu cache size: " + value + + ". Possible reason: sum of cache.cache_size and " + "cache.insert_buffer_size exceeds system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } +#else if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -1274,6 +1376,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } } +#endif return Status::OK(); } @@ -1299,6 +1402,33 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { Status Config::CheckCacheConfigInsertBufferSize(const std::string& value) { fiu_return_on("check_config_insert_buffer_size_fail", Status(SERVER_INVALID_ARGUMENT, "")); + +#if 1 + std::string err; + int64_t buffer_size = parse_bytes(value, err); + if (not err.empty()) { + return Status(SERVER_INVALID_ARGUMENT, err); + } else { + if (buffer_size <= 0) { + std::string msg = "Invalid insert buffer size: " + value + + ". Possible reason: cache.insert_buffer_size is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, "0"); + std::string err; + int64_t cache_size = parse_bytes(str, err); + + int64_t total_mem = 0, free_mem = 0; + CommonUtil::GetSystemMemInfo(total_mem, free_mem); + if (buffer_size + cache_size >= total_mem) { + std::string msg = "Invalid insert buffer size: " + value + + ". Possible reason: sum of cache.cache_size and " + "cache.insert_buffer_size exceeds system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } +#else if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: cache_config.insert_buffer_size is not a positive integer."; @@ -1312,7 +1442,8 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) { } std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, "0"); - int64_t cache_size = std::stoll(str) * GB; + std::string err; + int64_t cache_size = parse_bytes(str, err); int64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); @@ -1322,6 +1453,7 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } } +#endif return Status::OK(); } @@ -1337,6 +1469,41 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) { return Status::OK(); } +Status +Config::CheckCacheConfigPreloadCollection(const std::string& value) { + fiu_return_on("check_config_preload_collection_fail", Status(SERVER_INVALID_ARGUMENT, "")); + + if (value.empty() || value == "*") { + return Status::OK(); + } + + std::vector tables; + StringHelpFunctions::SplitStringByDelimeter(value, ",", tables); + + std::unordered_set table_set; + + for (auto& collection : tables) { + if (!ValidationUtil::ValidateCollectionName(collection).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid collection name: " + collection); + } + bool exist = false; + auto status = DBWrapper::DB()->HasNativeCollection(collection, exist); + if (!(status.ok() && exist)) { + return Status(SERVER_COLLECTION_NOT_EXIST, "Collection " + collection + " not exist"); + } + table_set.insert(collection); + } + + if (table_set.size() != tables.size()) { + std::string msg = + "Invalid preload tables. " + "Possible reason: cache.preload_collection contains duplicate collection."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + return Status::OK(); +} + /* engine config */ Status Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { @@ -1384,26 +1551,13 @@ Config::CheckEngineConfigSimdType(const std::string& value) { #ifdef MILVUS_GPU_VERSION -Status -Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { - fiu_return_on("check_config_gpu_search_threshold_fail", Status(SERVER_INVALID_ARGUMENT, "")); - - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu search threshold: " + value + - ". Possible reason: engine_config.gpu_search_threshold is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - /* gpu resource config */ Status Config::CheckGpuResourceConfigEnable(const std::string& value) { fiu_return_on("check_config_gpu_resource_enable_fail", Status(SERVER_INVALID_ARGUMENT, "")); if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = - "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; + std::string msg = "Invalid gpu resource config: " + value + ". Possible reason: gpu.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1411,11 +1565,39 @@ Config::CheckGpuResourceConfigEnable(const std::string& value) { Status Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { - fiu_return_on("check_gpu_resource_config_cache_capacity_fail", Status(SERVER_INVALID_ARGUMENT, "")); + fiu_return_on("check_gpu_cache_size_fail", Status(SERVER_INVALID_ARGUMENT, "")); +#if 1 + std::string err; + int64_t gpu_cache_size = parse_bytes(value, err); + if (not err.empty()) { + return Status(SERVER_INVALID_ARGUMENT, err); + } else { + if (gpu_cache_size < 0) { + std::string msg = "gpu.cache_size must greater than 0, now is " + std::to_string(gpu_cache_size) + "."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + std::vector gpu_ids; + STATUS_CHECK(GetGpuResourceConfigBuildIndexResources(gpu_ids)); + + for (int64_t gpu_id : gpu_ids) { + int64_t gpu_memory; + if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_size >= gpu_memory) { + std::string msg = + "Invalid gpu cache capacity: " + value + ". Possible reason: gpu.cache_size exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_size > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache size value is too big" << std::endl; + } + } + } +#else if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; + std::string msg = + "Invalid gpu cache capacity: " + value + ". Possible reason: gpu.cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t gpu_cache_capacity = std::stoll(value) * GB; @@ -1429,13 +1611,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { return Status(SERVER_UNEXPECTED_ERROR, msg); } else if (gpu_cache_capacity >= gpu_memory) { std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: gpu_resource_config.cache_capacity exceeds GPU memory."; + ". Possible reason: gpu.cache_capacity exceeds GPU memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; } } } +#endif return Status::OK(); } @@ -1445,19 +1628,31 @@ Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + ". Possible reason: gpu.cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { float gpu_cache_threshold = std::stof(value); if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + ". Possible reason: gpu.cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); } +Status +Config::CheckGpuResourceConfigGpuSearchThreshold(const std::string& value) { + fiu_return_on("check_config_gpu_search_threshold_fail", Status(SERVER_INVALID_ARGUMENT, "")); + + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu search threshold: " + value + + ". Possible reason: gpu.gpu_search_threshold is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + Status CheckGpuResource(const std::string& value) { std::string s = value; @@ -1466,8 +1661,8 @@ CheckGpuResource(const std::string& value) { const std::regex pat("gpu(\\d+)"); std::smatch m; if (!std::regex_match(s, m, pat)) { - std::string msg = "Invalid gpu resource: " + value + - ". Possible reason: gpu_resource_config is not in the format of cpux or gpux"; + std::string msg = + "Invalid gpu resource: " + value + ". Possible reason: gpu is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -1475,8 +1670,8 @@ CheckGpuResource(const std::string& value) { try { int32_t gpu_index = std::stoi(s.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid gpu resource: " + value + - ". Possible reason: gpu_resource_config does not match with the hardware."; + std::string msg = + "Invalid gpu resource: " + value + ". Possible reason: gpu does not match with the hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } catch (...) { @@ -1489,12 +1684,12 @@ CheckGpuResource(const std::string& value) { Status Config::CheckGpuResourceConfigSearchResources(const std::vector& value) { - fiu_return_on("check_gpu_resource_config_search_fail", Status(SERVER_INVALID_ARGUMENT, "")); + fiu_return_on("check_gpu_search_fail", Status(SERVER_INVALID_ARGUMENT, "")); if (value.empty()) { std::string msg = "Invalid gpu search resource. " - "Possible reason: gpu_resource_config.search_resources is empty."; + "Possible reason: gpu.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -1507,7 +1702,7 @@ Config::CheckGpuResourceConfigSearchResources(const std::vector& va if (value_set.size() != value.size()) { std::string msg = "Invalid gpu build search resource. " - "Possible reason: gpu_resource_config.gpu_search_resources contains duplicate resources."; + "Possible reason: gpu.gpu_search_resources contains duplicate resources."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -1516,12 +1711,12 @@ Config::CheckGpuResourceConfigSearchResources(const std::vector& va Status Config::CheckGpuResourceConfigBuildIndexResources(const std::vector& value) { - fiu_return_on("check_gpu_resource_config_build_index_fail", Status(SERVER_INVALID_ARGUMENT, "")); + fiu_return_on("check_gpu_build_index_fail", Status(SERVER_INVALID_ARGUMENT, "")); if (value.empty()) { std::string msg = "Invalid gpu build index resource. " - "Possible reason: gpu_resource_config.build_index_resources is empty."; + "Possible reason: gpu.build_index_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -1534,7 +1729,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector if (value_set.size() != value.size()) { std::string msg = "Invalid gpu build index resource. " - "Possible reason: gpu_resource_config.build_index_resources contains duplicate resources."; + "Possible reason: gpu.build_index_resources contains duplicate resources."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -1558,7 +1753,7 @@ Config::CheckWalConfigEnable(const std::string& value) { fiu_do_on("check_config_wal_enable_fail", exist_error = true); if (exist_error) { - std::string msg = "Invalid wal config: " + value + ". Possible reason: wal_config.enable is not a boolean."; + std::string msg = "Invalid wal config: " + value + ". Possible reason: wal.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1571,7 +1766,7 @@ Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) { if (exist_error) { std::string msg = - "Invalid wal config: " + value + ". Possible reason: wal_config.recovery_error_ignore is not a boolean."; + "Invalid wal config: " + value + ". Possible reason: wal.recovery_error_ignore is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1579,12 +1774,14 @@ Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) { Status Config::CheckWalConfigBufferSize(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); + std::string err; + auto buffer_size = parse_bytes(value, err); + auto exist_error = not err.empty(); fiu_do_on("check_config_wal_buffer_size_fail", exist_error = true); - if (exist_error) { - std::string msg = "Invalid wal buffer size: " + value + - ". Possible reason: wal_config.buffer_size is not a positive integer."; + if (exist_error || buffer_size < 0) { + std::string msg = + "Invalid wal buffer size: " + value + ". Possible reason: wal.buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1594,7 +1791,7 @@ Status Config::CheckWalConfigWalPath(const std::string& value) { fiu_return_on("check_wal_path_fail", Status(SERVER_INVALID_ARGUMENT, "")); if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "wal_config.wal_path is empty!"); + return Status(SERVER_INVALID_ARGUMENT, "wal.path is empty!"); } return ValidationUtil::ValidateStoragePath(value); @@ -1602,72 +1799,21 @@ Config::CheckWalConfigWalPath(const std::string& value) { /* logs config */ Status -Config::CheckLogsTraceEnable(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); - fiu_do_on("check_logs_trace_enable_fail", exist_error = true); - - if (exist_error) { - std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.trace.enable is not a boolean."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckLogsDebugEnable(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); - fiu_do_on("check_logs_debug_enable_fail", exist_error = true); - - if (exist_error) { - std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.debug.enable is not a boolean."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckLogsInfoEnable(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); - fiu_do_on("check_logs_info_enable_fail", exist_error = true); - - if (exist_error) { - std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.info.enable is not a boolean."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckLogsWarningEnable(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); - fiu_do_on("check_logs_warning_enable_fail", exist_error = true); - - if (exist_error) { - std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.warning.enable is not a boolean."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckLogsErrorEnable(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); - fiu_do_on("check_logs_error_enable_fail", exist_error = true); - - if (exist_error) { - std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.error.enable is not a boolean."; - return Status(SERVER_INVALID_ARGUMENT, msg); +Config::CheckLogsLevel(const std::string& value) { + fiu_return_on("check_logs_level_fail", Status(SERVER_INVALID_ARGUMENT, "")); + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "logs.level is empty!"); } - return Status::OK(); + return ValidationUtil::ValidateLogLevel(value); } Status -Config::CheckLogsFatalEnable(const std::string& value) { +Config::CheckLogsTraceEnable(const std::string& value) { auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok(); - fiu_do_on("check_logs_fatal_enable_fail", exist_error = true); + fiu_do_on("check_logs_trace_enable_fail", exist_error = true); if (exist_error) { - std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.fatal.enable is not a boolean."; + std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.trace.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -1685,7 +1831,9 @@ Config::CheckLogsPath(const std::string& value) { Status Config::CheckLogsMaxLogFileSize(const std::string& value) { - auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); + std::string err; + int64_t max_log_file_size = parse_bytes(value, err); + auto exist_error = not err.empty(); fiu_do_on("check_logs_max_log_file_size_fail", exist_error = true); if (exist_error) { @@ -1693,7 +1841,6 @@ Config::CheckLogsMaxLogFileSize(const std::string& value) { ". Possible reason: logs.max_log_file_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t max_log_file_size = std::stoll(value); if (max_log_file_size < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN || max_log_file_size > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) { std::string msg = "Invalid max_log_file_size: " + value + @@ -1825,51 +1972,104 @@ Config::ExecCallBacks(const std::string& node, const std::string& sub_node, cons return status; } -/* server config */ +/* cluster config */ Status -Config::GetServerConfigAddress(std::string& value) { - value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - return CheckServerConfigAddress(value); +Config::GetClusterConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_CLUSTER, CONFIG_CLUSTER_ENABLE, CONFIG_CLUSTER_ENABLE_DEFAULT); + STATUS_CHECK(CheckClusterConfigEnable(str)); + return StringHelpFunctions::ConvertToBoolean(str, value); } Status -Config::GetServerConfigPort(std::string& value) { - value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - return CheckServerConfigPort(value); +Config::GetClusterConfigRole(std::string& value) { + value = GetConfigStr(CONFIG_CLUSTER, CONFIG_CLUSTER_ROLE, CONFIG_CLUSTER_ROLE_DEFAULT); + return CheckClusterConfigRole(value); } +/* general config */ Status -Config::GetServerConfigDeployMode(std::string& value) { - value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); - return CheckServerConfigDeployMode(value); +Config::GetGeneralConfigTimezone(std::string& value) { + value = GetConfigStr(CONFIG_GENERAL, CONFIG_GENERAL_TIMEZONE, CONFIG_GENERAL_TIMEZONE_DEFAULT); + return CheckGeneralConfigTimezone(value); } Status -Config::GetServerConfigTimeZone(std::string& value) { - value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - return CheckServerConfigTimeZone(value); +Config::GetGeneralConfigMetaURI(std::string& value) { + value = GetConfigStr(CONFIG_GENERAL, CONFIG_GENERAL_METAURI, CONFIG_GENERAL_METAURI_DEFAULT); + return CheckGeneralConfigMetaURI(value); } +/* network config */ Status -Config::GetServerConfigWebEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_WEB_ENABLE, CONFIG_SERVER_WEB_ENABLE_DEFAULT); - STATUS_CHECK(CheckServerConfigWebEnable(str)); - return StringHelpFunctions::ConvertToBoolean(str, value); +Config::GetNetworkConfigBindAddress(std::string& value) { + value = GetConfigStr(CONFIG_NETWORK, CONFIG_NETWORK_BIND_ADDRESS, CONFIG_NETWORK_BIND_ADDRESS_DEFAULT); + return CheckNetworkConfigBindAddress(value); } Status -Config::GetServerConfigWebPort(std::string& value) { - value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_WEB_PORT, CONFIG_SERVER_WEB_PORT_DEFAULT); - return CheckServerConfigWebPort(value); +Config::GetNetworkConfigBindPort(std::string& value) { + value = GetConfigStr(CONFIG_NETWORK, CONFIG_NETWORK_BIND_PORT, CONFIG_NETWORK_BIND_PORT_DEFAULT); + return CheckNetworkConfigBindPort(value); } -/* DB config */ Status -Config::GetDBConfigBackendUrl(std::string& value) { - value = GetConfigStr(CONFIG_DB, CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); - return CheckDBConfigBackendUrl(value); +Config::GetNetworkConfigHTTPEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_NETWORK, CONFIG_NETWORK_HTTP_ENABLE, CONFIG_NETWORK_HTTP_ENABLE_DEFAULT); + STATUS_CHECK(CheckNetworkConfigHTTPEnable(str)); + return StringHelpFunctions::ConvertToBoolean(str, value); } +Status +Config::GetNetworkConfigHTTPPort(std::string& value) { + value = GetConfigStr(CONFIG_NETWORK, CONFIG_NETWORK_HTTP_PORT, CONFIG_NETWORK_HTTP_PORT_DEFAULT); + return CheckNetworkConfigHTTPPort(value); +} + +/* server config */ +// Status +// Config::GetServerConfigAddress(std::string& value) { +// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); +// return CheckServerConfigAddress(value); +// } + +// Status +// Config::GetServerConfigPort(std::string& value) { +// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); +// return CheckServerConfigPort(value); +// } + +// Status +// Config::GetServerConfigDeployMode(std::string& value) { +// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); +// return CheckServerConfigDeployMode(value); +// } + +// Status +// Config::GetServerConfigTimeZone(std::string& value) { +// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); +// return CheckServerConfigTimeZone(value); +// } + +// Status +// Config::GetServerConfigWebEnable(bool& value) { +// std::string str = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_WEB_ENABLE, CONFIG_SERVER_WEB_ENABLE_DEFAULT); +// STATUS_CHECK(CheckServerConfigWebEnable(str)); +// return StringHelpFunctions::ConvertToBoolean(str, value); +// } + +// Status +// Config::GetServerConfigWebPort(std::string& value) { +// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_WEB_PORT, CONFIG_SERVER_WEB_PORT_DEFAULT); +// return CheckServerConfigWebPort(value); +// } + +/* DB config */ +// Status +// Config::GetDBConfigBackendUrl(std::string& value) { +// value = GetConfigStr(CONFIG_DB, CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); +// return CheckDBConfigBackendUrl(value); +// } + Status Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = @@ -1888,33 +2088,22 @@ Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { return Status::OK(); } +/* storage config */ Status -Config::GetDBConfigPreloadCollection(std::string& value) { - value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_COLLECTION); - return Status::OK(); +Config::GetStorageConfigPath(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_PATH, CONFIG_STORAGE_PATH_DEFAULT); + return CheckStorageConfigPath(value); } Status -Config::GetDBConfigAutoFlushInterval(int64_t& value) { - std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_AUTO_FLUSH_INTERVAL, CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT); - STATUS_CHECK(CheckDBConfigAutoFlushInterval(str)); +Config::GetStorageConfigAutoFlushInterval(int64_t& value) { + std::string str = + GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_AUTO_FLUSH_INTERVAL, CONFIG_STORAGE_AUTO_FLUSH_INTERVAL_DEFAULT); + STATUS_CHECK(CheckStorageConfigAutoFlushInterval(str)); value = std::stoll(str); return Status::OK(); } -/* storage config */ -Status -Config::GetStorageConfigPrimaryPath(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_PRIMARY_PATH, CONFIG_STORAGE_PRIMARY_PATH_DEFAULT); - return CheckStorageConfigPrimaryPath(value); -} - -Status -Config::GetStorageConfigSecondaryPath(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, CONFIG_STORAGE_SECONDARY_PATH_DEFAULT); - return CheckStorageConfigSecondaryPath(value); -} - Status Config::GetStorageConfigFileCleanupTimeup(int64_t& value) { std::string str = @@ -1989,7 +2178,9 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); STATUS_CHECK(CheckCacheConfigCpuCacheCapacity(str)); - value = std::stoll(str); + std::string err; + value = parse_bytes(str, err); + // value = std::stoll(str); return Status::OK(); } @@ -2007,7 +2198,9 @@ Config::GetCacheConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT); STATUS_CHECK(CheckCacheConfigInsertBufferSize(str)); - value = std::stoll(str); + std::string err; + value = parse_bytes(str, err); + // value = std::stoll(str); return Status::OK(); } @@ -2021,6 +2214,12 @@ Config::GetCacheConfigCacheInsertData(bool& value) { return Status::OK(); } +Status +Config::GetCacheConfigPreloadCollection(std::string& value) { + value = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_PRELOAD_COLLECTION); + return Status::OK(); +} + /* engine config */ Status Config::GetEngineConfigUseBlasThreshold(int64_t& value) { @@ -2045,17 +2244,6 @@ Config::GetEngineConfigSimdType(std::string& value) { return CheckEngineConfigSimdType(value); } -#ifdef MILVUS_GPU_VERSION -Status -Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { - std::string str = - GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); - STATUS_CHECK(CheckEngineConfigGpuSearchThreshold(str)); - value = std::stoll(str); - return Status::OK(); -} -#endif - /* gpu resource config */ #ifdef MILVUS_GPU_VERSION @@ -2073,13 +2261,15 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { STATUS_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); fiu_do_on("Config.GetGpuResourceConfigCacheCapacity.diable_gpu_resource", gpu_resource_enable = false); if (!gpu_resource_enable) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + std::string msg = "GPU not supported. Possible reason: gpu.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); STATUS_CHECK(CheckGpuResourceConfigCacheCapacity(str)); - value = std::stoll(str); + std::string err; + value = parse_bytes(str, err); + // value = std::stoll(str); return Status::OK(); } @@ -2089,7 +2279,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { STATUS_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); fiu_do_on("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource", gpu_resource_enable = false); if (!gpu_resource_enable) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + std::string msg = "GPU not supported. Possible reason: gpu.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, @@ -2099,13 +2289,22 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { return Status::OK(); } +Status +Config::GetGpuResourceConfigGpuSearchThreshold(int64_t& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, + CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD_DEFAULT); + STATUS_CHECK(CheckGpuResourceConfigGpuSearchThreshold(str)); + value = std::stoll(str); + return Status::OK(); +} + Status Config::GetGpuResourceConfigSearchResources(std::vector& value) { bool gpu_resource_enable = false; STATUS_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); fiu_do_on("get_gpu_config_search_resources.disable_gpu_resource_fail", gpu_resource_enable = false); if (!gpu_resource_enable) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + std::string msg = "GPU not supported. Possible reason: gpu.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, @@ -2126,7 +2325,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { STATUS_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); fiu_do_on("get_gpu_config_build_index_resources.disable_gpu_resource_fail", gpu_resource_enable = false); if (!gpu_resource_enable) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + std::string msg = "GPU not supported. Possible reason: gpu.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = @@ -2182,7 +2381,9 @@ Status Config::GetWalConfigBufferSize(int64_t& buffer_size) { std::string str = GetConfigStr(CONFIG_WAL, CONFIG_WAL_BUFFER_SIZE, CONFIG_WAL_BUFFER_SIZE_DEFAULT); STATUS_CHECK(CheckWalConfigBufferSize(str)); - buffer_size = std::stoll(str); + std::string err; + buffer_size = parse_bytes(str, err); + // buffer_size = std::stoll(str); if (buffer_size > CONFIG_WAL_BUFFER_SIZE_MAX) { buffer_size = CONFIG_WAL_BUFFER_SIZE_MAX; } else if (buffer_size < CONFIG_WAL_BUFFER_SIZE_MIN) { @@ -2200,49 +2401,16 @@ Config::GetWalConfigWalPath(std::string& wal_path) { /* logs config */ Status -Config::GetLogsTraceEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_TRACE_ENABLE, CONFIG_LOGS_TRACE_ENABLE_DEFAULT); - STATUS_CHECK(CheckLogsTraceEnable(str)); - STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); - return Status::OK(); -} - -Status -Config::GetLogsDebugEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_DEBUG_ENABLE, CONFIG_LOGS_DEBUG_ENABLE_DEFAULT); - STATUS_CHECK(CheckLogsDebugEnable(str)); - STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); - return Status::OK(); -} - -Status -Config::GetLogsInfoEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_INFO_ENABLE, CONFIG_LOGS_INFO_ENABLE_DEFAULT); - STATUS_CHECK(CheckLogsInfoEnable(str)); - STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); - return Status::OK(); -} - -Status -Config::GetLogsWarningEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_WARNING_ENABLE, CONFIG_LOGS_WARNING_ENABLE_DEFAULT); - STATUS_CHECK(CheckLogsWarningEnable(str)); - STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); +Config::GetLogsLevel(std::string& value) { + value = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LEVEL, CONFIG_LOGS_LEVEL_DEFAULT); + STATUS_CHECK(CheckLogsLevel(value)); return Status::OK(); } Status -Config::GetLogsErrorEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_ERROR_ENABLE, CONFIG_LOGS_ERROR_ENABLE_DEFAULT); - STATUS_CHECK(CheckLogsErrorEnable(str)); - STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); - return Status::OK(); -} - -Status -Config::GetLogsFatalEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_FATAL_ENABLE, CONFIG_LOGS_FATAL_ENABLE_DEFAULT); - STATUS_CHECK(CheckLogsFatalEnable(str)); +Config::GetLogsTraceEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_TRACE_ENABLE, CONFIG_LOGS_TRACE_ENABLE_DEFAULT); + STATUS_CHECK(CheckLogsTraceEnable(str)); STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -2258,7 +2426,9 @@ Status Config::GetLogsMaxLogFileSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_MAX_LOG_FILE_SIZE, CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT); STATUS_CHECK(CheckLogsMaxLogFileSize(str)); - value = std::stoll(str); + std::string err; + value = parse_bytes(str, err); + // value = std::stoll(str); return Status::OK(); } @@ -2277,57 +2447,100 @@ Config::GetServerRestartRequired(bool& required) { } /////////////////////////////////////////////////////////////////////////////// -/* server config */ +/* cluster config */ Status -Config::SetServerConfigAddress(const std::string& value) { - STATUS_CHECK(CheckServerConfigAddress(value)); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); +Config::SetClusterConfigEnable(const std::string& value) { + STATUS_CHECK(CheckClusterConfigEnable(value)); + return SetConfigValueInMem(CONFIG_CLUSTER, CONFIG_CLUSTER_ENABLE, value); } - Status -Config::SetServerConfigPort(const std::string& value) { - STATUS_CHECK(CheckServerConfigPort(value)); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); +Config::SetClusterConfigRole(const std::string& value) { + STATUS_CHECK(CheckClusterConfigRole(value)); + return SetConfigValueInMem(CONFIG_CLUSTER, CONFIG_CLUSTER_ROLE, value); } +/* general config */ Status -Config::SetServerConfigDeployMode(const std::string& value) { - STATUS_CHECK(CheckServerConfigDeployMode(value)); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); +Config::SetGeneralConfigTimezone(const std::string& value) { + STATUS_CHECK(CheckGeneralConfigTimezone(value)); + return SetConfigValueInMem(CONFIG_GENERAL, CONFIG_GENERAL_TIMEZONE, value); } Status -Config::SetServerConfigTimeZone(const std::string& value) { - STATUS_CHECK(CheckServerConfigTimeZone(value)); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); +Config::SetGeneralConfigMetaURI(const std::string& value) { + STATUS_CHECK(CheckGeneralConfigMetaURI(value)); + return SetConfigValueInMem(CONFIG_GENERAL, CONFIG_GENERAL_METAURI, value); } +/* network config */ Status -Config::SetServerConfigWebEnable(const std::string& value) { - STATUS_CHECK(CheckServerConfigWebEnable(value)); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_WEB_ENABLE, value); +Config::SetNetworkConfigBindAddress(const std::string& value) { + STATUS_CHECK(CheckNetworkConfigBindAddress(value)); + return SetConfigValueInMem(CONFIG_NETWORK, CONFIG_NETWORK_BIND_ADDRESS, value); } Status -Config::SetServerConfigWebPort(const std::string& value) { - STATUS_CHECK(CheckServerConfigWebPort(value)); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_WEB_PORT, value); +Config::SetNetworkConfigBindPort(const std::string& value) { + STATUS_CHECK(CheckNetworkConfigBindPort(value)); + return SetConfigValueInMem(CONFIG_NETWORK, CONFIG_NETWORK_BIND_PORT, value); } -/* db config */ Status -Config::SetDBConfigBackendUrl(const std::string& value) { - STATUS_CHECK(CheckDBConfigBackendUrl(value)); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); +Config::SetNetworkConfigHTTPEnable(const std::string& value) { + STATUS_CHECK(CheckNetworkConfigHTTPEnable(value)); + return SetConfigValueInMem(CONFIG_NETWORK, CONFIG_NETWORK_HTTP_ENABLE, value); } Status -Config::SetDBConfigPreloadCollection(const std::string& value) { - STATUS_CHECK(CheckDBConfigPreloadCollection(value)); - std::string cor_value = value == "*" ? "\'*\'" : value; - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_COLLECTION, cor_value); +Config::SetNetworkConfigHTTPPort(const std::string& value) { + STATUS_CHECK(CheckNetworkConfigHTTPPort(value)); + return SetConfigValueInMem(CONFIG_NETWORK, CONFIG_NETWORK_HTTP_PORT, value); } +/* server config */ +// Status +// Config::SetServerConfigAddress(const std::string& value) { +// STATUS_CHECK(CheckServerConfigAddress(value)); +// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); +// } + +// Status +// Config::SetServerConfigPort(const std::string& value) { +// STATUS_CHECK(CheckServerConfigPort(value)); +// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); +// } + +// Status +// Config::SetServerConfigDeployMode(const std::string& value) { +// STATUS_CHECK(CheckServerConfigDeployMode(value)); +// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); +// } + +// Status +// Config::SetServerConfigTimeZone(const std::string& value) { +// STATUS_CHECK(CheckServerConfigTimeZone(value)); +// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); +// } + +// Status +// Config::SetServerConfigWebEnable(const std::string& value) { +// STATUS_CHECK(CheckServerConfigWebEnable(value)); +// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_WEB_ENABLE, value); +// } + +// Status +// Config::SetServerConfigWebPort(const std::string& value) { +// STATUS_CHECK(CheckServerConfigWebPort(value)); +// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_WEB_PORT, value); +// } + +/* db config */ +// Status +// Config::SetDBConfigBackendUrl(const std::string& value) { +// STATUS_CHECK(CheckDBConfigBackendUrl(value)); +// return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); +// } + Status Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { STATUS_CHECK(CheckDBConfigArchiveDiskThreshold(value)); @@ -2340,23 +2553,17 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } -Status -Config::SetDBConfigAutoFlushInterval(const std::string& value) { - STATUS_CHECK(CheckDBConfigAutoFlushInterval(value)); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_AUTO_FLUSH_INTERVAL, value); -} - /* storage config */ Status -Config::SetStorageConfigPrimaryPath(const std::string& value) { - STATUS_CHECK(CheckStorageConfigPrimaryPath(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_PRIMARY_PATH, value); +Config::SetStorageConfigPath(const std::string& value) { + STATUS_CHECK(CheckStorageConfigPath(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_PATH, value); } Status -Config::SetStorageConfigSecondaryPath(const std::string& value) { - STATUS_CHECK(CheckStorageConfigSecondaryPath(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, value); +Config::SetStorageConfigAutoFlushInterval(const std::string& value) { + STATUS_CHECK(CheckStorageConfigAutoFlushInterval(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_AUTO_FLUSH_INTERVAL, value); } Status @@ -2448,6 +2655,13 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) { return ExecCallBacks(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } +Status +Config::SetCacheConfigPreloadCollection(const std::string& value) { + STATUS_CHECK(CheckCacheConfigPreloadCollection(value)); + std::string cor_value = value == "*" ? "\'*\'" : value; + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_PRELOAD_COLLECTION, cor_value); +} + /* engine config */ Status Config::SetEngineConfigUseBlasThreshold(const std::string& value) { @@ -2468,15 +2682,6 @@ Config::SetEngineConfigSimdType(const std::string& value) { return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_SIMD_TYPE, value); } -#ifdef MILVUS_GPU_VERSION -Status -Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { - STATUS_CHECK(CheckEngineConfigGpuSearchThreshold(value)); - STATUS_CHECK(SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value)); - return ExecCallBacks(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); -} -#endif - /* gpu resource config */ #ifdef MILVUS_GPU_VERSION Status @@ -2499,6 +2704,13 @@ Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); } +Status +Config::SetGpuResourceConfigGpuSearchThreshold(const std::string& value) { + STATUS_CHECK(CheckGpuResourceConfigGpuSearchThreshold(value)); + STATUS_CHECK(SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, value)); + return ExecCallBacks(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, value); +} + Status Config::SetGpuResourceConfigSearchResources(const std::string& value) { std::vector res_vec; @@ -2552,39 +2764,15 @@ Config::SetWalConfigWalPath(const std::string& value) { /* logs config */ Status -Config::SetLogsTraceEnable(const std::string& value) { - STATUS_CHECK(CheckLogsTraceEnable(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_TRACE_ENABLE, value); -} - -Status -Config::SetLogsDebugEnable(const std::string& value) { - STATUS_CHECK(CheckLogsDebugEnable(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_DEBUG_ENABLE, value); -} - -Status -Config::SetLogsInfoEnable(const std::string& value) { - STATUS_CHECK(CheckLogsInfoEnable(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_INFO_ENABLE, value); +Config::SetLogsLevel(const std::string& value) { + STATUS_CHECK(CheckLogsLevel(value)); + return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LEVEL, value); } Status -Config::SetLogsWarningEnable(const std::string& value) { - STATUS_CHECK(CheckLogsWarningEnable(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_WARNING_ENABLE, value); -} - -Status -Config::SetLogsErrorEnable(const std::string& value) { - STATUS_CHECK(CheckLogsErrorEnable(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_ERROR_ENABLE, value); -} - -Status -Config::SetLogsFatalEnable(const std::string& value) { - STATUS_CHECK(CheckLogsFatalEnable(value)); - return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_FATAL_ENABLE, value); +Config::SetLogsTraceEnable(const std::string& value) { + STATUS_CHECK(CheckLogsTraceEnable(value)); + return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_TRACE_ENABLE, value); } Status diff --git a/core/src/config/Config.h b/core/src/config/Config.h index 714a2216..a39d0065 100644 --- a/core/src/config/Config.h +++ b/core/src/config/Config.h @@ -28,55 +28,49 @@ using ConfigCallBackF = std::function; extern const char* CONFIG_NODE_DELIMITER; extern const char* CONFIG_VERSION; -/* server config */ -extern const char* CONFIG_SERVER; -extern const char* CONFIG_SERVER_ADDRESS; -extern const char* CONFIG_SERVER_ADDRESS_DEFAULT; -extern const char* CONFIG_SERVER_PORT; -extern const char* CONFIG_SERVER_PORT_DEFAULT; -extern const char* CONFIG_SERVER_DEPLOY_MODE; -extern const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT; -extern const char* CONFIG_SERVER_TIME_ZONE; -extern const char* CONFIG_SERVER_TIME_ZONE_DEFAULT; -extern const char* CONFIG_SERVER_WEB_ENABLE; -extern const char* CONFIG_SERVER_WEB_ENABLE_DEFAULT; -extern const char* CONFIG_SERVER_WEB_PORT; -extern const char* CONFIG_SERVER_WEB_PORT_DEFAULT; +/* cluster config */ +extern const char* CONFIG_CLUSTER; +extern const char* CONFIG_CLUSTER_ENABLE; +extern const char* CONFIG_CLUSTER_ENABLE_DEFAULT; +extern const char* CONFIG_CLUSTER_ROLE; +extern const char* CONFIG_CLUSTER_ROLE_DEFAULT; + +/* general config */ +extern const char* CONFIG_GENERAL; +extern const char* CONFIG_GENERAL_TIMEZONE; +extern const char* CONFIG_GENERAL_TIMEZONE_DEFAULT; +extern const char* CONFIG_GENERAL_METAURI; +extern const char* CONFIG_GENERAL_METAURI_DEFAULT; + +/* network config */ +extern const char* CONFIG_NETWORK; +extern const char* CONFIG_NETWORK_BIND_ADDRESS; +extern const char* CONFIG_NETWORK_BIND_ADDRESS_DEFAULT; +extern const char* CONFIG_NETWORK_BIND_PORT; +extern const char* CONFIG_NETWORK_BIND_PORT_DEFAULT; +extern const char* CONFIG_NETWORK_HTTP_ENABLE; +extern const char* CONFIG_NETWORK_HTTP_ENABLE_DEFAULT; +extern const char* CONFIG_NETWORK_HTTP_PORT; +extern const char* CONFIG_NETWORK_HTTP_PORT_DEFAULT; /* db config */ extern const char* CONFIG_DB; -extern const char* CONFIG_DB_BACKEND_URL; -extern const char* CONFIG_DB_BACKEND_URL_DEFAULT; +// extern const char* CONFIG_DB_BACKEND_URL; +// extern const char* CONFIG_DB_BACKEND_URL_DEFAULT; extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD; extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT; extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD; extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT; -extern const char* CONFIG_DB_PRELOAD_COLLECTION; -extern const char* CONFIG_DB_PRELOAD_COLLECTION_DEFAULT; -extern const char* CONFIG_DB_AUTO_FLUSH_INTERVAL; -extern const char* CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT; /* storage config */ extern const char* CONFIG_STORAGE; -extern const char* CONFIG_STORAGE_PRIMARY_PATH; -extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT; -extern const char* CONFIG_STORAGE_SECONDARY_PATH; -extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT; +extern const char* CONFIG_STORAGE_PATH; +extern const char* CONFIG_STORAGE_PATH_DEFAULT; +extern const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL; +extern const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL_DEFAULT; extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT; extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN; extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX; -// extern const char* CONFIG_STORAGE_S3_ENABLE; -// extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT; -// extern const char* CONFIG_STORAGE_S3_ADDRESS; -// extern const char* CONFIG_STORAGE_S3_ADDRESS_DEFAULT; -// extern const char* CONFIG_STORAGE_S3_PORT; -// extern const char* CONFIG_STORAGE_S3_PORT_DEFAULT; -// extern const char* CONFIG_STORAGE_S3_ACCESS_KEY; -// extern const char* CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT; -// extern const char* CONFIG_STORAGE_S3_SECRET_KEY; -// extern const char* CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT; -// extern const char* CONFIG_STORAGE_S3_BUCKET; -// extern const char* CONFIG_STORAGE_S3_BUCKET_DEFAULT; /* cache config */ extern const char* CONFIG_CACHE; @@ -88,6 +82,8 @@ extern const char* CONFIG_CACHE_INSERT_BUFFER_SIZE; extern const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT; extern const char* CONFIG_CACHE_CACHE_INSERT_DATA; extern const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT; +extern const char* CONFIG_CACHE_PRELOAD_COLLECTION; +extern const char* CONFIG_CACHE_PRELOAD_COLLECTION_DEFAULT; /* metric config */ extern const char* CONFIG_METRIC; @@ -106,8 +102,6 @@ extern const char* CONFIG_ENGINE_OMP_THREAD_NUM; extern const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT; extern const char* CONFIG_ENGINE_SIMD_TYPE; extern const char* CONFIG_ENGINE_SIMD_TYPE_DEFAULT; -extern const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD; -extern const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT; /* gpu resource config */ extern const char* CONFIG_GPU_RESOURCE; @@ -117,6 +111,8 @@ extern const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY; extern const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT; extern const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD; extern const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT; +extern const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD; +extern const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD_DEFAULT; extern const char* CONFIG_GPU_RESOURCE_DELIMITER; extern const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES; extern const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT; @@ -142,18 +138,10 @@ extern const char* CONFIG_WAL_WAL_PATH_DEFAULT; /* logs config */ extern const char* CONFIG_LOGS; +extern const char* CONFIG_LOGS_LEVEL; +extern const char* CONFIG_LOGS_LEVEL_DEFAULT; extern const char* CONFIG_LOGS_TRACE_ENABLE; extern const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT; -extern const char* CONFIG_LOGS_DEBUG_ENABLE; -extern const char* CONFIG_LOGS_DEBUG_ENABLE_DEFAULT; -extern const char* CONFIG_LOGS_INFO_ENABLE; -extern const char* CONFIG_LOGS_INFO_ENABLE_DEFAULT; -extern const char* CONFIG_LOGS_WARNING_ENABLE; -extern const char* CONFIG_LOGS_WARNING_ENABLE_DEFAULT; -extern const char* CONFIG_LOGS_ERROR_ENABLE; -extern const char* CONFIG_LOGS_ERROR_ENABLE_DEFAULT; -extern const char* CONFIG_LOGS_FATAL_ENABLE; -extern const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT; extern const char* CONFIG_LOGS_PATH; extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE; extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT; @@ -215,51 +203,43 @@ class Config { Status CheckConfigVersion(const std::string& value); - /* server config */ + /* cluster config */ Status - CheckServerConfigAddress(const std::string& value); + CheckClusterConfigEnable(const std::string& value); Status - CheckServerConfigPort(const std::string& value); + CheckClusterConfigRole(const std::string& value); + + /* general config */ Status - CheckServerConfigDeployMode(const std::string& value); + CheckGeneralConfigTimezone(const std::string& value); Status - CheckServerConfigTimeZone(const std::string& value); + CheckGeneralConfigMetaURI(const std::string& value); + + /* network config */ Status - CheckServerConfigWebEnable(const std::string& value); + CheckNetworkConfigBindAddress(const std::string& value); Status - CheckServerConfigWebPort(const std::string& value); - - /* db config */ + CheckNetworkConfigBindPort(const std::string& value); Status - CheckDBConfigBackendUrl(const std::string& value); + CheckNetworkConfigHTTPEnable(const std::string& value); Status - CheckDBConfigPreloadCollection(const std::string& value); + CheckNetworkConfigHTTPPort(const std::string& value); + + /* db config */ + // Status + // CheckDBConfigBackendUrl(const std::string& value); Status CheckDBConfigArchiveDiskThreshold(const std::string& value); Status CheckDBConfigArchiveDaysThreshold(const std::string& value); - Status - CheckDBConfigAutoFlushInterval(const std::string& value); /* storage config */ Status - CheckStorageConfigPrimaryPath(const std::string& value); + CheckStorageConfigPath(const std::string& value); Status - CheckStorageConfigSecondaryPath(const std::string& value); + CheckStorageConfigAutoFlushInterval(const std::string& value); Status CheckStorageConfigFileCleanupTimeout(const std::string& value); - // Status - // CheckStorageConfigS3Enable(const std::string& value); - // Status - // CheckStorageConfigS3Address(const std::string& value); - // Status - // CheckStorageConfigS3Port(const std::string& value); - // Status - // CheckStorageConfigS3AccessKey(const std::string& value); - // Status - // CheckStorageConfigS3SecretKey(const std::string& value); - // Status - // CheckStorageConfigS3Bucket(const std::string& value); /* metric config */ Status @@ -278,6 +258,8 @@ class Config { CheckCacheConfigInsertBufferSize(const std::string& value); Status CheckCacheConfigCacheInsertData(const std::string& value); + Status + CheckCacheConfigPreloadCollection(const std::string& value); /* engine config */ Status @@ -288,9 +270,6 @@ class Config { CheckEngineConfigSimdType(const std::string& value); #ifdef MILVUS_GPU_VERSION - Status - CheckEngineConfigGpuSearchThreshold(const std::string& value); - /* gpu resource config */ Status CheckGpuResourceConfigEnable(const std::string& value); @@ -299,6 +278,8 @@ class Config { Status CheckGpuResourceConfigCacheThreshold(const std::string& value); Status + CheckGpuResourceConfigGpuSearchThreshold(const std::string& value); + Status CheckGpuResourceConfigSearchResources(const std::vector& value); Status CheckGpuResourceConfigBuildIndexResources(const std::vector& value); @@ -320,17 +301,9 @@ class Config { /* logs config */ Status - CheckLogsTraceEnable(const std::string& value); - Status - CheckLogsDebugEnable(const std::string& value); - Status - CheckLogsInfoEnable(const std::string& value); - Status - CheckLogsWarningEnable(const std::string& value); - Status - CheckLogsErrorEnable(const std::string& value); + CheckLogsLevel(const std::string& level); Status - CheckLogsFatalEnable(const std::string& value); + CheckLogsTraceEnable(const std::string& value); Status CheckLogsPath(const std::string& value); Status @@ -350,51 +323,43 @@ class Config { ExecCallBacks(const std::string& node, const std::string& sub_node, const std::string& value); public: - /* server config */ + /* cluster config */ + Status + GetClusterConfigEnable(bool& value); + Status + GetClusterConfigRole(std::string& value); + + /* general config */ Status - GetServerConfigAddress(std::string& value); + GetGeneralConfigTimezone(std::string& value); Status - GetServerConfigPort(std::string& value); + GetGeneralConfigMetaURI(std::string& value); + + /* network config */ Status - GetServerConfigDeployMode(std::string& value); + GetNetworkConfigBindAddress(std::string& value); Status - GetServerConfigTimeZone(std::string& value); + GetNetworkConfigBindPort(std::string& value); Status - GetServerConfigWebEnable(bool& value); + GetNetworkConfigHTTPEnable(bool& value); Status - GetServerConfigWebPort(std::string& value); + GetNetworkConfigHTTPPort(std::string& value); /* db config */ - Status - GetDBConfigBackendUrl(std::string& value); + // Status + // GetDBConfigBackendUrl(std::string& value); Status GetDBConfigArchiveDiskThreshold(int64_t& value); Status GetDBConfigArchiveDaysThreshold(int64_t& value); - Status - GetDBConfigPreloadCollection(std::string& value); - Status - GetDBConfigAutoFlushInterval(int64_t& value); /* storage config */ Status - GetStorageConfigPrimaryPath(std::string& value); + GetStorageConfigPath(std::string& value); Status - GetStorageConfigSecondaryPath(std::string& value); + GetStorageConfigAutoFlushInterval(int64_t& value); Status GetStorageConfigFileCleanupTimeup(int64_t& value); - // Status - // GetStorageConfigS3Enable(bool& value); - // Status - // GetStorageConfigS3Address(std::string& value); - // Status - // GetStorageConfigS3Port(std::string& value); - // Status - // GetStorageConfigS3AccessKey(std::string& value); - // Status - // GetStorageConfigS3SecretKey(std::string& value); - // Status - // GetStorageConfigS3Bucket(std::string& value); /* metric config */ Status @@ -413,6 +378,8 @@ class Config { GetCacheConfigInsertBufferSize(int64_t& value); Status GetCacheConfigCacheInsertData(bool& value); + Status + GetCacheConfigPreloadCollection(std::string& value); /* engine config */ Status @@ -423,9 +390,6 @@ class Config { GetEngineConfigSimdType(std::string& value); #ifdef MILVUS_GPU_VERSION - Status - GetEngineConfigGpuSearchThreshold(int64_t& value); - /* gpu resource config */ Status GetGpuResourceConfigEnable(bool& value); @@ -434,6 +398,8 @@ class Config { Status GetGpuResourceConfigCacheThreshold(float& value); Status + GetGpuResourceConfigGpuSearchThreshold(int64_t& value); + Status GetGpuResourceConfigSearchResources(std::vector& value); Status GetGpuResourceConfigBuildIndexResources(std::vector& value); @@ -455,17 +421,9 @@ class Config { /* logs config */ Status - GetLogsTraceEnable(bool& value); - Status - GetLogsDebugEnable(bool& value); - Status - GetLogsInfoEnable(bool& value); + GetLogsLevel(std::string& value); Status - GetLogsWarningEnable(bool& value); - Status - GetLogsErrorEnable(bool& value); - Status - GetLogsFatalEnable(bool& value); + GetLogsTraceEnable(bool& value); Status GetLogsPath(std::string& value); Status @@ -477,51 +435,43 @@ class Config { GetServerRestartRequired(bool& required); public: - /* server config */ + /* cluster config */ Status - SetServerConfigAddress(const std::string& value); + SetClusterConfigEnable(const std::string& value); Status - SetServerConfigPort(const std::string& value); + SetClusterConfigRole(const std::string& value); + + /* general config */ Status - SetServerConfigDeployMode(const std::string& value); + SetGeneralConfigTimezone(const std::string& value); Status - SetServerConfigTimeZone(const std::string& value); + SetGeneralConfigMetaURI(const std::string& value); + + /* network config */ Status - SetServerConfigWebEnable(const std::string& value); + SetNetworkConfigBindAddress(const std::string& value); Status - SetServerConfigWebPort(const std::string& value); - - /* db config */ + SetNetworkConfigBindPort(const std::string& value); Status - SetDBConfigBackendUrl(const std::string& value); + SetNetworkConfigHTTPEnable(const std::string& value); Status - SetDBConfigPreloadCollection(const std::string& value); + SetNetworkConfigHTTPPort(const std::string& value); + + /* db config */ + // Status + // SetDBConfigBackendUrl(const std::string& value); Status SetDBConfigArchiveDiskThreshold(const std::string& value); Status SetDBConfigArchiveDaysThreshold(const std::string& value); - Status - SetDBConfigAutoFlushInterval(const std::string& value); /* storage config */ Status - SetStorageConfigPrimaryPath(const std::string& value); + SetStorageConfigPath(const std::string& value); Status - SetStorageConfigSecondaryPath(const std::string& value); + SetStorageConfigAutoFlushInterval(const std::string& value); Status SetStorageConfigFileCleanupTimeout(const std::string& value); - // Status - // SetStorageConfigS3Enable(const std::string& value); - // Status - // SetStorageConfigS3Address(const std::string& value); - // Status - // SetStorageConfigS3Port(const std::string& value); - // Status - // SetStorageConfigS3AccessKey(const std::string& value); - // Status - // SetStorageConfigS3SecretKey(const std::string& value); - // Status - // SetStorageConfigS3Bucket(const std::string& value); /* metric config */ Status @@ -540,6 +490,8 @@ class Config { SetCacheConfigInsertBufferSize(const std::string& value); Status SetCacheConfigCacheInsertData(const std::string& value); + Status + SetCacheConfigPreloadCollection(const std::string& value); /* engine config */ Status @@ -549,8 +501,6 @@ class Config { Status SetEngineConfigSimdType(const std::string& value); #ifdef MILVUS_GPU_VERSION - Status - SetEngineConfigGpuSearchThreshold(const std::string& value); /* gpu resource config */ Status @@ -560,6 +510,8 @@ class Config { Status SetGpuResourceConfigCacheThreshold(const std::string& value); Status + SetGpuResourceConfigGpuSearchThreshold(const std::string& value); + Status SetGpuResourceConfigSearchResources(const std::string& value); Status SetGpuResourceConfigBuildIndexResources(const std::string& value); @@ -581,17 +533,9 @@ class Config { /* logs config */ Status - SetLogsTraceEnable(const std::string& value); - Status - SetLogsDebugEnable(const std::string& value); + SetLogsLevel(const std::string& value); Status - SetLogsInfoEnable(const std::string& value); - Status - SetLogsWarningEnable(const std::string& value); - Status - SetLogsErrorEnable(const std::string& value); - Status - SetLogsFatalEnable(const std::string& value); + SetLogsTraceEnable(const std::string& value); Status SetLogsPath(const std::string& value); Status diff --git a/core/src/config/Utils.cpp b/core/src/config/Utils.cpp new file mode 100644 index 00000000..64e56fe8 --- /dev/null +++ b/core/src/config/Utils.cpp @@ -0,0 +1,85 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include "config/Utils.h" + +#include +#include +#include + +namespace milvus { +namespace server { + +std::unordered_map BYTE_UNITS = { + {"b", 1}, + {"k", 1024}, + {"m", 1024 * 1024}, + {"g", 1024 * 1024 * 1024}, +}; + +bool +is_number(const std::string& s) { + return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end(); +} + +bool +is_alpha(const std::string& s) { + return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end(); +} + +std::string +str_tolower(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); + return s; +} + +int64_t +parse_bytes(const std::string& str, std::string& err) { + try { + std::string s = str; + if (is_number(s)) + return std::stoll(s); + if (s.length() == 0) + return 0; + + auto last_two = s.substr(s.length() - 2, 2); + auto last_one = s.substr(s.length() - 1); + if (is_alpha(last_two) && is_alpha(last_one)) + if (last_one == "b" or last_one == "B") + s = s.substr(0, s.length() - 1); + auto& units = BYTE_UNITS; + auto suffix = str_tolower(s.substr(s.length() - 1)); + + std::string digits_part; + if (is_number(suffix)) { + digits_part = s; + suffix = 'b'; + } else { + digits_part = s.substr(0, s.length() - 1); + } + + if (units.find(suffix) != units.end() or is_number(suffix)) { + auto digits = std::stoll(digits_part); + return digits * units[suffix]; + } else { + std::stringstream ss; + ss << "The specified value for memory (" << str << ") should specify the units." + << "The postfix should be one of the `b` `k` `m` `g` characters"; + err = ss.str(); + } + } catch (...) { + err = "Unknown error happened on parse bytes."; + } + return 0; +} + +} // namespace server +} // namespace milvus diff --git a/core/src/config/Utils.h b/core/src/config/Utils.h new file mode 100644 index 00000000..7a39417b --- /dev/null +++ b/core/src/config/Utils.h @@ -0,0 +1,23 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#pragma once + +#include + +namespace milvus { +namespace server { + +int64_t +parse_bytes(const std::string& str, std::string& err); + +} // namespace server +} // namespace milvus diff --git a/core/src/config/handler/GpuResourceConfigHandler.cpp b/core/src/config/handler/GpuResourceConfigHandler.cpp index ae844ab9..08d98f8d 100644 --- a/core/src/config/handler/GpuResourceConfigHandler.cpp +++ b/core/src/config/handler/GpuResourceConfigHandler.cpp @@ -94,19 +94,20 @@ GpuResourceConfigHandler::AddGpuSearchThresholdListener() { ConfigCallBackF lambda_gpu_threshold = [this](const std::string& value) -> Status { if (!gpu_enable_) { std::string msg = - std::string("GPU resources is disable. Cannot set config ") + CONFIG_ENGINE_GPU_SEARCH_THRESHOLD; + std::string("GPU resources is disabled. Cannot set config ") + CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD; return Status(SERVER_UNEXPECTED_ERROR, msg); } auto& config = Config::GetInstance(); - auto status = config.GetEngineConfigGpuSearchThreshold(threshold_); + auto status = config.GetGpuResourceConfigGpuSearchThreshold(threshold_); if (status.ok()) { OnGpuSearchThresholdChanged(threshold_); } return status; }; - config.RegisterCallBack(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_, lambda_gpu_threshold); + config.RegisterCallBack(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, identity_, + lambda_gpu_threshold); } void @@ -152,7 +153,7 @@ GpuResourceConfigHandler::RemoveGpuBuildResourcesListener() { void GpuResourceConfigHandler::RemoveGpuSearchThresholdListener() { auto& config = Config::GetInstance(); - config.CancelCallBack(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_); + config.CancelCallBack(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, identity_); } void diff --git a/core/src/db/meta/MetaFactory.cpp b/core/src/db/meta/MetaFactory.cpp index 03f7c434..c4cf585d 100644 --- a/core/src/db/meta/MetaFactory.cpp +++ b/core/src/db/meta/MetaFactory.cpp @@ -48,6 +48,7 @@ MetaFactory::Build(const DBMetaOptions& meta_options, const int& mode) { std::string uri = meta_options.backend_uri_; utils::MetaUriInfo uri_info; + LOG_ENGINE_DEBUG_ << "MetaUri: " << uri << std::endl; auto status = utils::ParseMetaUri(uri, uri_info); if (!status.ok()) { LOG_ENGINE_ERROR_ << "Wrong URI format: URI = " << uri; diff --git a/core/src/db/wal/WalDefinations.h b/core/src/db/wal/WalDefinations.h index 6491a02f..6808e902 100644 --- a/core/src/db/wal/WalDefinations.h +++ b/core/src/db/wal/WalDefinations.h @@ -27,6 +27,7 @@ using TableSchemaPtr = std::shared_ptr; using TableMetaPtr = std::shared_ptr>; #define UNIT_MB (1024 * 1024) +#define UNIT_B 1 #define LSN_OFFSET_MASK 0x00000000ffffffff enum class MXLogType { None, InsertBinary, InsertVector, Delete, Update, Flush, Entity }; diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index dda5e1fc..22452e86 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -33,7 +33,7 @@ PrometheusMetrics::Init() { // Following should be read from config file. std::string server_port, push_port, push_address; - STATUS_CHECK(config.GetServerConfigPort(server_port)); + STATUS_CHECK(config.GetNetworkConfigBindPort(server_port)); STATUS_CHECK(config.GetMetricConfigPort(push_port)); STATUS_CHECK(config.GetMetricConfigAddress(push_address)); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 5628bb27..eaf180e5 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -106,7 +106,7 @@ class OptimizerInst { int64_t gpu_search_threshold; config.GetGpuResourceConfigBuildIndexResources(build_gpus); config.GetGpuResourceConfigSearchResources(search_gpus); - config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold); + config.GetGpuResourceConfigGpuSearchThreshold(gpu_search_threshold); std::string build_msg = "Build index gpu:"; for (auto build_id : build_gpus) { build_msg.append(" gpu" + std::to_string(build_id)); diff --git a/core/src/scheduler/selector/FaissFlatPass.cpp b/core/src/scheduler/selector/FaissFlatPass.cpp index 243d8f35..22523880 100644 --- a/core/src/scheduler/selector/FaissFlatPass.cpp +++ b/core/src/scheduler/selector/FaissFlatPass.cpp @@ -25,7 +25,7 @@ namespace scheduler { void FaissFlatPass::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } diff --git a/core/src/scheduler/selector/FaissIVFFlatPass.cpp b/core/src/scheduler/selector/FaissIVFFlatPass.cpp index 79885405..33670da9 100644 --- a/core/src/scheduler/selector/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/selector/FaissIVFFlatPass.cpp @@ -25,7 +25,7 @@ void FaissIVFFlatPass::Init() { #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } diff --git a/core/src/scheduler/selector/FaissIVFPQPass.cpp b/core/src/scheduler/selector/FaissIVFPQPass.cpp index 21c45931..c91484ce 100644 --- a/core/src/scheduler/selector/FaissIVFPQPass.cpp +++ b/core/src/scheduler/selector/FaissIVFPQPass.cpp @@ -27,7 +27,7 @@ void FaissIVFPQPass::Init() { #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } diff --git a/core/src/scheduler/selector/FaissIVFSQ8HPass.cpp b/core/src/scheduler/selector/FaissIVFSQ8HPass.cpp index 400509e6..66a45cff 100644 --- a/core/src/scheduler/selector/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/selector/FaissIVFSQ8HPass.cpp @@ -25,7 +25,7 @@ namespace scheduler { void FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } diff --git a/core/src/scheduler/selector/FaissIVFSQ8Pass.cpp b/core/src/scheduler/selector/FaissIVFSQ8Pass.cpp index d71a10e2..056ae012 100644 --- a/core/src/scheduler/selector/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/selector/FaissIVFSQ8Pass.cpp @@ -25,7 +25,7 @@ void FaissIVFSQ8Pass::Init() { #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 4c44ed3d..adc3acbf 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -34,36 +34,26 @@ DBWrapper::StartService() { // db config engine::DBOptions opt; - s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); - if (!s.ok()) { - std::cerr << s.ToString() << std::endl; - return s; - } - - s = config.GetDBConfigAutoFlushInterval(opt.auto_flush_interval_); + s = config.GetGeneralConfigMetaURI(opt.meta_.backend_uri_); if (!s.ok()) { std::cerr << s.ToString() << std::endl; return s; } std::string path; - s = config.GetStorageConfigPrimaryPath(path); + s = config.GetStorageConfigPath(path); if (!s.ok()) { std::cerr << s.ToString() << std::endl; return s; } - opt.meta_.path_ = path + "/db"; - std::string db_slave_path; - s = config.GetStorageConfigSecondaryPath(db_slave_path); + s = config.GetStorageConfigAutoFlushInterval(opt.auto_flush_interval_); if (!s.ok()) { std::cerr << s.ToString() << std::endl; return s; } - StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); - s = config.GetStorageConfigFileCleanupTimeup(opt.file_cleanup_timeout_); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -90,8 +80,25 @@ DBWrapper::StartService() { std::cerr << s.ToString() << std::endl; return s; } - opt.insert_buffer_size_ = insert_buffer_size * engine::GB; + opt.insert_buffer_size_ = insert_buffer_size; + +#if 1 + bool cluster_enable = false; + std::string cluster_role; + STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); + STATUS_CHECK(config.GetClusterConfigRole(cluster_role)); + if (not cluster_enable) { + opt.mode_ = engine::DBOptions::MODE::SINGLE; + } else if (cluster_role == "ro") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY; + } else if (cluster_role == "rw") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; + } else { + std::cerr << "Error: cluster.role is not one of rw and ro." << std::endl; + kill(0, SIGUSR1); + } +#else std::string mode; s = config.GetServerConfigDeployMode(mode); if (!s.ok()) { @@ -110,6 +117,7 @@ DBWrapper::StartService() { << "single, cluster_readonly, and cluster_writable." << std::endl; kill(0, SIGUSR1); } +#endif // get wal configurations s = config.GetWalConfigEnable(opt.wal_enable_); @@ -127,12 +135,15 @@ DBWrapper::StartService() { kill(0, SIGUSR1); } - s = config.GetWalConfigBufferSize(opt.buffer_size_); + int64_t wal_buffer_size = 0; + s = config.GetWalConfigBufferSize(wal_buffer_size); if (!s.ok()) { std::cerr << "ERROR! Failed to get buffer_size configuration." << std::endl; std::cerr << s.ToString() << std::endl; kill(0, SIGUSR1); } + wal_buffer_size /= (1024 * 1024); + opt.buffer_size_ = wal_buffer_size; s = config.GetWalConfigWalPath(opt.mxlog_path_); if (!s.ok()) { @@ -227,7 +238,7 @@ DBWrapper::StartService() { // preload collection std::string preload_collections; - s = config.GetDBConfigPreloadCollection(preload_collections); + s = config.GetCacheConfigPreloadCollection(preload_collections); if (!s.ok()) { std::cerr << s.ToString() << std::endl; return s; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 239be363..cdbdb088 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "config/Config.h" #include "index/archive/KnowhereResource.h" @@ -154,7 +155,7 @@ Server::Start() { Config& config = Config::GetInstance(); std::string meta_uri; - STATUS_CHECK(config.GetDBConfigBackendUrl(meta_uri)); + STATUS_CHECK(config.GetGeneralConfigMetaURI(meta_uri)); if (meta_uri.length() > 6 && strcasecmp("sqlite", meta_uri.substr(0, 6).c_str()) == 0) { std::cout << "WARNNING: You are using SQLite as the meta data management, " "which can't be used in production. Please change it to MySQL!" @@ -169,7 +170,7 @@ Server::Start() { /* log path is defined in Config file, so InitLog must be called after LoadConfig */ std::string time_zone; - s = config.GetServerConfigTimeZone(time_zone); + s = config.GetGeneralConfigTimezone(time_zone); if (!s.ok()) { std::cerr << "Fail to get server config timezone" << std::endl; return s; @@ -194,6 +195,11 @@ Server::Start() { tzset(); { + std::unordered_map level_to_int{ + {"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1}, + }; + + std::string level; bool trace_enable = false; bool debug_enable = false; bool info_enable = false; @@ -203,12 +209,25 @@ Server::Start() { std::string logs_path; int64_t max_log_file_size = 0; int64_t delete_exceeds = 0; + + STATUS_CHECK(config.GetLogsLevel(level)); + switch (level_to_int[level]) { + case 5: + debug_enable = true; + case 4: + info_enable = true; + case 3: + warning_enable = true; + case 2: + error_enable = true; + case 1: + fatal_enable = true; + break; + default: + return Status(SERVER_UNEXPECTED_ERROR, "invalid log level"); + } + STATUS_CHECK(config.GetLogsTraceEnable(trace_enable)); - STATUS_CHECK(config.GetLogsDebugEnable(debug_enable)); - STATUS_CHECK(config.GetLogsInfoEnable(info_enable)); - STATUS_CHECK(config.GetLogsWarningEnable(warning_enable)); - STATUS_CHECK(config.GetLogsErrorEnable(error_enable)); - STATUS_CHECK(config.GetLogsFatalEnable(fatal_enable)); STATUS_CHECK(config.GetLogsPath(logs_path)); STATUS_CHECK(config.GetLogsMaxLogFileSize(max_log_file_size)); STATUS_CHECK(config.GetLogsLogRotateNum(delete_exceeds)); @@ -216,12 +235,18 @@ Server::Start() { max_log_file_size, delete_exceeds); } - std::string deploy_mode; - STATUS_CHECK(config.GetServerConfigDeployMode(deploy_mode)); + bool cluster_enable = false; + std::string cluster_role; + STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); + STATUS_CHECK(config.GetClusterConfigRole(cluster_role)); + + // std::string deploy_mode; + // STATUS_CHECK(config.GetServerConfigDeployMode(deploy_mode)); - if (deploy_mode == "single" || deploy_mode == "cluster_writable") { + // if (deploy_mode == "single" || deploy_mode == "cluster_writable") { + if ((not cluster_enable) || cluster_role == "rw") { std::string db_path; - STATUS_CHECK(config.GetStorageConfigPrimaryPath(db_path)); + STATUS_CHECK(config.GetStorageConfigPath(db_path)); try { // True if a new directory was created, otherwise false. @@ -232,7 +257,11 @@ Server::Start() { s = InstanceLockCheck::Check(db_path); if (!s.ok()) { - std::cerr << "deploy_mode: " << deploy_mode << " instance lock db path failed." << std::endl; + if (not cluster_enable) { + std::cerr << "single instance lock db path failed." << s.message() << std::endl; + } else { + std::cerr << cluster_role << " instance lock db path failed." << s.message() << std::endl; + } return s; } @@ -251,7 +280,11 @@ Server::Start() { } s = InstanceLockCheck::Check(wal_path); if (!s.ok()) { - std::cerr << "deploy_mode: " << deploy_mode << " instance lock wal path failed." << std::endl; + if (not cluster_enable) { + std::cerr << "single instance lock wal path failed." << s.message() << std::endl; + } else { + std::cerr << cluster_role << " instance lock wal path failed." << s.message() << std::endl; + } return s; } } diff --git a/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp b/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp index de34e331..f871ef5b 100644 --- a/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp +++ b/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp @@ -37,6 +37,17 @@ Status ReLoadSegmentsRequest::OnExecute() { auto& config = Config::GetInstance(); +#if 1 + bool cluster_enable = false; + std::string cluster_role; + STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); + STATUS_CHECK(config.GetClusterConfigRole(cluster_role)); + + if ((not cluster_enable) || cluster_role == "rw") { + // TODO: No need to reload segment files + return Status(SERVER_SUCCESS, ""); + } +#else std::string deploy_mode; auto status = config.GetServerConfigDeployMode(deploy_mode); if (!status.ok()) { @@ -48,6 +59,7 @@ ReLoadSegmentsRequest::OnExecute() { // TODO: No need to reload segment files return Status(SERVER_SUCCESS, ""); } +#endif try { std::string hdr = "ReloadSegmentsRequest(collection=" + collection_name_ + ")"; diff --git a/core/src/server/grpc_impl/GrpcServer.cpp b/core/src/server/grpc_impl/GrpcServer.cpp index 4cc62fe3..81f1be47 100644 --- a/core/src/server/grpc_impl/GrpcServer.cpp +++ b/core/src/server/grpc_impl/GrpcServer.cpp @@ -77,8 +77,8 @@ GrpcServer::StartService() { Config& config = Config::GetInstance(); std::string address, port; - STATUS_CHECK(config.GetServerConfigAddress(address)); - STATUS_CHECK(config.GetServerConfigPort(port)); + STATUS_CHECK(config.GetNetworkConfigBindAddress(address)); + STATUS_CHECK(config.GetNetworkConfigBindPort(port)); std::string server_address(address + ":" + port); diff --git a/core/src/server/init/StorageChecker.cpp b/core/src/server/init/StorageChecker.cpp index 5f398f8a..431aba50 100644 --- a/core/src/server/init/StorageChecker.cpp +++ b/core/src/server/init/StorageChecker.cpp @@ -44,6 +44,16 @@ StorageChecker::CheckStoragePermission() { return Status(SERVER_UNEXPECTED_ERROR, err_msg); } +#if 1 + bool cluster_enable = false; + std::string cluster_role; + STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); + STATUS_CHECK(config.GetClusterConfigRole(cluster_role)); + + if (cluster_enable && cluster_role == "ro") { + return Status::OK(); + } +#else std::string deploy_mode; status = config.GetServerConfigDeployMode(deploy_mode); if (!status.ok()) { @@ -53,10 +63,11 @@ StorageChecker::CheckStoragePermission() { if (deploy_mode == "cluster_readonly") { return Status::OK(); } +#endif /* Check db directory write permission */ std::string primary_path; - status = config.GetStorageConfigPrimaryPath(primary_path); + status = config.GetStorageConfigPath(primary_path); if (!status.ok()) { return status; } @@ -64,35 +75,13 @@ StorageChecker::CheckStoragePermission() { ret = access(primary_path.c_str(), F_OK | R_OK | W_OK); fiu_do_on("StorageChecker.CheckStoragePermission.db_primary_path_access_fail", ret = -1); if (0 != ret) { - std::string err_msg = " Access DB storage primary path " + primary_path + " fail. " + strerror(errno) + + std::string err_msg = " Access DB storage path " + primary_path + " fail. " + strerror(errno) + "(code: " + std::to_string(errno) + ")"; LOG_SERVER_FATAL_ << err_msg; std::cerr << err_msg << std::endl; return Status(SERVER_UNEXPECTED_ERROR, err_msg); } - std::string secondary_paths; - status = config.GetStorageConfigSecondaryPath(secondary_paths); - if (!status.ok()) { - return status; - } - - if (!secondary_paths.empty()) { - std::vector secondary_path_vector; - StringHelpFunctions::SplitStringByDelimeter(secondary_paths, ",", secondary_path_vector); - for (auto& path : secondary_path_vector) { - ret = access(path.c_str(), F_OK | R_OK | W_OK); - fiu_do_on("StorageChecker.CheckStoragePermission.db_secondary_path_access_fail", ret = -1); - if (0 != ret) { - std::string err_msg = " Access DB storage secondary path " + path + " fail. " + strerror(errno) + - "(code: " + std::to_string(errno) + ")"; - LOG_SERVER_FATAL_ << err_msg; - std::cerr << err_msg << std::endl; - return Status(SERVER_UNEXPECTED_ERROR, err_msg); - } - } - } - /* Check wal directory write permission */ bool wal_enable = false; status = config.GetWalConfigEnable(wal_enable); diff --git a/core/src/server/web_impl/WebServer.cpp b/core/src/server/web_impl/WebServer.cpp index d6bba382..40ddcdb3 100644 --- a/core/src/server/web_impl/WebServer.cpp +++ b/core/src/server/web_impl/WebServer.cpp @@ -24,7 +24,7 @@ void WebServer::Start() { auto& config = Config::GetInstance(); bool enable = true; - config.GetServerConfigWebEnable(enable); + config.GetNetworkConfigHTTPEnable(enable); if (enable && nullptr == thread_ptr_) { thread_ptr_ = std::make_shared(&WebServer::StartService, this); } @@ -47,7 +47,7 @@ WebServer::StartService() { Config& config = Config::GetInstance(); std::string port; - STATUS_CHECK(config.GetServerConfigWebPort(port)); + STATUS_CHECK(config.GetNetworkConfigHTTPPort(port)); { AppComponent components = AppComponent(std::stoi(port)); diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index 5987bc0e..6698e0d3 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -1019,7 +1019,7 @@ WebRequestHandler::GetAdvancedConfig(AdvancedConfigDto::ObjectWrapper& advanced_ advanced_config->use_blas_threshold = std::stol(reply); #ifdef MILVUS_GPU_VERSION - engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); + engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD); CommandLine(engine_cmd_string, reply); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) @@ -1077,9 +1077,10 @@ WebRequestHandler::SetAdvancedConfig(const AdvancedConfigDto::ObjectWrapper& adv } #ifdef MILVUS_GPU_VERSION - engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) + " " + - std::to_string(advanced_config->gpu_search_threshold->getValue()); - status = CommandLine(engine_cmd_string, reply); + auto gpu_cmd_prefix = "set_config " + std::string(CONFIG_GPU_RESOURCE) + "."; + auto gpu_cmd_string = gpu_cmd_prefix + std::string(CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD) + " " + + std::to_string(advanced_config->gpu_search_threshold->getValue()); + status = CommandLine(gpu_cmd_string, reply); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } diff --git a/core/src/utils/LogUtil.cpp b/core/src/utils/LogUtil.cpp index 869170b0..1491c738 100644 --- a/core/src/utils/LogUtil.cpp +++ b/core/src/utils/LogUtil.cpp @@ -199,7 +199,6 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) + "], now is " + std::to_string(max_log_file_size)); } - max_log_file_size *= 1024 * 1024; defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size)); el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); el::Helpers::installPreRollOutCallback(RolloutHandler); diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index caa4e21a..3fbd8d12 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include namespace milvus { @@ -651,6 +652,15 @@ ValidationUtil::ValidateStoragePath(const std::string& path) { return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path"); } +Status +ValidationUtil::ValidateLogLevel(const std::string& level) { + std::set supported_level{"debug", "info", "warning", "error", "fatal"}; + + return supported_level.find(level) != supported_level.end() + ? Status::OK() + : Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal."); +} + bool ValidationUtil::IsNumber(const std::string& s) { return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 39dc98d5..27a9f855 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -93,6 +93,9 @@ class ValidationUtil { static Status ValidateStoragePath(const std::string& path); + static Status + ValidateLogLevel(const std::string& level); + static bool IsNumber(const std::string& s); }; diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index 76518304..7e9f9b5a 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -42,6 +42,60 @@ INITIALIZE_EASYLOGGINGPP namespace { static const char* CONFIG_STR = + "version: 0.5\n" + "\n" + "cluster:\n" + " enable: false\n" + " role: rw\n" + "\n" + "general:\n" + " timezone: UTC+8\n" + " meta_uri: sqlite://:@:/\n" + "\n" + "network:\n" + " bind.address: 0.0.0.0\n" + " bind.port: 19530\n" + " http.enable: true\n" + " http.port: 19121\n" + "\n" + "storage:\n" + " path: /tmp/milvus\n" + " auto_flush_interval: 1\n" + "\n" + "wal:\n" + " enable: true\n" + " recovery_error_ignore: false\n" + " buffer_size: 256MB\n" + " path: /tmp/milvus/wal\n" + "\n" + "cache:\n" + " cache_size: 4GB\n" + " insert_buffer_size: 1GB\n" + " preload_collection:\n" + "\n" + "gpu:\n" + " enable: true\n" + " cache_size: 1GB\n" + " gpu_search_threshold: 1000\n" + " search_devices:\n" + " - gpu0\n" + " build_index_devices:\n" + " - gpu0\n" + "\n" + "logs:\n" + " level: debug\n" + " trace.enable: true\n" + " path: /tmp/milvus/logs\n" + " max_log_file_size: 1024MB\n" + " log_rotate_num: 0\n" + "\n" + "metric:\n" + " enable: false\n" + " address: 127.0.0.1\n" + " port: 9091\n" + "\n"; + +/*static const char* CONFIG_STR = "version: 0.4\n" "server_config:\n" " address: 0.0.0.0\n" @@ -103,6 +157,7 @@ static const char* CONFIG_STR = " max_log_file_size: 256\n" " delete_exceeds: 10\n" ""; +*/ void WriteToFile(const std::string &file_path, const char *content) { diff --git a/core/unittest/server/test_check.cpp b/core/unittest/server/test_check.cpp index c6d74a84..930a2bf0 100644 --- a/core/unittest/server/test_check.cpp +++ b/core/unittest/server/test_check.cpp @@ -37,11 +37,7 @@ class ServerCheckerTest : public testing::Test { db_primary_path = "/tmp/milvus-test/db"; boost::filesystem::create_directories(db_primary_path); - config.SetStorageConfigPrimaryPath(db_primary_path); - - db_secondary_path = "/tmp/milvus-test/db-secondary"; - boost::filesystem::create_directories(db_secondary_path); - config.SetStorageConfigSecondaryPath(db_secondary_path); + config.SetStorageConfigPath(db_primary_path); wal_path = "/tmp/milvus-test/wal"; boost::filesystem::create_directories(wal_path); @@ -77,15 +73,6 @@ TEST_F(ServerCheckerTest, STORAGE_FAIL_TEST) { ASSERT_FALSE(ms::StorageChecker::CheckStoragePermission().ok()); fiu_disable("StorageChecker.CheckStoragePermission.db_primary_path_access_fail"); - auto& config = ms::Config::GetInstance(); - std::string storage_secondary_path; - ASSERT_TRUE(config.GetStorageConfigSecondaryPath(storage_secondary_path).ok()); - ASSERT_TRUE(config.SetStorageConfigSecondaryPath("/tmp/milvus-test01,/tmp/milvus-test02").ok()); - fiu_enable("StorageChecker.CheckStoragePermission.db_secondary_path_access_fail", 1, NULL, 0); - ASSERT_FALSE(ms::StorageChecker::CheckStoragePermission().ok()); - fiu_disable("StorageChecker.CheckStoragePermission.db_secondary_path_access_fail"); - ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok()); - fiu_enable("StorageChecker.CheckStoragePermission.wal_path_access_fail", 1, NULL, 0); ASSERT_FALSE(ms::StorageChecker::CheckStoragePermission().ok()); fiu_disable("StorageChecker.CheckStoragePermission.wal_path_access_fail"); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 698f329d..3e3c5e17 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -51,7 +51,7 @@ TEST_F(ConfigTest, CONFIG_TEST) { config_mgr->DumpString(); milvus::server::ConfigNode& root_config = config_mgr->GetRootNode(); - milvus::server::ConfigNode& server_config = root_config.GetChild("server_config"); + milvus::server::ConfigNode& server_config = root_config.GetChild("network"); milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config"); const auto& im_config_mgr = *static_cast(config_mgr); @@ -74,9 +74,9 @@ TEST_F(ConfigTest, CONFIG_TEST) { bool not_exit_bool = server_config.GetBoolValue("not_exit", false); ASSERT_FALSE(not_exit_bool); - std::string address = server_config.GetValue("address"); + std::string address = server_config.GetValue("bind.address"); ASSERT_TRUE(!address.empty()); - int64_t port = server_config.GetInt64Value("port"); + int64_t port = server_config.GetInt64Value("bind.port"); ASSERT_NE(port, 0); server_config.SetValue("float_test", "2.5"); @@ -138,34 +138,34 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { /* server config */ std::string server_addr = "192.168.1.155"; - ASSERT_TRUE(config.SetServerConfigAddress(server_addr).ok()); - ASSERT_TRUE(config.GetServerConfigAddress(str_val).ok()); + ASSERT_TRUE(config.SetNetworkConfigBindAddress(server_addr).ok()); + ASSERT_TRUE(config.GetNetworkConfigBindAddress(str_val).ok()); ASSERT_TRUE(str_val == server_addr); std::string server_port = "12345"; - ASSERT_TRUE(config.SetServerConfigPort(server_port).ok()); - ASSERT_TRUE(config.GetServerConfigPort(str_val).ok()); + ASSERT_TRUE(config.SetNetworkConfigBindPort(server_port).ok()); + ASSERT_TRUE(config.GetNetworkConfigBindPort(str_val).ok()); ASSERT_TRUE(str_val == server_port); std::string web_port = "19999"; - ASSERT_TRUE(config.SetServerConfigWebPort(web_port).ok()); - ASSERT_TRUE(config.GetServerConfigWebPort(str_val).ok()); + ASSERT_TRUE(config.SetNetworkConfigHTTPPort(web_port).ok()); + ASSERT_TRUE(config.GetNetworkConfigHTTPPort(str_val).ok()); ASSERT_TRUE(str_val == web_port); - std::string server_mode = "cluster_readonly"; - ASSERT_TRUE(config.SetServerConfigDeployMode(server_mode).ok()); - ASSERT_TRUE(config.GetServerConfigDeployMode(str_val).ok()); + std::string server_mode = "ro"; + ASSERT_TRUE(config.SetClusterConfigRole(server_mode).ok()); + ASSERT_TRUE(config.GetClusterConfigRole(str_val).ok()); ASSERT_TRUE(str_val == server_mode); std::string server_time_zone = "UTC+6"; - ASSERT_TRUE(config.SetServerConfigTimeZone(server_time_zone).ok()); - ASSERT_TRUE(config.GetServerConfigTimeZone(str_val).ok()); + ASSERT_TRUE(config.SetGeneralConfigTimezone(server_time_zone).ok()); + ASSERT_TRUE(config.GetGeneralConfigTimezone(str_val).ok()); ASSERT_TRUE(str_val == server_time_zone); /* db config */ std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus"; - ASSERT_TRUE(config.SetDBConfigBackendUrl(db_backend_url).ok()); - ASSERT_TRUE(config.GetDBConfigBackendUrl(str_val).ok()); + ASSERT_TRUE(config.SetGeneralConfigMetaURI(db_backend_url).ok()); + ASSERT_TRUE(config.GetGeneralConfigMetaURI(str_val).ok()); ASSERT_TRUE(str_val == db_backend_url); int64_t db_archive_disk_threshold = 100; @@ -179,31 +179,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(int64_val == db_archive_days_threshold); int64_t db_auto_flush_interval = 1; - ASSERT_TRUE(config.SetDBConfigAutoFlushInterval(std::to_string(db_auto_flush_interval)).ok()); - ASSERT_TRUE(config.GetDBConfigAutoFlushInterval(int64_val).ok()); + ASSERT_TRUE(config.SetStorageConfigAutoFlushInterval(std::to_string(db_auto_flush_interval)).ok()); + ASSERT_TRUE(config.GetStorageConfigAutoFlushInterval(int64_val).ok()); ASSERT_TRUE(int64_val == db_auto_flush_interval); /* storage config */ std::string storage_primary_path = "/home/zilliz"; - ASSERT_TRUE(config.SetStorageConfigPrimaryPath(storage_primary_path).ok()); - ASSERT_TRUE(config.GetStorageConfigPrimaryPath(str_val).ok()); + ASSERT_TRUE(config.SetStorageConfigPath(storage_primary_path).ok()); + ASSERT_TRUE(config.GetStorageConfigPath(str_val).ok()); ASSERT_TRUE(str_val == storage_primary_path); - std::string storage_secondary_path = "/home/zilliz"; - ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok()); - ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok()); - ASSERT_TRUE(str_val == storage_secondary_path); - - storage_secondary_path = "/home/zilliz,/tmp/milvus"; - ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok()); - ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok()); - ASSERT_TRUE(str_val == storage_secondary_path); - - storage_secondary_path = ""; - ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok()); - ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok()); - ASSERT_TRUE(str_val == storage_secondary_path); - // bool storage_s3_enable = true; // ASSERT_TRUE(config.SetStorageConfigS3Enable(std::to_string(storage_s3_enable)).ok()); // ASSERT_TRUE(config.GetStorageConfigS3Enable(bool_val).ok()); @@ -289,8 +274,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { #ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold = 800; - ASSERT_TRUE(config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)).ok()); - ASSERT_TRUE(config.GetEngineConfigGpuSearchThreshold(int64_val).ok()); + auto status = config.SetGpuResourceConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); + ASSERT_TRUE(status.ok()) << status.message(); + ASSERT_TRUE(config.GetGpuResourceConfigGpuSearchThreshold(int64_val).ok()); ASSERT_TRUE(int64_val == engine_gpu_search_threshold); #endif @@ -345,7 +331,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetWalConfigRecoveryErrorIgnore(bool_val).ok()); ASSERT_TRUE(bool_val == wal_recovery_ignore); - int64_t wal_buffer_size = 128; + int64_t wal_buffer_size = 128 * 1024 * 1024; // 128 M ASSERT_TRUE(config.SetWalConfigBufferSize(std::to_string(wal_buffer_size)).ok()); ASSERT_TRUE(config.GetWalConfigBufferSize(int64_val).ok()); ASSERT_TRUE(int64_val == wal_buffer_size); @@ -356,45 +342,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(str_val == wal_path); /* logs config */ + std::string logs_level = "debug"; + ASSERT_TRUE(config.SetLogsLevel(logs_level).ok()); + ASSERT_TRUE(config.GetLogsLevel(str_val).ok()); + ASSERT_TRUE(str_val == logs_level); + bool logs_trace_enable = false; ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_trace_enable)).ok()); ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok()); ASSERT_TRUE(bool_val == logs_trace_enable); - bool logs_debug_enable = false; - ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_debug_enable)).ok()); - ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok()); - ASSERT_TRUE(bool_val == logs_debug_enable); - - bool logs_info_enable = false; - ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_info_enable)).ok()); - ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok()); - ASSERT_TRUE(bool_val == logs_info_enable); - - bool logs_warning_enable = false; - ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_warning_enable)).ok()); - ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok()); - ASSERT_TRUE(bool_val == logs_warning_enable); - - bool logs_error_enable = false; - ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_error_enable)).ok()); - ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok()); - ASSERT_TRUE(bool_val == logs_error_enable); - - bool logs_fatal_enable = false; - ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_fatal_enable)).ok()); - ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok()); - ASSERT_TRUE(bool_val == logs_fatal_enable); - std::string logs_path = "/tmp/aaa/logs"; ASSERT_TRUE(config.SetLogsPath(logs_path).ok()); ASSERT_TRUE(config.GetLogsPath(str_val).ok()); ASSERT_TRUE(str_val == logs_path); - int64_t logs_max_log_file_size = 1000; - ASSERT_TRUE(config.SetLogsMaxLogFileSize(std::to_string(logs_max_log_file_size)).ok()); + std::string logs_max_log_file_size = "1000MB"; + auto s = config.SetLogsMaxLogFileSize(logs_max_log_file_size); + ASSERT_TRUE(s.ok()) << s.message(); ASSERT_TRUE(config.GetLogsMaxLogFileSize(int64_val).ok()); - ASSERT_TRUE(int64_val == logs_max_log_file_size); + ASSERT_TRUE(int64_val == 1000 * 1024 * 1024); // 1000MB int64_t logs_log_rotate_num = 100; ASSERT_TRUE(config.SetLogsLogRotateNum(std::to_string(logs_log_rotate_num)).ok()); @@ -421,7 +388,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { auto s = config.LoadConfigFile(conf_file); ASSERT_TRUE(s.ok()) << s.message(); s = config.ResetDefaultConfig(); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(s.ok()) << s.message(); std::string get_cmd, set_cmd; std::string result, dummy; @@ -431,17 +398,17 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { /* server config */ std::string server_addr = "192.168.1.155"; - get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); - set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); + get_cmd = gen_get_command(ms::CONFIG_NETWORK, ms::CONFIG_NETWORK_BIND_ADDRESS); + set_cmd = gen_set_command(ms::CONFIG_NETWORK, ms::CONFIG_NETWORK_BIND_ADDRESS, server_addr); s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(s.ok()) << s.message(); s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(s.ok()) << s.message(); /* db config */ std::string db_backend_url = "sqlite://milvus:zilliz@:/"; - get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL); - set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url); + get_cmd = gen_get_command(ms::CONFIG_GENERAL, ms::CONFIG_GENERAL_METAURI); + set_cmd = gen_set_command(ms::CONFIG_GENERAL, ms::CONFIG_GENERAL_METAURI, db_backend_url); s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); @@ -458,13 +425,23 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { /* storage config */ std::string storage_primary_path = "/tmp/milvus1"; - get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH); - set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path); + get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH); + set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH, storage_primary_path); s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); + std::string storage_auto_flush_interval = "42"; + get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_AUTO_FLUSH_INTERVAL); + set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_AUTO_FLUSH_INTERVAL, storage_auto_flush_interval); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + + /* cache config */ std::string cache_cpu_cache_capacity = "1"; get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); @@ -532,8 +509,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { #ifdef MILVUS_GPU_VERSION std::string engine_gpu_search_threshold = "800"; - get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); - set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, + engine_gpu_search_threshold); s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); @@ -620,42 +598,40 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.LoadConfigFile(config_path + "dummy.yaml").ok()); /* server config */ - ASSERT_FALSE(config.SetServerConfigAddress("0.0.0").ok()); - ASSERT_FALSE(config.SetServerConfigAddress("0.0.0.256").ok()); + ASSERT_FALSE(config.SetNetworkConfigBindAddress("0.0.0").ok()); + ASSERT_FALSE(config.SetNetworkConfigBindAddress("0.0.0.256").ok()); - ASSERT_FALSE(config.SetServerConfigPort("a").ok()); - ASSERT_FALSE(config.SetServerConfigPort("99999").ok()); + ASSERT_FALSE(config.SetNetworkConfigBindPort("a").ok()); + ASSERT_FALSE(config.SetNetworkConfigBindPort("99999").ok()); - ASSERT_FALSE(config.SetServerConfigWebPort("a").ok()); - ASSERT_FALSE(config.SetServerConfigWebPort("99999").ok()); - ASSERT_FALSE(config.SetServerConfigWebPort("-1").ok()); + ASSERT_FALSE(config.SetNetworkConfigHTTPPort("a").ok()); + ASSERT_FALSE(config.SetNetworkConfigHTTPPort("99999").ok()); + ASSERT_FALSE(config.SetNetworkConfigHTTPPort("-1").ok()); - ASSERT_FALSE(config.SetServerConfigDeployMode("cluster").ok()); + ASSERT_FALSE(config.SetClusterConfigRole("cluster").ok()); - ASSERT_FALSE(config.SetServerConfigTimeZone("GM").ok()); - ASSERT_FALSE(config.SetServerConfigTimeZone("GMT8").ok()); - ASSERT_FALSE(config.SetServerConfigTimeZone("UTCA").ok()); + ASSERT_FALSE(config.SetGeneralConfigTimezone("GM").ok()); + ASSERT_FALSE(config.SetGeneralConfigTimezone("GMT8").ok()); + ASSERT_FALSE(config.SetGeneralConfigTimezone("UTCA").ok()); /* db config */ - ASSERT_FALSE(config.SetDBConfigBackendUrl("http://www.google.com").ok()); - ASSERT_FALSE(config.SetDBConfigBackendUrl("sqlite://:@:").ok()); - ASSERT_FALSE(config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus").ok()); + ASSERT_FALSE(config.SetGeneralConfigMetaURI("http://www.google.com").ok()); + ASSERT_FALSE(config.SetGeneralConfigMetaURI("sqlite://:@:").ok()); + ASSERT_FALSE(config.SetGeneralConfigMetaURI("mysql://root:123456@127.0.0.1/milvus").ok()); ASSERT_FALSE(config.SetDBConfigArchiveDiskThreshold("0x10").ok()); ASSERT_FALSE(config.SetDBConfigArchiveDaysThreshold("0x10").ok()); - ASSERT_FALSE(config.SetDBConfigAutoFlushInterval("0.1").ok()); /* storage config */ - ASSERT_FALSE(config.SetStorageConfigPrimaryPath("").ok()); - ASSERT_FALSE(config.SetStorageConfigPrimaryPath("./milvus").ok()); - ASSERT_FALSE(config.SetStorageConfigPrimaryPath("../milvus").ok()); - ASSERT_FALSE(config.SetStorageConfigPrimaryPath("/**milvus").ok()); - ASSERT_FALSE(config.SetStorageConfigPrimaryPath("/milvus--/path").ok()); + ASSERT_FALSE(config.SetStorageConfigPath("").ok()); + ASSERT_FALSE(config.SetStorageConfigPath("./milvus").ok()); + ASSERT_FALSE(config.SetStorageConfigPath("../milvus").ok()); + ASSERT_FALSE(config.SetStorageConfigPath("/**milvus").ok()); + ASSERT_FALSE(config.SetStorageConfigPath("/milvus--/path").ok()); - ASSERT_FALSE(config.SetStorageConfigSecondaryPath("../milvus,./zilliz").ok()); - ASSERT_FALSE(config.SetStorageConfigSecondaryPath("/home/^^__^^,/zilliz").ok()); + ASSERT_FALSE(config.SetStorageConfigAutoFlushInterval("0.1").ok()); // ASSERT_FALSE(config.SetStorageConfigS3Enable("10").ok()); // @@ -680,7 +656,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { /* cache config */ ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("a").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("0").ok()); - ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("2048").ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("2048G").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("-1").ok()); ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("a").ok()); @@ -689,7 +665,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("a").ok()); ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("0").ok()); - ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("2048").ok()); + ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("2048GB").ok()); ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("-1").ok()); ASSERT_FALSE(config.SetCacheConfigCacheInsertData("N").ok()); @@ -704,7 +680,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetEngineConfigSimdType("None").ok()); #ifdef MILVUS_GPU_VERSION - ASSERT_FALSE(config.SetEngineConfigGpuSearchThreshold("-1").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigGpuSearchThreshold("-1").ok()); #endif /* gpu resource config */ @@ -712,7 +688,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetGpuResourceConfigEnable("ok").ok()); ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("a").ok()); - ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("128").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("128GB").ok()); ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("-1").ok()); ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("a").ok()); @@ -734,15 +710,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok()); /* wal config */ + ASSERT_FALSE(config.SetLogsLevel("invalid").ok()); ASSERT_FALSE(config.SetLogsTraceEnable("invalid").ok()); - ASSERT_FALSE(config.SetLogsDebugEnable("invalid").ok()); - ASSERT_FALSE(config.SetLogsInfoEnable("invalid").ok()); - ASSERT_FALSE(config.SetLogsWarningEnable("invalid").ok()); - ASSERT_FALSE(config.SetLogsErrorEnable("invalid").ok()); - ASSERT_FALSE(config.SetLogsFatalEnable("invalid").ok()); ASSERT_FALSE(config.SetLogsPath("").ok()); ASSERT_FALSE(config.SetLogsMaxLogFileSize("-1").ok()); - ASSERT_FALSE(config.SetLogsMaxLogFileSize("511").ok()); + ASSERT_FALSE(config.SetLogsMaxLogFileSize("511MB").ok()); ASSERT_FALSE(config.SetLogsLogRotateNum("-1").ok()); ASSERT_FALSE(config.SetLogsLogRotateNum("1025").ok()); } @@ -759,7 +731,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) { config.GetConfigJsonStr(config_json_str); std::cout << config_json_str << std::endl; - ASSERT_TRUE(config.ResetDefaultConfig().ok()); + auto s = config.ResetDefaultConfig(); + ASSERT_TRUE(s.ok()) << s.message(); } TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { @@ -776,41 +749,36 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { fiu_disable("check_config_version_fail"); /* server config */ - fiu_enable("check_config_address_fail", 1, NULL, 0); + fiu_enable("check_config_bind_address_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_address_fail"); + fiu_disable("check_config_bind_address_fail"); - fiu_enable("check_config_port_fail", 1, NULL, 0); + fiu_enable("check_config_bind_port_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_port_fail"); + fiu_disable("check_config_bind_port_fail"); - fiu_enable("check_config_deploy_mode_fail", 1, NULL, 0); + fiu_enable("check_config_cluster_role_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_deploy_mode_fail"); + fiu_disable("check_config_cluster_role_fail"); - fiu_enable("check_config_time_zone_fail", 1, NULL, 0); + fiu_enable("check_config_timezone_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_time_zone_fail"); + fiu_disable("check_config_timezone_fail"); /* db config */ - fiu_enable("check_config_primary_path_fail", 1, NULL, 0); - s = config.ValidateConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_config_primary_path_fail"); - - fiu_enable("check_config_secondary_path_fail", 1, NULL, 0); + fiu_enable("check_config_path_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_secondary_path_fail"); + fiu_disable("check_config_path_fail"); - fiu_enable("check_config_backend_url_fail", 1, NULL, 0); + fiu_enable("check_config_meta_uri_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_backend_url_fail"); + fiu_disable("check_config_meta_uri_fail"); fiu_enable("check_config_archive_disk_threshold_fail", 1, NULL, 0); s = config.ValidateConfig(); @@ -835,10 +803,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { fiu_disable("check_config_enable_monitor_fail"); /* cache config */ - fiu_enable("check_config_cpu_cache_capacity_fail", 1, NULL, 0); + fiu_enable("check_config_cache_size_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_cpu_cache_capacity_fail"); + fiu_disable("check_config_cache_size_fail"); fiu_enable("check_config_cpu_cache_threshold_fail", 1, NULL, 0); s = config.ValidateConfig(); @@ -877,25 +845,25 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { ASSERT_FALSE(s.ok()); fiu_disable("check_config_gpu_resource_enable_fail"); - fiu_enable("check_gpu_resource_config_cache_capacity_fail", 1, NULL, 0); + fiu_enable("check_gpu_cache_size_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_gpu_resource_config_cache_capacity_fail"); + fiu_disable("check_gpu_cache_size_fail"); fiu_enable("check_config_gpu_resource_cache_threshold_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); fiu_disable("check_config_gpu_resource_cache_threshold_fail"); - fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0); + fiu_enable("check_gpu_search_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_gpu_resource_config_search_fail"); + fiu_disable("check_gpu_search_fail"); - fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0); + fiu_enable("check_gpu_build_index_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_gpu_resource_config_build_index_fail"); + fiu_disable("check_gpu_build_index_fail"); #endif fiu_enable("get_config_json_config_path_fail", 1, NULL, 0); @@ -992,35 +960,15 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { fiu_disable("check_wal_path_fail"); /* logs config */ - fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0); + fiu_enable("check_logs_level_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_trace_enable_fail"); + fiu_disable("check_logs_level_fail"); - fiu_enable("check_logs_debug_enable_fail", 1, NULL, 0); - s = config.ValidateConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_debug_enable_fail"); - - fiu_enable("check_logs_info_enable_fail", 1, NULL, 0); - s = config.ValidateConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_info_enable_fail"); - - fiu_enable("check_logs_warning_enable_fail", 1, NULL, 0); - s = config.ValidateConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_warning_enable_fail"); - - fiu_enable("check_logs_error_enable_fail", 1, NULL, 0); - s = config.ValidateConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_error_enable_fail"); - - fiu_enable("check_logs_fatal_enable_fail", 1, NULL, 0); + fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0); s = config.ValidateConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_fatal_enable_fail"); + fiu_disable("check_logs_trace_enable_fail"); fiu_enable("check_logs_path_fail", 1, NULL, 0); s = config.ValidateConfig(); @@ -1050,41 +998,36 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { ASSERT_TRUE(s.ok()); /* server config */ - fiu_enable("check_config_address_fail", 1, NULL, 0); + fiu_enable("check_config_bind_address_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_address_fail"); + fiu_disable("check_config_bind_address_fail"); - fiu_enable("check_config_port_fail", 1, NULL, 0); + fiu_enable("check_config_bind_port_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_port_fail"); + fiu_disable("check_config_bind_port_fail"); - fiu_enable("check_config_deploy_mode_fail", 1, NULL, 0); + fiu_enable("check_config_cluster_role_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_deploy_mode_fail"); + fiu_disable("check_config_cluster_role_fail"); - fiu_enable("check_config_time_zone_fail", 1, NULL, 0); + fiu_enable("check_config_timezone_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_time_zone_fail"); + fiu_disable("check_config_timezone_fail"); /* db config */ - fiu_enable("check_config_primary_path_fail", 1, NULL, 0); - s = config.ResetDefaultConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_config_primary_path_fail"); - - fiu_enable("check_config_secondary_path_fail", 1, NULL, 0); + fiu_enable("check_config_path_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_secondary_path_fail"); + fiu_disable("check_config_path_fail"); - fiu_enable("check_config_backend_url_fail", 1, NULL, 0); + fiu_enable("check_config_meta_uri_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_backend_url_fail"); + fiu_disable("check_config_meta_uri_fail"); fiu_enable("check_config_preload_collection_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); @@ -1119,10 +1062,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { fiu_disable("check_config_enable_monitor_fail"); /* cache config */ - fiu_enable("check_config_cpu_cache_capacity_fail", 1, NULL, 0); + fiu_enable("check_config_cache_size_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_config_cpu_cache_capacity_fail"); + fiu_disable("check_config_cache_size_fail"); fiu_enable("check_config_cpu_cache_threshold_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); @@ -1161,25 +1104,25 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { ASSERT_FALSE(s.ok()); fiu_disable("check_config_gpu_resource_enable_fail"); - fiu_enable("check_gpu_resource_config_cache_capacity_fail", 1, NULL, 0); + fiu_enable("check_gpu_cache_size_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_gpu_resource_config_cache_capacity_fail"); + fiu_disable("check_gpu_cache_size_fail"); fiu_enable("check_config_gpu_resource_cache_threshold_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); fiu_disable("check_config_gpu_resource_cache_threshold_fail"); - fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0); + fiu_enable("check_gpu_search_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_gpu_resource_config_search_fail"); + fiu_disable("check_gpu_search_fail"); - fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0); + fiu_enable("check_gpu_build_index_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_gpu_resource_config_build_index_fail"); + fiu_disable("check_gpu_build_index_fail"); #endif s = config.ResetDefaultConfig(); @@ -1207,35 +1150,15 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { fiu_disable("check_wal_path_fail"); /* logs config */ - fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0); + fiu_enable("check_logs_level_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_trace_enable_fail"); + fiu_disable("check_logs_level_fail"); - fiu_enable("check_logs_debug_enable_fail", 1, NULL, 0); - s = config.ResetDefaultConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_debug_enable_fail"); - - fiu_enable("check_logs_info_enable_fail", 1, NULL, 0); - s = config.ResetDefaultConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_info_enable_fail"); - - fiu_enable("check_logs_warning_enable_fail", 1, NULL, 0); - s = config.ResetDefaultConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_warning_enable_fail"); - - fiu_enable("check_logs_error_enable_fail", 1, NULL, 0); - s = config.ResetDefaultConfig(); - ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_error_enable_fail"); - - fiu_enable("check_logs_fatal_enable_fail", 1, NULL, 0); + fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); ASSERT_FALSE(s.ok()); - fiu_disable("check_logs_fatal_enable_fail"); + fiu_disable("check_logs_trace_enable_fail"); fiu_enable("check_logs_path_fail", 1, NULL, 0); s = config.ResetDefaultConfig(); @@ -1358,18 +1281,12 @@ TEST_F(ConfigTest, SERVER_CONFIG_UPDATE_TEST) { ASSERT_EQ("false", yaml_value); // test path - cmd_set = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, "/tmp/milvus_config_unittest"); + cmd_set = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH, "/tmp/milvus_config_unittest"); ASSERT_TRUE(config.ProcessConfigCli(reply_set, cmd_set).ok()); - ASSERT_TRUE(lambda(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, - ms::CONFIG_STORAGE_PRIMARY_PATH_DEFAULT, yaml_value).ok()); + ASSERT_TRUE(lambda(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH, + ms::CONFIG_STORAGE_PATH_DEFAULT, yaml_value).ok()); ASSERT_EQ("/tmp/milvus_config_unittest", yaml_value); - cmd_set = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_SECONDARY_PATH, "/home/zilliz,/home/milvus"); - ASSERT_TRUE(config.ProcessConfigCli(reply_set, cmd_set).ok()); - ASSERT_TRUE(lambda(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_SECONDARY_PATH, - ms::CONFIG_STORAGE_SECONDARY_PATH_DEFAULT, yaml_value).ok()); - ASSERT_EQ("/home/zilliz,/home/milvus", yaml_value); - #ifdef MILVUS_GPU_VERSION cmd_set = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, "gpu0"); ASSERT_TRUE(config.ProcessConfigCli(reply_set, cmd_set).ok()); diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 99a67de7..0d27e62e 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -84,14 +84,13 @@ class RpcHandlerTest : public testing::Test { milvus::engine::DBOptions opt; - milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); + milvus::server::Config::GetInstance().SetGeneralConfigMetaURI("sqlite://:@:/"); milvus::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); milvus::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); - milvus::server::Config::GetInstance().SetStorageConfigPrimaryPath("/tmp/milvus_test"); - milvus::server::Config::GetInstance().SetStorageConfigSecondaryPath(""); + milvus::server::Config::GetInstance().SetStorageConfigPath("/tmp/milvus_test"); milvus::server::Config::GetInstance().SetCacheConfigCacheInsertData(""); milvus::server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); - milvus::server::Config::GetInstance().SetServerConfigPort("19531"); + milvus::server::Config::GetInstance().SetNetworkConfigBindPort("19531"); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // DBWrapper::GetInstance().GetInstance().StartService(); diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 43a1594d..1edd98bf 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -249,12 +249,14 @@ TEST(UtilTest, LOG_TEST) { fiu_init(0); fiu_enable("LogUtil.InitLog.set_max_log_size_small_than_min", 1, NULL, 0); - auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + auto status = milvus::server::InitLog(true, true, true, true, true, true, + "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB ASSERT_FALSE(status.ok()); fiu_disable("LogUtil.InitLog.set_max_log_size_small_than_min"); fiu_enable("LogUtil.InitLog.delete_exceeds_small_than_min", 1, NULL, 0); - status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + status = milvus::server::InitLog(true, true, true, true, true, true, + "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB ASSERT_FALSE(status.ok()); fiu_disable("LogUtil.InitLog.delete_exceeds_small_than_min"); @@ -264,7 +266,8 @@ TEST(UtilTest, LOG_TEST) { fiu_enable("LogUtil.InitLog.trace_enable_to_false", 1, NULL, 0); fiu_enable("LogUtil.InitLog.error_enable_to_false", 1, NULL, 0); fiu_enable("LogUtil.InitLog.fatal_enable_to_false", 1, NULL, 0); - status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + status = milvus::server::InitLog(true, true, true, true, true, true, + "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB ASSERT_TRUE(status.ok()) << status.message(); fiu_disable("LogUtil.InitLog.fatal_enable_to_false"); fiu_disable("LogUtil.InitLog.error_enable_to_false"); @@ -273,7 +276,8 @@ TEST(UtilTest, LOG_TEST) { fiu_disable("LogUtil.InitLog.debug_enable_to_false"); fiu_disable("LogUtil.InitLog.info_enable_to_false"); - status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + status = milvus::server::InitLog(true, true, true, true, true, true, + "/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB ASSERT_TRUE(status.ok()) << status.message(); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 42a01397..f4c019c3 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -142,109 +142,64 @@ namespace { static const char* CONTROLLER_TEST_VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" "\n" - "version: 0.4\n" + "version: 0.5\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# Server Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "server_config:\n" - " address: 0.0.0.0\n" - " port: 19530\n" - " deploy_mode: single\n" - " time_zone: UTC+8\n" - " web_enable: true\n" - " web_port: 19121\n" + "cluster:\n" + " enable: false\n" + " role: rw\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# DataBase Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "db_config:\n" - " backend_url: sqlite://:@:/\n" - " preload_collection:\n" + "general:\n" + " timezone: UTC+8\n" + " meta_uri: sqlite://:@:/\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# Storage Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "storage_config:\n" - " primary_path: /tmp/milvus\n" - " secondary_path:\n" - " s3_enable: false\n" - " s3_address: 127.0.0.1\n" - " s3_port: 9000\n" - " s3_access_key: minioadmin\n" - " s3_secret_key: minioadmin\n" - " s3_bucket: milvus-bucket\n" + "network:\n" + " bind.address: 0.0.0.0\n" + " bind.port: 19530\n" + " http.enable: true\n" + " http.port: 19121\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# Metric Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "metric_config:\n" - " enable_monitor: false\n" - " address: 127.0.0.1\n" - " port: 9091\n" + "storage:\n" + " path: /tmp/milvus\n" + " auto_flush_interval: 1\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# Cache Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "cache_config:\n" - " cpu_cache_capacity: 4\n" - " insert_buffer_size: 1\n" - " cache_insert_data: false\n" + "wal:\n" + " enable: true\n" + " recovery_error_ignore: false\n" + " buffer_size: 256MB\n" + " path: /tmp/milvus/wal\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# Engine Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "engine_config:\n" - " use_blas_threshold: 1100\n" - #ifdef MILVUS_GPU_VERSION - " gpu_search_threshold: 1000\n" + "cache:\n" + " cache_size: 4GB\n" + " insert_buffer_size: 1GB\n" + " preload_collection:\n" "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# GPU Resource Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "gpu_resource_config:\n" + "gpu:\n" " enable: true\n" - " cache_capacity: 1\n" - " search_resources:\n" + " cache_size: 1GB\n" + " gpu_search_threshold: 1000\n" + " search_devices:\n" " - gpu0\n" - " build_index_resources:\n" + " build_index_devices:\n" " - gpu0\n" - #endif "\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "# Tracing Config | Description | Type | Default " - " |\n" - "#----------------------+------------------------------------------------------------+------------+----------------" - "-+\n" - "tracing_config:\n" - " json_config_path:\n" - ""; + "logs:\n" + " level: debug\n" + " trace.enable: true\n" + " path: /tmp/milvus/logs\n" + " max_log_file_size: 1024MB\n" + " log_rotate_num: 0\n" + "\n" + "metric:\n" + " enable: false\n" + " address: 127.0.0.1\n" + " port: 9091\n" + "\n"; + } // namespace static const char* CONTROLLER_TEST_COLLECTION_NAME = "controller_unit_test"; static const char* CONTROLLER_TEST_CONFIG_DIR = "/tmp/milvus_web_controller_test/"; +static const char* CONTROLLER_TEST_CONFIG_WAL_DIR = "/tmp/milvus_web_controller_test/wal"; static const char* CONTROLLER_TEST_CONFIG_FILE = "config.yaml"; class TestClient : public oatpp::web::client::ApiClient { @@ -382,13 +337,14 @@ class WebControllerTest : public ::testing::Test { milvus::engine::DBOptions opt; - milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); + milvus::server::Config::GetInstance().SetGeneralConfigMetaURI("sqlite://:@:/"); boost::filesystem::remove_all(CONTROLLER_TEST_CONFIG_DIR); - milvus::server::Config::GetInstance().SetStorageConfigPrimaryPath(CONTROLLER_TEST_CONFIG_DIR); + milvus::server::Config::GetInstance().SetStorageConfigPath(CONTROLLER_TEST_CONFIG_DIR); + milvus::server::Config::GetInstance().SetWalConfigWalPath(CONTROLLER_TEST_CONFIG_WAL_DIR); milvus::server::DBWrapper::GetInstance().StartService(); - milvus::server::Config::GetInstance().SetServerConfigWebPort("29999"); + milvus::server::Config::GetInstance().SetNetworkConfigHTTPPort("29999"); milvus::server::web::WebServer::GetInstance().Start(); @@ -1421,11 +1377,11 @@ TEST_F(WebControllerTest, CONFIG) { OString collection_name_s = "milvus_test_webcontroller_test_preload_collection_s"; GenCollection(client_ptr, conncetion_ptr, collection_name_s, 16, 10, "L2"); - OString body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "\"}}"; + OString body_str = "{\"cache\": {\"preload_collection\": \"" + collection_name + "\"}}"; response = client_ptr->op("config", body_str, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); - body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "," + collection_name_s + "\"}}"; + body_str = "{\"cache\": {\"preload_collection\": \"" + collection_name + "," + collection_name_s + "\"}}"; response = client_ptr->op("config", body_str, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); auto set_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); @@ -1467,7 +1423,7 @@ TEST_F(WebControllerTest, ADVANCED_CONFIG) { auto config_dto = milvus::server::web::AdvancedConfigDto::createShared(); response = client_ptr->setAdvanced(config_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); config_dto->cpu_cache_capacity = 3; response = client_ptr->setAdvanced(config_dto, conncetion_ptr); @@ -1489,7 +1445,7 @@ TEST_F(WebControllerTest, ADVANCED_CONFIG) { // test fault // cpu cache capacity exceed total memory - config_dto->cpu_cache_capacity = 10000000; + config_dto->cpu_cache_capacity = 10000L * (1024L * 1024 * 1024); // 10000 GB response = client_ptr->setAdvanced(config_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); } @@ -1543,10 +1499,10 @@ TEST_F(WebControllerTest, GPU_CONFIG) { ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); //// test fault config - // cache capacity exceed GPU mem size - gpu_config_dto->cache_capacity = 100000; + // cache capacity exceed GPU mem size (GiB) + gpu_config_dto->cache_capacity = 100000L * 1024 * 1024 * 1024; // 100000 GiB response = client_ptr->setGPUConfig(gpu_config_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); + ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()) << response->readBodyToString()->c_str(); gpu_config_dto->cache_capacity = 1; // duplicate resources diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 81eea19c..3449b95a 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -19,6 +19,64 @@ namespace { + +static const char* VALID_CONFIG_STR = + "# Default values are used when you make no changes to the following parameters.\n" + "\n" + "version: 0.5\n" + "\n" + "cluster:\n" + " enable: false\n" + " role: rw\n" + "\n" + "general:\n" + " timezone: UTC+8\n" + " meta_uri: sqlite://:@:/\n" + "\n" + "network:\n" + " bind.address: 0.0.0.0\n" + " bind.port: 19530\n" + " http.enable: true\n" + " http.port: 19121\n" + "\n" + "storage:\n" + " path: /tmp/milvus\n" + " auto_flush_interval: 1\n" + "\n" + "wal:\n" + " enable: true\n" + " recovery_error_ignore: false\n" + " buffer_size: 256MB\n" + " path: /tmp/milvus/wal\n" + "\n" + "cache:\n" + " cache_size: 4GB\n" + " insert_buffer_size: 1GB\n" + " preload_collection:\n" + "\n" + "gpu:\n" + " enable: true\n" + " cache_size: 1GB\n" + " gpu_search_threshold: 1000\n" + " search_devices:\n" + " - gpu0\n" + " build_index_devices:\n" + " - gpu0\n" + "\n" + "logs:\n" + " level: debug\n" + " trace.enable: true\n" + " path: /tmp/milvus/logs\n" + " max_log_file_size: 1024MB\n" + " log_rotate_num: 0\n" + "\n" + "metric:\n" + " enable: false\n" + " address: 127.0.0.1\n" + " port: 9091\n" + "\n"; + +/* static const char* VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" "\n" @@ -84,6 +142,7 @@ static const char* VALID_CONFIG_STR = " max_log_file_size: 256\n" " delete_exceeds: 10\n" ""; +*/ static const char* INVALID_CONFIG_STR = "*INVALID*"; diff --git a/tests/milvus_python_test/test_compact.py b/tests/milvus_python_test/test_compact.py index ffdd6554..9b54f642 100644 --- a/tests/milvus_python_test/test_compact.py +++ b/tests/milvus_python_test/test_compact.py @@ -378,6 +378,7 @@ class TestCompactBase: assert status.OK() @pytest.mark.timeout(COMPACT_TIMEOUT) + @pytest.mark.repeat(10) def test_index_creation_after_compact(self, connect, collection, get_simple_index): ''' target: test index creation after compact diff --git a/tests/milvus_python_test/test_config.py b/tests/milvus_python_test/test_config.py index 88646f77..50fb53f3 100644 --- a/tests/milvus_python_test/test_config.py +++ b/tests/milvus_python_test/test_config.py @@ -35,58 +35,56 @@ class TestCacheConfig: ''' reset configs so the tests are stable ''' - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", 4) + status, reply = connect.set_config("cache", "cache_size", '4GB') assert status.OK() - status, reply = connect.set_config("cache_config", "cache_insert_data", "false") + status, config_value = connect.get_config("cache", "cache_size") + assert config_value == '4GB' + status, reply = connect.set_config("cache", "insert_buffer_size", '1GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "cpu_cache_capacity") - assert config_value == '4' - status, reply = connect.set_config("cache_config", "insert_buffer_size", 1) - assert status.OK() - status, config_value = connect.get_config("cache_config", "insert_buffer_size") - assert config_value == '1' + status, config_value = connect.get_config("cache", "insert_buffer_size") + assert config_value == '1GB' @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_cpu_cache_capacity_invalid_parent_key(self, connect, collection): + def test_get_cache_size_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: cache_config + method: call get_config without parent_key: cache expected: status not ok ''' invalid_configs = gen_invalid_cache_config() invalid_configs.extend(["Cache_config", "cache config", "cache_Config", "cacheconfig"]) for config in invalid_configs: - status, config_value = connect.get_config(config, "cpu_cache_capacity") + status, config_value = connect.get_config(config, "cache_size") assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_cpu_cache_capacity_invalid_child_key(self, connect, collection): + def test_get_cache_size_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: cpu_cache_capacity + method: call get_config without child_key: cache_size expected: status not ok ''' invalid_configs = gen_invalid_cache_config() - invalid_configs.extend(["Cpu_cache_capacity", "cpu cache_capacity", "cpucachecapacity"]) + invalid_configs.extend(["Cpu_cache_size", "cpu cache_size", "cpucachecapacity"]) for config in invalid_configs: - status, config_value = connect.get_config("cache_config", config) + status, config_value = connect.get_config("cache", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_cpu_cache_capacity_valid(self, connect, collection): + def test_get_cache_size_valid(self, connect, collection): ''' - target: get cpu_cache_capacity + target: get cache_size method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("cache_config", "cpu_cache_capacity") + status, config_value = connect.get_config("cache", "cache_size") assert status.OK() @pytest.mark.level(2) def test_get_insert_buffer_size_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: cache_config + method: call get_config without parent_key: cache expected: status not ok ''' invalid_configs = gen_invalid_cache_config() @@ -105,7 +103,7 @@ class TestCacheConfig: invalid_configs = gen_invalid_cache_config() invalid_configs.extend(["Insert_buffer_size", "insert buffer_size", "insertbuffersize"]) for config in invalid_configs: - status, config_value = connect.get_config("cache_config", config) + status, config_value = connect.get_config("cache", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -115,46 +113,31 @@ class TestCacheConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("cache_config", "insert_buffer_size") + status, config_value = connect.get_config("cache", "insert_buffer_size") assert status.OK() @pytest.mark.level(2) - def test_get_cache_insert_data_invalid_parent_key(self, connect, collection): - ''' - target: get invalid parent key - method: call get_config without parent_key: cache_config - expected: status not ok - ''' - invalid_configs = gen_invalid_cache_config() - invalid_configs.extend(["Cache_config", "cache config", "cache_Config", "cacheconfig"]) - for config in invalid_configs: - status, config_value = connect.get_config(config, "cache_insert_data") - assert not status.OK() - - @pytest.mark.level(2) - def test_get_cache_insert_data_invalid_child_key(self, connect, collection): + def test_get_preload_collection_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: cache_insert_data + method: call get_config without child_key: preload_collection expected: status not ok ''' - invalid_configs = gen_invalid_cache_config() - invalid_configs.extend(["Cache_insert_data", "cacheinsertdata", " cache_insert_data"]) + invalid_configs = ["preloadtable", "preload_collection "] for config in invalid_configs: - status, config_value = connect.get_config("cache_config", config) + status, config_value = connect.get_config("cache", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_cache_insert_data_valid(self, connect, collection): + def test_get_preload_collection_valid(self, connect, collection): ''' - target: get cache_insert_data + target: get preload_collection method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("cache_config", "cache_insert_data") + status, config_value = connect.get_config("cache", "preload_collection") assert status.OK() - """ ****************************************************************** The following cases are used to test `set_config` function @@ -177,21 +160,21 @@ class TestCacheConfig: return int(mem_total / 1024 / 1024 / 1024) @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cpu_cache_capacity_invalid_parent_key(self, connect, collection): + def test_set_cache_size_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: cache_config + method: call set_config without parent_key: cache expected: status not ok ''' self.reset_configs(connect) invalid_configs = gen_invalid_cache_config() invalid_configs.extend(["Cache_config", "cache config", "cache_Config", "cacheconfig"]) for config in invalid_configs: - status, reply = connect.set_config(config, "cpu_cache_capacity", 4) + status, reply = connect.set_config(config, "cache_size", '4GB') assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cache_config_invalid_child_key(self, connect, collection): + def test_set_cache_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key @@ -200,55 +183,56 @@ class TestCacheConfig: self.reset_configs(connect) invalid_configs = gen_invalid_cache_config() for config in invalid_configs: - status, reply = connect.set_config("cache_config", config, 4) + status, reply = connect.set_config("cache", config, '4GB') assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cpu_cache_capacity_valid(self, connect, collection): + def test_set_cache_size_valid(self, connect, collection): ''' - target: set cpu_cache_capacity + target: set cache_size method: call set_config correctly expected: status ok, set successfully ''' self.reset_configs(connect) - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", 8) + status, reply = connect.set_config("cache", "cache_size", '8GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "cpu_cache_capacity") + status, config_value = connect.get_config("cache", "cache_size") assert status.OK() - assert config_value == '8' + assert config_value == '8GB' - def test_set_cpu_cache_capacity_valid_multiple_times(self, connect, collection): + @pytest.mark.level(2) + def test_set_cache_size_valid_multiple_times(self, connect, collection): ''' - target: set cpu_cache_capacity + target: set cache_size method: call set_config correctly and repeatedly expected: status ok ''' self.reset_configs(connect) for i in range(20): - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", 4) + status, reply = connect.set_config("cache", "cache_size", '4GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "cpu_cache_capacity") + status, config_value = connect.get_config("cache", "cache_size") assert status.OK() - assert config_value == '4' + assert config_value == '4GB' for i in range(20): - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", 8) + status, reply = connect.set_config("cache", "cache_size", '8GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "cpu_cache_capacity") + status, config_value = connect.get_config("cache", "cache_size") assert status.OK() - assert config_value == '8' + assert config_value == '8GB' @pytest.mark.level(2) def test_set_insert_buffer_size_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: cache_config + method: call set_config without parent_key: cache expected: status not ok ''' self.reset_configs(connect) invalid_configs = gen_invalid_cache_config() invalid_configs.extend(["Cache_config", "cache config", "cache_Config", "cacheconfig"]) for config in invalid_configs: - status, reply = connect.set_config(config, "insert_buffer_size", 1) + status, reply = connect.set_config(config, "insert_buffer_size", '1GB') assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -259,11 +243,11 @@ class TestCacheConfig: expected: status ok, set successfully ''' self.reset_configs(connect) - status, reply = connect.set_config("cache_config", "insert_buffer_size", 2) + status, reply = connect.set_config("cache", "insert_buffer_size", '2GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "insert_buffer_size") + status, config_value = connect.get_config("cache", "insert_buffer_size") assert status.OK() - assert config_value == '2' + assert config_value == '2GB' @pytest.mark.level(2) def test_set_insert_buffer_size_valid_multiple_times(self, connect, collection): @@ -274,112 +258,47 @@ class TestCacheConfig: ''' self.reset_configs(connect) for i in range(20): - status, reply = connect.set_config("cache_config", "insert_buffer_size", 1) + status, reply = connect.set_config("cache", "insert_buffer_size", '1GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "insert_buffer_size") + status, config_value = connect.get_config("cache", "insert_buffer_size") assert status.OK() - assert config_value == '1' + assert config_value == '1GB' for i in range(20): - status, reply = connect.set_config("cache_config", "insert_buffer_size", 2) + status, reply = connect.set_config("cache", "insert_buffer_size", '2GB') assert status.OK() - status, config_value = connect.get_config("cache_config", "insert_buffer_size") + status, config_value = connect.get_config("cache", "insert_buffer_size") assert status.OK() - assert config_value == '2' + assert config_value == '2GB' @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cache_config_out_of_memory_value_A(self, connect, collection): + def test_set_cache_out_of_memory_value_A(self, connect, collection): ''' - target: set cpu_cache_capacity / insert_buffer_size to be out-of-memory + target: set cache_size / insert_buffer_size to be out-of-memory method: call set_config with child values bigger than current system memory - expected: status not ok (cpu_cache_capacity + insert_buffer_size < system memory) + expected: status not ok (cache_size + insert_buffer_size < system memory) ''' self.reset_configs(connect) mem_total = self.get_memory_total(connect) logging.getLogger().info(mem_total) - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", mem_total + 1) + status, reply = connect.set_config("cache", "cache_size", str(int(mem_total + 1))+'GB') assert not status.OK() - status, reply = connect.set_config("cache_config", "insert_buffer_size", mem_total + 1) + status, reply = connect.set_config("cache", "insert_buffer_size", str(int(mem_total + 1))+'GB') assert not status.OK() - @pytest.mark.skip(reason="Still needs discussion") - def test_set_cache_config_out_of_memory_value_B(self, connect, collection): + def test_set_preload_collection_valid(self, connect, collection): ''' - target: set cpu_cache_capacity / insert_buffer_size to be out-of-memory - method: call set_config with invalid values - expected: status not ok (cpu_cache_capacity + insert_buffer_size < system memory) + target: set preload_collection + method: call set_config correctly + expected: status ok, set successfully ''' - self.reset_configs(connect) - mem_available = self.get_memory_available(connect) - logging.getLogger().info(mem_available) - status, cpu_cache_capacity = connect.get_config("cache_config", "cpu_cache_capacity") - assert status.OK() - logging.getLogger().info(cpu_cache_capacity) - status, insert_buffer_size = connect.get_config("cache_config", "insert_buffer_size") + status, reply = connect.set_config("cache", "preload_collection", "") assert status.OK() - logging.getLogger().info(insert_buffer_size) - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", mem_available - int(insert_buffer_size) + 1) - assert not status.OK() - status, reply = connect.set_config("cache_config", "insert_buffer_size", mem_available - int(cpu_cache_capacity) + 1) - assert not status.OK() - - @pytest.mark.skip(reason="Still needs discussion") - def test_set_cache_config_out_of_memory_value_C(self, connect, collection): - ''' - target: set cpu_cache_capacity / insert_buffer_size to be out-of-memory - method: call set_config multiple times - expected: status not ok (cpu_cache_capacity + insert_buffer_size < system memory) - ''' - self.reset_configs(connect) - mem_available = self.get_memory_available(connect) - logging.getLogger().info(mem_available) - status, insert_buffer_size = connect.get_config("cache_config", "insert_buffer_size") + status, config_value = connect.get_config("cache", "preload_collection") assert status.OK() - if mem_available >= int(insert_buffer_size) + 20: - status, reply = connect.set_config("cache_config", "cpu_cache_capacity", mem_available - int(insert_buffer_size) - 10) - assert status.OK() - status, reply = connect.set_config("cache_config", "insert_buffer_size", int(insert_buffer_size) + 20) - assert not status.OK() - status, reply = connect.set_config("cache_config", "insert_buffer_size", int(insert_buffer_size) + 1) - assert status.OK() - status, insert_buffer_size_new = connect.get_config("cache_config", "insert_buffer_size") - assert int(insert_buffer_size_new) == int(insert_buffer_size) + 1 - self.reset_configs(connect) - - @pytest.mark.level(2) - def test_set_cache_insert_data_invalid_parent_key(self, connect, collection): - ''' - target: set invalid parent key - method: call set_config without parent_key: cache_config - expected: status not ok - ''' - self.reset_configs(connect) - invalid_configs = gen_invalid_cache_config() - invalid_configs.extend(["Cache_config", "cache config", "cache_Config", "cacheconfig"]) - for config in invalid_configs: - status, reply = connect.set_config(config, "cache_insert_data", "1") - assert not status.OK() - self.reset_configs(connect) - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cache_insert_data_valid(self, connect, collection): - ''' - target: set cache_insert_data - method: call get_config correctly - expected: status ok, set successfully - ''' - self.reset_configs(connect) - # On/Off true/false 1/0 YES/NO - valid_configs = ["Off", "false", 0, "NO", "On", "true", "1", "YES"] - for config in valid_configs: - status, reply = connect.set_config("cache_config", "cache_insert_data", config) - assert status.OK() - status, config_value = connect.get_config("cache_config", "cache_insert_data") - assert status.OK() - assert config_value == str(config) - self.reset_configs(connect) + assert config_value == "" -class TestEngineConfig: +class TestGPUConfig: """ ****************************************************************** The following cases are used to test `get_config` function @@ -390,53 +309,16 @@ class TestEngineConfig: if args["handler"] == "HTTP": pytest.skip("skip in http mode") - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_use_blas_threshold_invalid_parent_key(self, connect, collection): - ''' - target: get invalid parent key - method: call get_config without parent_key: engine_config - expected: status not ok - ''' - invalid_configs = gen_invalid_engine_config() - invalid_configs.extend(["Engine_config", "engine config"]) - for config in invalid_configs: - status, config_value = connect.get_config(config, "use_blas_threshold") - assert not status.OK() - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_use_blas_threshold_invalid_child_key(self, connect, collection): - ''' - target: get invalid child key - method: call get_config without child_key: use_blas_threshold - expected: status not ok - ''' - invalid_configs = gen_invalid_engine_config() - invalid_configs.extend(["Use_blas_threshold", "use blas threshold"]) - for config in invalid_configs: - status, config_value = connect.get_config("engine_config", config) - assert not status.OK() - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_use_blas_threshold_valid(self, connect, collection): - ''' - target: get use_blas_threshold - method: call get_config correctly - expected: status ok - ''' - status, config_value = connect.get_config("engine_config", "use_blas_threshold") - assert status.OK() - @pytest.mark.level(2) def test_get_gpu_search_threshold_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: engine_config + method: call get_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - invalid_configs = gen_invalid_engine_config() - invalid_configs.extend(["Engine_config", "engine config"]) + invalid_configs = ["Engine_config", "engine config"] for config in invalid_configs: status, config_value = connect.get_config(config, "gpu_search_threshold") assert not status.OK() @@ -450,10 +332,9 @@ class TestEngineConfig: ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - invalid_configs = gen_invalid_engine_config() - invalid_configs.extend(["Gpu_search_threshold", "gpusearchthreshold"]) + invalid_configs = ["Gpu_search_threshold", "gpusearchthreshold"] for config in invalid_configs: - status, config_value = connect.get_config("engine_config", config) + status, config_value = connect.get_config("gpu", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -465,80 +346,36 @@ class TestEngineConfig: ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, config_value = connect.get_config("engine_config", "gpu_search_threshold") + status, config_value = connect.get_config("gpu", "gpu_search_threshold") assert status.OK() - """ ****************************************************************** The following cases are used to test `set_config` function ****************************************************************** """ @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_use_blas_threshold_invalid_parent_key(self, connect, collection): - ''' - target: set invalid parent key - method: call set_config without parent_key: engine_config - expected: status not ok - ''' - invalid_configs = gen_invalid_engine_config() - invalid_configs.extend(["Engine_config", "engine config"]) - for config in invalid_configs: - status, reply = connect.set_config(config, "use_blas_threshold", 1000) - assert not status.OK() - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_engine_config_invalid_child_key(self, connect, collection): + def test_set_gpu_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key expected: status not ok ''' - invalid_configs = gen_invalid_engine_config() + invalid_configs = gen_invalid_gpu_config() for config in invalid_configs: - status, reply = connect.set_config("engine_config", config, 1000) + status, reply = connect.set_config("gpu", config, 1000) assert not status.OK() - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_use_blas_threshold_valid(self, connect, collection): - ''' - target: set use_blas_threshold - method: call set_config correctly - expected: status ok - ''' - status, reply = connect.set_config("engine_config", "use_blas_threshold", '2000') - assert status.OK() - status, config_value = connect.get_config("engine_config", "use_blas_threshold") - assert status.OK() - assert config_value == '2000' - - @pytest.mark.level(2) - def test_set_use_blas_threshold_valid_multiple_times(self, connect, collection): - ''' - target: set use_blas_threshold - method: call set_config correctly and repeatedly - expected: status ok - ''' - for i in range(1, 100): - status, reply = connect.set_config("engine_config", "use_blas_threshold", i * 100) - assert status.OK() - status, config_value = connect.get_config("engine_config", "use_blas_threshold") - assert status.OK() - assert config_value == str(i * 100) - # reset - status, reply = connect.set_config("engine_config", "use_blas_threshold", 1100) - assert status.OK() - @pytest.mark.timeout(CONFIG_TIMEOUT) def test_set_gpu_search_threshold_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: engine_config + method: call set_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - invalid_configs = gen_invalid_engine_config() + invalid_configs = gen_invalid_gpu_config() invalid_configs.extend(["Engine_config", "engine config"]) for config in invalid_configs: status, reply = connect.set_config(config, "gpu_search_threshold", 1000) @@ -553,84 +390,53 @@ class TestEngineConfig: ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, reply = connect.set_config("engine_config", "gpu_search_threshold", 2000) + status, reply = connect.set_config("gpu", "gpu_search_threshold", 2000) assert status.OK() - status, config_value = connect.get_config("engine_config", "gpu_search_threshold") + status, config_value = connect.get_config("gpu", "gpu_search_threshold") assert status.OK() assert config_value == '2000' - @pytest.mark.level(2) - def test_set_gpu_search_threshold_valid_multiple_times(self, connect, collection): - ''' - target: set gpu_search_threshold - method: call set_config correctly and repeatedly - expected: status ok - ''' - if str(connect._cmd("mode")[1]) == "CPU": - pytest.skip("Only support GPU mode") - for i in range(1, 100): - status, reply = connect.set_config("engine_config", "gpu_search_threshold", i * 100) - assert status.OK() - # reset config - status, reply = connect.set_config("engine_config", "use_blas_threshold", 1100) - assert status.OK() - if str(connect._cmd("mode")[1]) == "GPU": - status, reply = connect.set_config("engine_config", "gpu_search_threshold", 1000) - assert status.OK() - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_engine_config_invalid_values(self, connect, collection): + def test_set_gpu_invalid_values(self, connect, collection): ''' - target: set engine_config + target: set gpu method: call set_config with invalid child values expected: status not ok ''' for i in [-1, "1000\n", "1000\t", "1000.0", 1000.35]: - status, reply = connect.set_config("engine_config", "use_blas_threshold", i) + status, reply = connect.set_config("gpu", "use_blas_threshold", i) assert not status.OK() if str(connect._cmd("mode")[1]) == "GPU": - status, reply = connect.set_config("engine_config", "gpu_search_threshold", i) + status, reply = connect.set_config("gpu", "gpu_search_threshold", i) assert not status.OK() - -class TestGPUResourceConfig: - """ - ****************************************************************** - The following cases are used to test `get_config` function - ****************************************************************** - """ - @pytest.fixture(scope="function", autouse=True) - def skip_http_check(self, args): - if args["handler"] == "HTTP": - pytest.skip("skip in http mode") - @pytest.mark.timeout(CONFIG_TIMEOUT) def reset_configs(self, connect): ''' reset configs so the tests are stable ''' - status, reply = connect.set_config("gpu_resource_config", "enable", "true") + status, reply = connect.set_config("gpu", "enable", "true") assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "enable") + status, config_value = connect.get_config("gpu", "enable") assert config_value == "true" - status, reply = connect.set_config("gpu_resource_config", "cache_capacity", 1) + status, reply = connect.set_config("gpu", "cache_size", 1) assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "cache_capacity") + status, config_value = connect.get_config("gpu", "cache_size") assert config_value == '1' - status, reply = connect.set_config("gpu_resource_config", "search_resources", "gpu0") + status, reply = connect.set_config("gpu", "search_devices", "gpu0") assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "search_resources") + status, config_value = connect.get_config("gpu", "search_devices") assert config_value == 'gpu0' - status, reply = connect.set_config("gpu_resource_config", "build_index_resources", "gpu0") + status, reply = connect.set_config("gpu", "build_index_devices", "gpu0") assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "build_index_resources") + status, config_value = connect.get_config("gpu", "build_index_devices") assert config_value == 'gpu0' @pytest.mark.timeout(CONFIG_TIMEOUT) def test_get_gpu_enable_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: gpu_resource_config + method: call get_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -652,7 +458,7 @@ class TestGPUResourceConfig: pytest.skip("Only support GPU mode") invalid_configs = ["Enable", "enable ", "disable", "true"] for config in invalid_configs: - status, config_value = connect.get_config("gpu_resource_config", config) + status, config_value = connect.get_config("gpu", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -664,15 +470,15 @@ class TestGPUResourceConfig: ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, config_value = connect.get_config("gpu_resource_config", "enable") + status, config_value = connect.get_config("gpu", "enable") assert status.OK() assert config_value == "true" or config_value == "false" @pytest.mark.level(2) - def test_get_cache_capacity_invalid_parent_key(self, connect, collection): + def test_get_cache_size_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: gpu_resource_config + method: call get_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -680,40 +486,40 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, config_value = connect.get_config(config, "cache_capacity") + status, config_value = connect.get_config(config, "cache_size") assert not status.OK() @pytest.mark.level(2) - def test_get_cache_capacity_invalid_child_key(self, connect, collection): + def test_get_cache_size_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: cache_capacity + method: call get_config without child_key: cache_size expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") invalid_configs = ["Cache_capacity", "cachecapacity"] for config in invalid_configs: - status, config_value = connect.get_config("gpu_resource_config", config) + status, config_value = connect.get_config("gpu", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_cache_capacity_valid(self, connect, collection): + def test_get_cache_size_valid(self, connect, collection): ''' - target: get cache_capacity + target: get cache_size method: call get_config correctly expected: status ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, config_value = connect.get_config("gpu_resource_config", "cache_capacity") + status, config_value = connect.get_config("gpu", "cache_size") assert status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_search_resources_invalid_parent_key(self, connect, collection): + def test_get_search_devices_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: gpu_resource_config + method: call get_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -721,41 +527,41 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, config_value = connect.get_config(config, "search_resources") + status, config_value = connect.get_config(config, "search_devices") assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_search_resources_invalid_child_key(self, connect, collection): + def test_get_search_devices_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: search_resources + method: call get_config without child_key: search_devices expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") invalid_configs = ["Search_resources"] for config in invalid_configs: - status, config_value = connect.get_config("gpu_resource_config", config) + status, config_value = connect.get_config("gpu", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_search_resources_valid(self, connect, collection): + def test_get_search_devices_valid(self, connect, collection): ''' - target: get search_resources + target: get search_devices method: call get_config correctly expected: status ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, config_value = connect.get_config("gpu_resource_config", "search_resources") + status, config_value = connect.get_config("gpu", "search_devices") logging.getLogger().info(config_value) assert status.OK() @pytest.mark.level(2) - def test_get_build_index_resources_invalid_parent_key(self, connect, collection): + def test_get_build_index_devices_invalid_parent_key(self, connect, collection): ''' target: get invalid parent key - method: call get_config without parent_key: gpu_resource_config + method: call get_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -763,33 +569,33 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, config_value = connect.get_config(config, "build_index_resources") + status, config_value = connect.get_config(config, "build_index_devices") assert not status.OK() @pytest.mark.level(2) - def test_get_build_index_resources_invalid_child_key(self, connect, collection): + def test_get_build_index_devices_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: build_index_resources + method: call get_config without child_key: build_index_devices expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") invalid_configs = ["Build_index_resources"] for config in invalid_configs: - status, config_value = connect.get_config("gpu_resource_config", config) + status, config_value = connect.get_config("gpu", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_build_index_resources_valid(self, connect, collection): + def test_get_build_index_devices_valid(self, connect, collection): ''' - target: get build_index_resources + target: get build_index_devices method: call get_config correctly expected: status ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, config_value = connect.get_config("gpu_resource_config", "build_index_resources") + status, config_value = connect.get_config("gpu", "build_index_devices") logging.getLogger().info(config_value) assert status.OK() @@ -803,7 +609,7 @@ class TestGPUResourceConfig: def test_set_gpu_enable_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: gpu_resource_config + method: call set_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -815,7 +621,7 @@ class TestGPUResourceConfig: assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_gpu_resource_config_invalid_child_key(self, connect, collection): + def test_set_gpu_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key @@ -826,7 +632,7 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, reply = connect.set_config("gpu_resource_config", config, "true") + status, reply = connect.set_config("gpu", config, "true") assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -839,7 +645,7 @@ class TestGPUResourceConfig: if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") for i in [-1, -2, 100]: - status, reply = connect.set_config("gpu_resource_config", "enable", i) + status, reply = connect.set_config("gpu", "enable", i) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -853,17 +659,17 @@ class TestGPUResourceConfig: pytest.skip("Only support GPU mode") valid_configs = ["off", "False", "0", "nO", "on", "True", 1, "yES"] for config in valid_configs: - status, reply = connect.set_config("gpu_resource_config", "enable", config) + status, reply = connect.set_config("gpu", "enable", config) assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "enable") + status, config_value = connect.get_config("gpu", "enable") assert status.OK() assert config_value == str(config) @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cache_capacity_invalid_parent_key(self, connect, collection): + def test_set_cache_size_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: gpu_resource_config + method: call set_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -871,25 +677,25 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, reply = connect.set_config(config, "cache_capacity", 2) + status, reply = connect.set_config(config, "cache_size", 2) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cache_capacity_valid(self, connect, collection): + def test_set_cache_size_valid(self, connect, collection): ''' - target: set cache_capacity + target: set cache_size method: call set_config correctly expected: status ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, reply = connect.set_config("gpu_resource_config", "cache_capacity", 2) + status, reply = connect.set_config("gpu", "cache_size", 2) assert status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_cache_capacity_invalid_values(self, connect, collection): + def test_set_cache_size_invalid_values(self, connect, collection): ''' - target: set cache_capacity + target: set cache_size method: call set_config with invalid child values expected: status not ok ''' @@ -898,14 +704,14 @@ class TestGPUResourceConfig: self.reset_configs(connect) for i in [-1, "1\n", "1\t"]: logging.getLogger().info(i) - status, reply = connect.set_config("gpu_resource_config", "cache_capacity", i) + status, reply = connect.set_config("gpu", "cache_size", i) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_search_resources_invalid_parent_key(self, connect, collection): + def test_set_search_devices_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: gpu_resource_config + method: call set_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -913,41 +719,41 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, reply = connect.set_config(config, "search_resources", "gpu0") + status, reply = connect.set_config(config, "search_devices", "gpu0") assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_search_resources_valid(self, connect, collection): + def test_set_search_devices_valid(self, connect, collection): ''' - target: set search_resources + target: set search_devices method: call set_config correctly expected: status ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, reply = connect.set_config("gpu_resource_config", "search_resources", "gpu0") + status, reply = connect.set_config("gpu", "search_devices", "gpu0") assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "search_resources") + status, config_value = connect.get_config("gpu", "search_devices") assert config_value == "gpu0" @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_search_resources_invalid_values(self, connect, collection): + def test_set_search_devices_invalid_values(self, connect, collection): ''' - target: set search_resources + target: set search_devices method: call set_config with invalid child values expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") for i in [-1, "10", "gpu-1", "gpu0, gpu1", "gpu22,gpu44","gpu10000","gpu 0","-gpu0"]: - status, reply = connect.set_config("gpu_resource_config", "search_resources", i) + status, reply = connect.set_config("gpu", "search_devices", i) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_build_index_resources_invalid_parent_key(self, connect, collection): + def test_set_build_index_devices_invalid_parent_key(self, connect, collection): ''' target: set invalid parent key - method: call set_config without parent_key: gpu_resource_config + method: call set_config without parent_key: gpu expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": @@ -955,39 +761,39 @@ class TestGPUResourceConfig: invalid_configs = ["Gpu_resource_config", "gpu resource config", \ "gpu_resource"] for config in invalid_configs: - status, reply = connect.set_config(config, "build_index_resources", "gpu0") + status, reply = connect.set_config(config, "build_index_devices", "gpu0") assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_build_index_resources_valid(self, connect, collection): + def test_set_build_index_devices_valid(self, connect, collection): ''' - target: set build_index_resources + target: set build_index_devices method: call set_config correctly expected: status ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") - status, reply = connect.set_config("gpu_resource_config", "build_index_resources", "gpu0") + status, reply = connect.set_config("gpu", "build_index_devices", "gpu0") assert status.OK() - status, config_value = connect.get_config("gpu_resource_config", "build_index_resources") + status, config_value = connect.get_config("gpu", "build_index_devices") assert config_value == "gpu0" @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_build_index_resources_invalid_values(self, connect, collection): + def test_set_build_index_devices_invalid_values(self, connect, collection): ''' - target: set build_index_resources + target: set build_index_devices method: call set_config with invalid child values expected: status not ok ''' if str(connect._cmd("mode")[1]) == "CPU": pytest.skip("Only support GPU mode") for i in [-1, "10", "gpu-1", "gpu0, gpu1", "gpu22,gpu44","gpu10000","gpu 0","-gpu0"]: - status, reply = connect.set_config("gpu_resource_config", "build_index_resources", i) + status, reply = connect.set_config("gpu", "build_index_devices", i) assert not status.OK() self.reset_configs(connect) -class TestServerConfig: +class TestNetworkConfig: """ ****************************************************************** The following cases are used to test `get_config` function @@ -1007,7 +813,7 @@ class TestServerConfig: ''' invalid_configs = ["Address", "addresses", "address "] for config in invalid_configs: - status, config_value = connect.get_config("server_config", config) + status, config_value = connect.get_config("network", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1017,7 +823,7 @@ class TestServerConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("server_config", "address") + status, config_value = connect.get_config("network", "bind.address") assert status.OK() @pytest.mark.level(2) @@ -1029,7 +835,7 @@ class TestServerConfig: ''' invalid_configs = ["Port", "PORT", "port "] for config in invalid_configs: - status, config_value = connect.get_config("server_config", config) + status, config_value = connect.get_config("network", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1039,74 +845,29 @@ class TestServerConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("server_config", "port") + status, config_value = connect.get_config("network", "http.port") assert status.OK() @pytest.mark.level(2) - def test_get_deploy_mode_invalid_child_key(self, connect, collection): + def test_get_http_port_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: deploy_mode + method: call get_config without child_key: http.port expected: status not ok ''' - invalid_configs = ["Deploy_mode", "deploymode", "deploy_mode "] + invalid_configs = ["webport", "Web_port", "http.port "] for config in invalid_configs: - status, config_value = connect.get_config("server_config", config) + status, config_value = connect.get_config("network", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_deploy_mode_valid(self, connect, collection): + def test_get_http_port_valid(self, connect, collection): ''' - target: get deploy_mode + target: get http.port method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("server_config", "deploy_mode") - assert status.OK() - - @pytest.mark.level(2) - def test_get_time_zone_invalid_child_key(self, connect, collection): - ''' - target: get invalid child key - method: call get_config without child_key: time_zone - expected: status not ok - ''' - invalid_configs = ["time", "timezone", "time_zone "] - for config in invalid_configs: - status, config_value = connect.get_config("server_config", config) - assert not status.OK() - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_time_zone_valid(self, connect, collection): - ''' - target: get time_zone - method: call get_config correctly - expected: status ok - ''' - status, config_value = connect.get_config("server_config", "time_zone") - assert status.OK() - assert "UTC" in config_value - - @pytest.mark.level(2) - def test_get_web_port_invalid_child_key(self, connect, collection): - ''' - target: get invalid child key - method: call get_config without child_key: web_port - expected: status not ok - ''' - invalid_configs = ["webport", "Web_port", "web_port "] - for config in invalid_configs: - status, config_value = connect.get_config("server_config", config) - assert not status.OK() - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_web_port_valid(self, connect, collection): - ''' - target: get web_port - method: call get_config correctly - expected: status ok - ''' - status, config_value = connect.get_config("server_config", "web_port") + status, config_value = connect.get_config("network", "http.port") assert status.OK() @@ -1116,21 +877,21 @@ class TestServerConfig: ****************************************************************** """ def gen_valid_timezones(self): - time_zones = [] + timezones = [] for i in range(0, 13): - time_zones.append("UTC+" + str(i)) - time_zones.append("UTC-" + str(i)) - time_zones.extend(["UTC+13", "UTC+14"]) - return time_zones + timezones.append("UTC+" + str(i)) + timezones.append("UTC-" + str(i)) + timezones.extend(["UTC+13", "UTC+14"]) + return timezones @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_server_config_invalid_child_key(self, connect, collection): + def test_set_network_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key expected: status not ok ''' - status, reply = connect.set_config("server_config", "child_key", 19530) + status, reply = connect.set_config("network", "child_key", 19530) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1140,9 +901,9 @@ class TestServerConfig: method: call set_config correctly expected: status ok, set successfully ''' - status, reply = connect.set_config("server_config", "address", '0.0.0.0') + status, reply = connect.set_config("network", "bind.address", '0.0.0.0') assert status.OK() - status, config_value = connect.get_config("server_config", "address") + status, config_value = connect.get_config("network", "bind.address") assert status.OK() assert config_value == '0.0.0.0' @@ -1153,9 +914,9 @@ class TestServerConfig: expected: status ok, set successfully ''' for valid_port in [1025, 65534, 12345, "19530"]: - status, reply = connect.set_config("server_config", "port", valid_port) + status, reply = connect.set_config("network", "http.port", valid_port) assert status.OK() - status, config_value = connect.get_config("server_config", "port") + status, config_value = connect.get_config("network", "http.port") assert status.OK() assert config_value == str(valid_port) @@ -1167,84 +928,34 @@ class TestServerConfig: ''' for invalid_port in [1024, 65535, "0", "True", "19530 ", "100000"]: logging.getLogger().info(invalid_port) - status, reply = connect.set_config("server_config", "port", invalid_port) - assert not status.OK() - - def test_set_deploy_mode_valid(self, connect, collection): - ''' - target: set deploy_mode - method: call set_config correctly - expected: status ok, set successfully - ''' - for valid_deploy_mode in ["cluster_readonly", "cluster_writable", "single"]: - status, reply = connect.set_config("server_config", "deploy_mode", valid_deploy_mode) - assert status.OK() - status, config_value = connect.get_config("server_config", "deploy_mode") - assert status.OK() - assert config_value == valid_deploy_mode - - def test_set_deploy_mode_invalid(self, connect, collection): - ''' - target: set deploy_mode - method: call set_config with invalid deploy_mode - expected: status not ok - ''' - for invalid_deploy_mode in [65535, "0", "Single", "cluster", "cluster-readonly"]: - status, reply = connect.set_config("server_config", "deploy_mode", invalid_deploy_mode) - assert not status.OK() - - def test_set_time_zone_valid(self, connect, collection): - ''' - target: set time_zone - method: call set_config correctly - expected: status ok, set successfully - ''' - for valid_time_zone in self.gen_valid_timezones(): - status, reply = connect.set_config("server_config", "time_zone", valid_time_zone) - assert status.OK() - status, config_value = connect.get_config("server_config", "time_zone") - assert status.OK() - assert config_value == valid_time_zone - # reset to default - status, reply = connect.set_config("server_config", "time_zone", "UTC+8") - assert status.OK() - - def test_set_time_zone_invalid(self, connect, collection): - ''' - target: set time_zone - method: call set_config with invalid time_zone - expected: status not ok - ''' - for invalid_time_zone in ["utc+8", "UTC++8", "GMT+8"]: - logging.getLogger().info(invalid_time_zone) - status, reply = connect.set_config("server_config", "time_zone", invalid_time_zone) + status, reply = connect.set_config("network", "http.port", invalid_port) assert not status.OK() - def test_set_web_port_valid(self, connect, collection): + def test_set_http_port_valid(self, connect, collection): ''' - target: set web_port + target: set http.port method: call set_config correctly expected: status ok, set successfully ''' - for valid_web_port in [1025, 65534, "12345", 19121]: - status, reply = connect.set_config("server_config", "web_port", valid_web_port) + for valid_http_port in [1025, 65534, "12345", 19121]: + status, reply = connect.set_config("network", "http.port", valid_http_port) assert status.OK() - status, config_value = connect.get_config("server_config", "web_port") + status, config_value = connect.get_config("network", "http.port") assert status.OK() - assert config_value == str(valid_web_port) + assert config_value == str(valid_http_port) - def test_set_web_port_invalid(self, connect, collection): + def test_set_http_port_invalid(self, connect, collection): ''' - target: set web_port - method: call set_config with web_port number out of range(1024, 65535) + target: set http.port + method: call set_config with http.port number out of range(1024, 65535) expected: status not ok ''' - for invalid_web_port in [1024, 65535, "0", "True", "19530 ", "1000000"]: - status, reply = connect.set_config("server_config", "web_port", invalid_web_port) + for invalid_http_port in [1024, 65535, "0", "True", "19530 ", "1000000"]: + status, reply = connect.set_config("network", "http.port", invalid_http_port) assert not status.OK() -class TestDBConfig: +class TestGeneralConfig: """ ****************************************************************** The following cases are used to test `get_config` function @@ -1256,135 +967,89 @@ class TestDBConfig: pytest.skip("skip in http mode") @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_backend_url_invalid_child_key(self, connect, collection): + def test_get_meta_uri_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: backend_url + method: call get_config without child_key: meta_uri expected: status not ok ''' - invalid_configs = ["backend_Url", "backend-url", "backend_url "] + invalid_configs = ["backend_Url", "backend-url", "meta_uri "] for config in invalid_configs: - status, config_value = connect.get_config("db_config", config) + status, config_value = connect.get_config("general", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_backend_url_valid(self, connect, collection): + def test_get_meta_uri_valid(self, connect, collection): ''' - target: get backend_url + target: get meta_uri method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("db_config", "backend_url") + status, config_value = connect.get_config("general", "meta_uri") assert status.OK() @pytest.mark.level(2) - def test_get_preload_collection_invalid_child_key(self, connect, collection): + def test_get_timezone_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: preload_collection + method: call get_config without child_key: timezone expected: status not ok ''' - invalid_configs = ["preloadtable", "preload_collection "] + invalid_configs = ["time", "timezone "] for config in invalid_configs: - status, config_value = connect.get_config("db_config", config) + status, config_value = connect.get_config("general", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_preload_collection_valid(self, connect, collection): + def test_get_timezone_valid(self, connect, collection): ''' - target: get preload_collection + target: get timezone method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("db_config", "preload_collection") + status, config_value = connect.get_config("general", "timezone") assert status.OK() - - @pytest.mark.level(2) - def test_get_auto_flush_interval_invalid_child_key(self, connect, collection): - ''' - target: get invalid child key - method: call get_config without child_key: auto_flush_interval - expected: status not ok - ''' - invalid_configs = ["autoFlushInterval", "auto_flush", "auto_flush_interval "] - for config in invalid_configs: - status, config_value = connect.get_config("db_config", config) - assert not status.OK() - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_auto_flush_interval_valid(self, connect, collection): - ''' - target: get auto_flush_interval - method: call get_config correctly - expected: status ok - ''' - status, config_value = connect.get_config("db_config", "auto_flush_interval") - assert status.OK() - + assert "UTC" in config_value """ ****************************************************************** The following cases are used to test `set_config` function ****************************************************************** """ + def test_set_timezone_invalid(self, connect, collection): + ''' + target: set timezone + method: call set_config with invalid timezone + expected: status not ok + ''' + for invalid_timezone in ["utc+8", "UTC++8", "GMT+8"]: + logging.getLogger().info(invalid_timezone) + status, reply = connect.set_config("general", "timezone", invalid_timezone) + assert not status.OK() + @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_db_config_invalid_child_key(self, connect, collection): + def test_set_general_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key expected: status not ok ''' - status, reply = connect.set_config("db_config", "child_key", 1) + status, reply = connect.set_config("general", "child_key", 1) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_backend_url_valid(self, connect, collection): + def test_set_meta_uri_valid(self, connect, collection): ''' - target: set backend_url + target: set meta_uri method: call set_config correctly expected: status ok, set successfully ''' - status, reply = connect.set_config("db_config", "backend_url", 'sqlite://:@:/') + status, reply = connect.set_config("general", "meta_uri", 'sqlite://:@:/') assert status.OK() - status, config_value = connect.get_config("db_config", "backend_url") + status, config_value = connect.get_config("general", "meta_uri") assert status.OK() assert config_value == 'sqlite://:@:/' - def test_set_preload_collection_valid(self, connect, collection): - ''' - target: set preload_collection - method: call set_config correctly - expected: status ok, set successfully - ''' - status, reply = connect.set_config("db_config", "preload_collection", "") - assert status.OK() - status, config_value = connect.get_config("db_config", "preload_collection") - assert status.OK() - assert config_value == "" - - def test_set_auto_flush_interval_valid(self, connect, collection): - ''' - target: set auto_flush_interval - method: call set_config correctly - expected: status ok, set successfully - ''' - for valid_auto_flush_interval in [0, 15, "3", 1]: - status, reply = connect.set_config("db_config", "auto_flush_interval", valid_auto_flush_interval) - assert status.OK() - status, config_value = connect.get_config("db_config", "auto_flush_interval") - assert status.OK() - assert config_value == str(valid_auto_flush_interval) - - def test_set_auto_flush_interval_invalid(self, connect, collection): - ''' - target: set auto_flush_interval - method: call set_config with invalid auto_flush_interval - expected: status not ok - ''' - for invalid_auto_flush_interval in [-1, "1.5", "invalid", "1+2"]: - status, reply = connect.set_config("db_config", "auto_flush_interval", invalid_auto_flush_interval) - assert not status.OK() - class TestStorageConfig: """ @@ -1398,90 +1063,100 @@ class TestStorageConfig: pytest.skip("skip in http mode") @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_primary_path_invalid_child_key(self, connect, collection): + def test_get_path_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: primary_path + method: call get_config without child_key: path expected: status not ok ''' - invalid_configs = ["Primary_path", "primarypath", "primary_path "] + invalid_configs = ["Primary_path", "primarypath", "path "] for config in invalid_configs: - status, config_value = connect.get_config("storage_config", config) + status, config_value = connect.get_config("storage", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_primary_path_valid(self, connect, collection): + def test_get_path_valid(self, connect, collection): ''' - target: get primary_path + target: get path method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("storage_config", "primary_path") + status, config_value = connect.get_config("storage", "path") assert status.OK() @pytest.mark.level(2) - def test_get_secondary_path_invalid_child_key(self, connect, collection): + def test_get_auto_flush_interval_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: secondary_path + method: call get_config without child_key: auto_flush_interval expected: status not ok ''' - invalid_configs = ["secondarypath", "secondary_path "] + invalid_configs = ["autoFlushInterval", "auto_flush", "auto_flush_interval "] for config in invalid_configs: - status, config_value = connect.get_config("storage_config", config) + status, config_value = connect.get_config("storage", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_secondary_path_valid(self, connect, collection): + def test_get_auto_flush_interval_valid(self, connect, collection): ''' - target: get secondary_path + target: get auto_flush_interval method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("storage_config", "secondary_path") + status, config_value = connect.get_config("storage", "auto_flush_interval") assert status.OK() - """ ****************************************************************** The following cases are used to test `set_config` function ****************************************************************** """ @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_storage_config_invalid_child_key(self, connect, collection): + def test_set_storage_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key expected: status not ok ''' - status, reply = connect.set_config("storage_config", "child_key", "") + status, reply = connect.set_config("storage", "child_key", "") assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_primary_path_valid(self, connect, collection): + def test_set_path_valid(self, connect, collection): ''' - target: set primary_path + target: set path method: call set_config correctly expected: status ok, set successfully ''' - status, reply = connect.set_config("storage_config", "primary_path", '/var/lib/milvus') + status, reply = connect.set_config("storage", "path", '/var/lib/milvus') assert status.OK() - status, config_value = connect.get_config("storage_config", "primary_path") + status, config_value = connect.get_config("storage", "path") assert status.OK() assert config_value == '/var/lib/milvus' - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_secondary_path_valid(self, connect, collection): + def test_set_auto_flush_interval_valid(self, connect, collection): ''' - target: set secondary_path + target: set auto_flush_interval method: call set_config correctly expected: status ok, set successfully ''' - status, reply = connect.set_config("storage_config", "secondary_path", "") - assert status.OK() - status, config_value = connect.get_config("storage_config", "secondary_path") - assert status.OK() - assert config_value == "" + for valid_auto_flush_interval in [2, 1]: + logging.getLogger().info(valid_auto_flush_interval) + status, reply = connect.set_config("storage", "auto_flush_interval", valid_auto_flush_interval) + assert status.OK() + status, config_value = connect.get_config("storage", "auto_flush_interval") + assert status.OK() + assert config_value == str(valid_auto_flush_interval) + + def test_set_auto_flush_interval_invalid(self, connect, collection): + ''' + target: set auto_flush_interval + method: call set_config with invalid auto_flush_interval + expected: status not ok + ''' + for invalid_auto_flush_interval in [-1, "1.5", "invalid", "1+2"]: + status, reply = connect.set_config("storage", "auto_flush_interval", invalid_auto_flush_interval) + assert not status.OK() class TestMetricConfig: @@ -1496,25 +1171,25 @@ class TestMetricConfig: pytest.skip("skip in http mode") @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_enable_monitor_invalid_child_key(self, connect, collection): + def test_get_enable_invalid_child_key(self, connect, collection): ''' target: get invalid child key - method: call get_config without child_key: enable_monitor + method: call get_config without child_key: enable expected: status not ok ''' - invalid_configs = ["enablemonitor", "Enable_monitor", "enable_monitor "] + invalid_configs = ["enablemonitor", "Enable_monitor", "enable "] for config in invalid_configs: - status, config_value = connect.get_config("metric_config", config) + status, config_value = connect.get_config("metric", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_enable_monitor_valid(self, connect, collection): + def test_get_enable_valid(self, connect, collection): ''' - target: get enable_monitor + target: get enable method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("metric_config", "enable_monitor") + status, config_value = connect.get_config("metric", "enable") assert status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1526,7 +1201,7 @@ class TestMetricConfig: ''' invalid_configs = ["Address", "addresses", "address "] for config in invalid_configs: - status, config_value = connect.get_config("metric_config", config) + status, config_value = connect.get_config("metric", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1536,7 +1211,7 @@ class TestMetricConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("metric_config", "address") + status, config_value = connect.get_config("metric", "address") assert status.OK() @pytest.mark.level(2) @@ -1548,7 +1223,7 @@ class TestMetricConfig: ''' invalid_configs = ["Port", "PORT", "port "] for config in invalid_configs: - status, config_value = connect.get_config("metric_config", config) + status, config_value = connect.get_config("metric", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1558,7 +1233,7 @@ class TestMetricConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("metric_config", "port") + status, config_value = connect.get_config("metric", "port") assert status.OK() @@ -1568,27 +1243,27 @@ class TestMetricConfig: ****************************************************************** """ @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_metric_config_invalid_child_key(self, connect, collection): + def test_set_metric_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key expected: status not ok ''' - status, reply = connect.set_config("metric_config", "child_key", 19530) + status, reply = connect.set_config("metric", "child_key", 19530) assert not status.OK() - def test_set_enable_monitor_valid(self, connect, collection): + def test_set_enable_valid(self, connect, collection): ''' - target: set enable_monitor + target: set enable method: call set_config correctly expected: status ok, set successfully ''' - for valid_enable_monitor in ["Off", "false", 0, "yes", "On", "true", "1", "NO"]: - status, reply = connect.set_config("metric_config", "enable_monitor", valid_enable_monitor) + for valid_enable in ["Off", "false", 0, "yes", "On", "true", "1", "NO"]: + status, reply = connect.set_config("metric", "enable", valid_enable) assert status.OK() - status, config_value = connect.get_config("metric_config", "enable_monitor") + status, config_value = connect.get_config("metric", "enable") assert status.OK() - assert config_value == str(valid_enable_monitor) + assert config_value == str(valid_enable) @pytest.mark.timeout(CONFIG_TIMEOUT) def test_set_address_valid(self, connect, collection): @@ -1597,9 +1272,9 @@ class TestMetricConfig: method: call set_config correctly expected: status ok, set successfully ''' - status, reply = connect.set_config("metric_config", "address", '127.0.0.1') + status, reply = connect.set_config("metric", "address", '127.0.0.1') assert status.OK() - status, config_value = connect.get_config("metric_config", "address") + status, config_value = connect.get_config("metric", "address") assert status.OK() assert config_value == '127.0.0.1' @@ -1610,84 +1285,84 @@ class TestMetricConfig: expected: status ok, set successfully ''' for valid_port in [1025, 65534, "19530", "9091"]: - status, reply = connect.set_config("metric_config", "port", valid_port) + status, reply = connect.set_config("metric", "port", valid_port) assert status.OK() - status, config_value = connect.get_config("metric_config", "port") + status, config_value = connect.get_config("metric", "port") assert status.OK() assert config_value == str(valid_port) def test_set_port_invalid(self, connect, collection): ''' target: set port - method: call set_config with port number out of range(1024, 65535), or same as web_port number + method: call set_config with port number out of range(1024, 65535), or same as http.port number expected: status not ok ''' for invalid_port in [1024, 65535, "0", "True", "19530 ", "100000"]: - status, reply = connect.set_config("metric_config", "port", invalid_port) - assert not status.OK() - - -class TestTracingConfig: - """ - ****************************************************************** - The following cases are used to test `get_config` function - ****************************************************************** - """ - @pytest.fixture(scope="function", autouse=True) - def skip_http_check(self, args): - if args["handler"] == "HTTP": - pytest.skip("skip in http mode") - - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_json_config_path_invalid_child_key(self, connect, collection): - ''' - target: get invalid child key - method: call get_config without child_key: json_config_path - expected: status not ok - ''' - invalid_configs = ["json_config", "jsonconfigpath", "json_config_path "] - for config in invalid_configs: - status, config_value = connect.get_config("tracing_config", config) + status, reply = connect.set_config("metric", "port", invalid_port) assert not status.OK() - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_get_json_config_path_valid(self, connect, collection): - ''' - target: get json_config_path - method: call get_config correctly - expected: status ok - ''' - status, config_value = connect.get_config("tracing_config", "json_config_path") - assert status.OK() - - """ - ****************************************************************** - The following cases are used to test `set_config` function - ****************************************************************** - """ - @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_tracing_config_invalid_child_key(self, connect, collection): - ''' - target: set invalid child key - method: call set_config with invalid child_key - expected: status not ok - ''' - status, reply = connect.set_config("tracing_config", "child_key", "") - assert not status.OK() - - @pytest.mark.skip(reason="Currently not supported") - def test_set_json_config_path_valid(self, connect, collection): - ''' - target: set json_config_path - method: call set_config correctly - expected: status ok, set successfully - ''' - status, reply = connect.set_config("tracing_config", "json_config_path", "") - assert status.OK() - status, config_value = connect.get_config("tracing_config", "json_config_path") - assert status.OK() - assert config_value == "" +# class TestTracingConfig: +# """ +# ****************************************************************** +# The following cases are used to test `get_config` function +# ****************************************************************** +# """ +# @pytest.fixture(scope="function", autouse=True) +# def skip_http_check(self, args): +# if args["handler"] == "HTTP": +# pytest.skip("skip in http mode") +# +# @pytest.mark.timeout(CONFIG_TIMEOUT) +# def test_get_json_config_path_invalid_child_key(self, connect, collection): +# ''' +# target: get invalid child key +# method: call get_config without child_key: json_config_path +# expected: status not ok +# ''' +# invalid_configs = ["json_config", "jsonconfigpath", "json_config_path "] +# for config in invalid_configs: +# status, config_value = connect.get_config("tracing_config", config) +# assert not status.OK() +# +# @pytest.mark.timeout(CONFIG_TIMEOUT) +# def test_get_json_config_path_valid(self, connect, collection): +# ''' +# target: get json_config_path +# method: call get_config correctly +# expected: status ok +# ''' +# status, config_value = connect.get_config("tracing_config", "json_config_path") +# assert status.OK() +# +# +# """ +# ****************************************************************** +# The following cases are used to test `set_config` function +# ****************************************************************** +# """ +# @pytest.mark.timeout(CONFIG_TIMEOUT) +# def test_set_tracing_config_invalid_child_key(self, connect, collection): +# ''' +# target: set invalid child key +# method: call set_config with invalid child_key +# expected: status not ok +# ''' +# status, reply = connect.set_config("tracing_config", "child_key", "") +# assert not status.OK() +# +# @pytest.mark.skip(reason="Currently not supported") +# def test_set_json_config_path_valid(self, connect, collection): +# ''' +# target: set json_config_path +# method: call set_config correctly +# expected: status ok, set successfully +# ''' +# status, reply = connect.set_config("tracing_config", "json_config_path", "") +# assert status.OK() +# status, config_value = connect.get_config("tracing_config", "json_config_path") +# assert status.OK() +# assert config_value == "" class TestWALConfig: @@ -1710,7 +1385,7 @@ class TestWALConfig: ''' invalid_configs = ["enabled", "Enable", "enable "] for config in invalid_configs: - status, config_value = connect.get_config("wal_config", config) + status, config_value = connect.get_config("wal", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1720,7 +1395,7 @@ class TestWALConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("wal_config", "enable") + status, config_value = connect.get_config("wal", "enable") assert status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1732,7 +1407,7 @@ class TestWALConfig: ''' invalid_configs = ["recovery-error-ignore", "Recovery_error_ignore", "recovery_error_ignore "] for config in invalid_configs: - status, config_value = connect.get_config("wal_config", config) + status, config_value = connect.get_config("wal", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1742,7 +1417,7 @@ class TestWALConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("wal_config", "recovery_error_ignore") + status, config_value = connect.get_config("wal", "recovery_error_ignore") assert status.OK() @pytest.mark.level(2) @@ -1754,7 +1429,7 @@ class TestWALConfig: ''' invalid_configs = ["buffersize", "Buffer_size", "buffer_size "] for config in invalid_configs: - status, config_value = connect.get_config("wal_config", config) + status, config_value = connect.get_config("wal", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1764,7 +1439,7 @@ class TestWALConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("wal_config", "buffer_size") + status, config_value = connect.get_config("wal", "buffer_size") assert status.OK() @pytest.mark.level(2) @@ -1776,7 +1451,7 @@ class TestWALConfig: ''' invalid_configs = ["wal", "Wal_path", "wal_path "] for config in invalid_configs: - status, config_value = connect.get_config("wal_config", config) + status, config_value = connect.get_config("wal", config) assert not status.OK() @pytest.mark.timeout(CONFIG_TIMEOUT) @@ -1786,7 +1461,7 @@ class TestWALConfig: method: call get_config correctly expected: status ok ''' - status, config_value = connect.get_config("wal_config", "wal_path") + status, config_value = connect.get_config("wal", "path") assert status.OK() @@ -1796,13 +1471,13 @@ class TestWALConfig: ****************************************************************** """ @pytest.mark.timeout(CONFIG_TIMEOUT) - def test_set_wal_config_invalid_child_key(self, connect, collection): + def test_set_wal_invalid_child_key(self, connect, collection): ''' target: set invalid child key method: call set_config with invalid child_key expected: status not ok ''' - status, reply = connect.set_config("wal_config", "child_key", 256) + status, reply = connect.set_config("wal", "child_key", 256) assert not status.OK() def test_set_enable_valid(self, connect, collection): @@ -1812,9 +1487,9 @@ class TestWALConfig: expected: status ok, set successfully ''' for valid_enable in ["Off", "false", 0, "no", "On", "true", "1", "YES"]: - status, reply = connect.set_config("wal_config", "enable", valid_enable) + status, reply = connect.set_config("wal", "enable", valid_enable) assert status.OK() - status, config_value = connect.get_config("wal_config", "enable") + status, config_value = connect.get_config("wal", "enable") assert status.OK() assert config_value == str(valid_enable) @@ -1825,9 +1500,9 @@ class TestWALConfig: expected: status ok, set successfully ''' for valid_recovery_error_ignore in ["Off", "false", "0", "no", "On", "true", "1", "YES"]: - status, reply = connect.set_config("wal_config", "recovery_error_ignore", valid_recovery_error_ignore) + status, reply = connect.set_config("wal", "recovery_error_ignore", valid_recovery_error_ignore) assert status.OK() - status, config_value = connect.get_config("wal_config", "recovery_error_ignore") + status, config_value = connect.get_config("wal", "recovery_error_ignore") assert status.OK() assert config_value == valid_recovery_error_ignore @@ -1837,10 +1512,10 @@ class TestWALConfig: method: call set_config correctly expected: status ok, set successfully ''' - for valid_buffer_size in [64, 128, "4096", 1000, "256"]: - status, reply = connect.set_config("wal_config", "buffer_size", valid_buffer_size) + for valid_buffer_size in ["64MB", "128MB", "4096MB", "1000MB", "256MB"]: + status, reply = connect.set_config("wal", "buffer_size", valid_buffer_size) assert status.OK() - status, config_value = connect.get_config("wal_config", "buffer_size") + status, config_value = connect.get_config("wal", "buffer_size") assert status.OK() assert config_value == str(valid_buffer_size) @@ -1851,8 +1526,8 @@ class TestWALConfig: method: call set_config correctly expected: status ok, set successfully ''' - status, reply = connect.set_config("wal_config", "wal_path", "/var/lib/milvus/wal") + status, reply = connect.set_config("wal", "path", "/var/lib/milvus/wal") assert status.OK() - status, config_value = connect.get_config("wal_config", "wal_path") + status, config_value = connect.get_config("wal", "path") assert status.OK() assert config_value == "/var/lib/milvus/wal" diff --git a/tests/milvus_python_test/utils.py b/tests/milvus_python_test/utils.py index 351a76aa..d93504cc 100644 --- a/tests/milvus_python_test/utils.py +++ b/tests/milvus_python_test/utils.py @@ -38,15 +38,15 @@ def get_milvus(host, port, uri=None, handler=None, **kwargs): def disable_flush(connect): - status, reply = connect.set_config("db_config", "auto_flush_interval", big_flush_interval) + status, reply = connect.set_config("storage", "auto_flush_interval", big_flush_interval) assert status.OK() def enable_flush(connect): # reset auto_flush_interval=1 - status, reply = connect.set_config("db_config", "auto_flush_interval", default_flush_interval) + status, reply = connect.set_config("storage", "auto_flush_interval", default_flush_interval) assert status.OK() - status, config_value = connect.get_config("db_config", "auto_flush_interval") + status, config_value = connect.get_config("storage", "auto_flush_interval") assert status.OK() assert config_value == str(default_flush_interval) @@ -470,7 +470,7 @@ def gen_invalid_cache_config(): return invalid_configs -def gen_invalid_engine_config(): +def gen_invalid_gpu_config(): invalid_configs = [ -1, [1,2,3], -- GitLab