diff --git a/CHANGELOG.md b/CHANGELOG.md index 222ac5a8beed408833a40687b1972c2f6280f6b0..486e88c852d2f2a0445a486d94dfb712cc18a3e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,21 +10,35 @@ Please mark all change in change log and use the ticket from JIRA. - \#316 - Some files not merged after vectors added - \#327 - Search does not use GPU when index type is FLAT - \#340 - Test cases run failed on 0.6.0 +- \#353 - Rename config.h.in to version.h.in +- \#374 - sdk_simple return empty result +- \#377 - Create partition success if tag name only contains spaces +- \#397 - sdk_simple return incorrect result +- \#399 - Create partition should be failed if partition tag existed +- \#412 - Message returned is confused when partition created with null partition name +- \#416 - Drop the same partition success repeatally ## Feature - \#12 - Pure CPU version for Milvus - \#77 - Support table partition -- \#226 - Experimental shards middleware for Milvus - \#127 - Support new Index type IVFPQ +- \#226 - Experimental shards middleware for Milvus +- \#227 - Support new index types SPTAG-KDT and SPTAG-BKT +- \#346 - Support build index with multiple gpu ## Improvement +- \#255 - Add ivfsq8 test report detailed version +- \#260 - C++ SDK README +- \#266 - Rpc request source code refactor - \#275 - Rename C++ SDK IndexType - \#284 - Change C++ SDK to shared library -- \#260 - C++ SDK README -- \#314 - add Find FAISS in CMake +- \#306 - Use int64 for all config integer - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md +- \#404 - Add virtual method Init() in Pass abstract class +- \#409 - Add a Fallback pass in optimizer ## Task diff --git a/README.md b/README.md index 311d0d6843e9e29e924838ecb3babf2e807cc229..4a1151dbf80bf67ba9cb6d1d10e97390f89ff20c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Milvus is the world's fastest similarity search engine for massive-scale feature For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/en/aboutmilvus/overview/). -Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and C++ APIs. +Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.0/). diff --git a/README_CN.md b/README_CN.md index 5dad64af9b79c5ff9c149b24f3b0ae5fff3e691b..8f4d4b05aa45080ee6eff8a058ebcbfb23c282a8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,7 +20,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。基于异构众核计算框架设计，成本更低，性能更好。在有限的计算资源下，十亿向量搜索仅毫秒响应。 -Milvus 提供稳定的 Python、Java 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.0/) 获取最新发行版本的 Milvus。 diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f0562ada64abb20f618b66884fd3b0c9df6ef331..bf7a70d11a191c5e77652b6cb0c07e6d2938c0fc 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -2,7 +2,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" -cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string +cron_string = BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string pipeline { agent none @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run GPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('Build') { + stage('GPU Version Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('GPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload GPU Version Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish GPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish GPU Version') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy GPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy GPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("Dev Test") { + stage("GPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup GPU Version Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run CPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build') { + stage('Build CPU Version') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('CPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload CPU Version Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish CPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish CPU Version') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy CPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy CPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("Dev Test") { + stage("CPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup CPU Version Dev") { steps { container('milvus-test-env') { script { diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index f1daaf22ecb6e1892bf9e513e30072a3080493d1..7b479ff44a8c242d34c843bb724f0306e54999b8 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,11 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d357badfd355879489eecabbf7d35b3b2228df93..e27bc6c0a55dd0729bbd2fb515f3ee41c3b97109 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index c1de5907b07f9e933a4dd55a07c698ad0dd901da..08812ae01f1a102adf154d62571566848b8d0a5c 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // if ("${env.BINRARY_VERSION}" == "gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } else { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 27962ccb543f9c8e246753da20d1f7f6cc64c83d..eb5205ad269f2e78126283429b7618a443d74a54 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -16,6 +16,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -23,7 +24,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:gulcjmh" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) @@ -41,6 +42,9 @@ do g) GPU_VERSION="ON"; ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -66,6 +70,7 @@ parameter: -b: core code build directory -f: faiss root path -g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) @@ -74,7 +79,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -96,6 +101,7 @@ CMAKE_CMD="cmake \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b7e4e43e09ee233425d71a04e1ce501..8db8df41db936452793dd3013a2d15ad606d6f69 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f24935f78880ef5bad168755dfeff48fe487b7b1..671330c192e74351547e445e557e9f3d78861489 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -70,7 +70,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL "" endif() message(STATUS "Build version = ${MILVUS_VERSION}") -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h @ONLY) message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390f0ce9776169e92d9033c75f4648c76..41889f5cef063f620dee21e4b837b0abf1a3eee8 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -27,9 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -37,7 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the device used for search computation - - cpu - index_build_device: cpu # CPU used for building index +gpu_resource_config: + enable: false # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d1343ae1520358808cf69cb4fcf33a2c7f..531c633da7ee068ba6cd1bbdec9d0dbd8eeb4758 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -27,10 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] - gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -38,8 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the devices used for search computation, must be in format: cpu or gpux - - cpu +gpu_resource_config: + enable: true # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index aa79c084f1152d0eef03f49ffba2e3e5c8e699a6..14a75bb539f193905696a8cff2c9619249650ed8 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -75,7 +75,13 @@ set(thirdparty_files ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl/request grpc_request_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_impl_files) +set(grpc_server_files + ${grpc_request_files} + ${grpc_impl_files} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index d862bc0393ed63d47661564ac32964dd6d138b41..72229527fab6410e6c54aa7f7dd947bbc26d01e4 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -37,7 +37,7 @@ GpuCacheMgr::GpuCacheMgr() { Status s; int64_t gpu_cache_cap; - s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap); + s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } @@ -45,7 +45,7 @@ GpuCacheMgr::GpuCacheMgr() { cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold); + s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 2559b3a46be478a896df405cfa2efdec9681e397..dd230ce0d1174a3ca2e23f953150676bc6b34e26 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -84,12 +84,12 @@ DBImpl::Start() { return Status::OK(); } - ENGINE_LOG_TRACE << "DB service start"; + // ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { - ENGINE_LOG_TRACE << "StartTimerTasks"; + // ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -114,7 +114,7 @@ DBImpl::Stop() { meta_ptr_->CleanUp(); } - ENGINE_LOG_TRACE << "DB service stop"; + // ENGINE_LOG_TRACE << "DB service stop"; return Status::OK(); } @@ -279,6 +279,11 @@ DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& parti std::string partition_name; auto status = meta_ptr_->GetPartitionName(table_id, partition_tag, partition_name); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; + } + return DropPartition(partition_name); } @@ -553,7 +558,7 @@ DBImpl::StartMetricTask() { return; } - ENGINE_LOG_TRACE << "Start metric task"; + // ENGINE_LOG_TRACE << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -579,7 +584,7 @@ DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUTemperature(); server::Metrics::GetInstance().CPUTemperature(); - ENGINE_LOG_TRACE << "Metric task finished"; + // ENGINE_LOG_TRACE << "Metric task finished"; } Status @@ -751,7 +756,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << "Background compaction thread start"; + // ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -774,7 +779,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << "Background compaction thread exit"; + // ENGINE_LOG_TRACE << " Background compaction thread exit"; } void @@ -807,7 +812,7 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_TRACE << "Background build index thread start"; + // ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -830,7 +835,7 @@ DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_TRACE << "Background build index thread exit"; + // ENGINE_LOG_TRACE << "Background build index thread exit"; } Status @@ -853,8 +858,12 @@ DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vectorShowPartitions(table_id, partiton_array); for (auto& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); for (auto& schema : partiton_array) { - if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, tag)) { + if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) { partition_name_array.insert(schema.table_id_); } } diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a0ff2c51174cb123d9cd6466ae54bde..91459a8d5fac1217e616575790a0d53328cf03b1 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index e2db7179bee15fc46866df8b223be96428921634..ca307b90fc1039b56e1075210d4d3be62be9b452 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -152,7 +152,14 @@ ExecutionEngineImpl::HybridLoad() const { } const std::string key = location_ + ".quantizer"; - std::vector gpus = scheduler::get_gpu_pool(); + + server::Config& config = server::Config::GetInstance(); + std::vector gpus; + Status s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + ENGINE_LOG_ERROR << s.message(); + return; + } // cache hit { @@ -363,6 +370,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { Status ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) { + gpu_num_ = device_id; auto to_index_data = std::make_shared(PhysicalSize()); cache::DataObjPtr obj = std::static_pointer_cast(to_index_data); milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj); @@ -586,12 +594,16 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); - if (!s.ok()) { - return s; + std::vector gpu_ids; + Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); + for (auto id : gpu_ids) { + if (gpu_num_ == id) { + return Status::OK(); + } } - return Status::OK(); + std::string msg = "Invalid gpu_num"; + return Status(SERVER_INVALID_ARGUMENT, msg); } } // namespace engine diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426ec9d30c0f91ec9f5de0537df5266de4..da0e7cfb6489022050c31f462440be3f6417ddfc 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index 28f35e76fc186c772b08f7166fdf4afffa666386..d98b74be7d859e4d8f7a0bd0c46702f8a63d47a4 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -19,7 +19,7 @@ #include "db/Constants.h" #include "db/engine/ExecutionEngine.h" -#include "src/config.h" +#include "src/version.h" #include #include diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index bf8344780643295b17eeb80381c47143c2d70bdc..4406b87f7ea6fc9db536c675642af4bee9c453e1 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -1162,17 +1163,23 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p // not allow create partition under partition if (!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); } - if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if (!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if (!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicate partition is not allowed"); + } + + if (partition_name == "") { + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -1182,9 +1189,14 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -1231,6 +1243,12 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -1240,7 +1258,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& mysqlpp::Query allPartitionsQuery = connectionPtr->query(); allPartitionsQuery << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote - << table_id << " AND partition_tag = " << mysqlpp::quote << tag << " AND state <> " + << table_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allPartitionsQuery.str(); @@ -1252,7 +1270,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& const mysqlpp::Row& resRow = res[0]; resRow["table_id"].to_string(partition_name); } else { - return Status(DB_NOT_FOUND, "Partition " + tag + " of table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Partition " + valid_tag + " of table " + table_id + " not found"); } } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN GET PARTITION NAME", e.what()); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 22e953fe9d96a3ecf8edd7cae0cc24dd8afe3022..12128c074d78425c4405fea110aac8a7b2f063ee 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -757,17 +758,23 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& // not allow create partition under partition if(!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); } - if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if(!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if(!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicate partition is not allowed"); + } + if (partition_name == "") { + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -777,9 +784,14 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; + + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } - return CreateTable(table_schema); + return status; } Status @@ -814,13 +826,18 @@ SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + auto name = ConnectorPtr->select(columns(&TableSchema::table_id_), where(c(&TableSchema::owner_table_) == table_id - and c(&TableSchema::partition_tag_) == tag)); + and c(&TableSchema::partition_tag_) == valid_tag)); if (name.size() > 0) { partition_name = std::get<0>(name[0]); } else { - return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + tag + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + valid_tag + " not found"); } } catch (std::exception &e) { return HandleException("Encounter exception when get partition name", e.what()); diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 4b5c1b1de3ef3d01e7427d2b476dd0a856f8fb3a..53453d53aad53484bc5b7797d116b2af86889c9b 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -88,14 +88,14 @@ endif () include(ThirdPartyPackagesCore) if (CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") endif () endif () diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index 6191ecb771f61168dca171982fc76f908a54def5..48c2de8b1b74011ed8f9b9d6deb311b5f55be69a 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include "Log.h" namespace knowhere { @@ -50,6 +52,18 @@ struct Cfg { CheckValid() { return true; } + + void + Dump() { + KNOWHERE_LOG_DEBUG << DumpImpl().str(); + } + + virtual std::stringstream + DumpImpl() { + std::stringstream ss; + ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; + return ss; + } }; using Config = std::shared_ptr; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp index c64baebb93456a21c6773b792ceead78a554ba17..17a93fdcc7b0f11cb7fc61bbd1a6971048fd6bec 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -19,330 +19,330 @@ #include #include +#include #include #include -#include #undef mkdir -#include "knowhere/index/vector_index/IndexSPTAG.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" #include "knowhere/adapter/SptagAdapter.h" #include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" #include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" namespace knowhere { - CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { - if (IndexType == "KDT") { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - index_type_ = SPTAG::IndexAlgoType::KDT; - } else { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - index_type_ = SPTAG::IndexAlgoType::BKT; - } +CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { + if (IndexType == "KDT") { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::KDT; + } else { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::BKT; } - - BinarySet - CPUSPTAGRNG::Serialize() { - std::string index_config; - std::vector index_blobs; - - std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); - std::vector res(buffersize->size() + 1); - for (uint64_t i = 1; i < res.size(); i++) { - res[i] = new char[buffersize->at(i - 1)]; - auto ptr = &res[i][0]; - index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); - } - - index_ptr_->SaveIndex(index_config, index_blobs); - - size_t length = index_config.length(); - char* cstr = new char[length]; - snprintf(cstr, length, "%s", index_config.c_str()); - - BinarySet binary_set; - auto sample = std::make_shared(); - sample.reset(static_cast(index_blobs[0].Data())); - auto tree = std::make_shared(); - tree.reset(static_cast(index_blobs[1].Data())); - auto graph = std::make_shared(); - graph.reset(static_cast(index_blobs[2].Data())); - auto deleteid = std::make_shared(); - deleteid.reset(static_cast(index_blobs[3].Data())); - auto metadata1 = std::make_shared(); - metadata1.reset(static_cast(index_blobs[4].Data())); - auto metadata2 = std::make_shared(); - metadata2.reset(static_cast(index_blobs[5].Data())); - auto config = std::make_shared(); - config.reset(static_cast((void*)cstr)); - - binary_set.Append("samples", sample, index_blobs[0].Length()); - binary_set.Append("tree", tree, index_blobs[1].Length()); - binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); - binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); - binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); - binary_set.Append("config", config, length); - binary_set.Append("graph", graph, index_blobs[2].Length()); - -// MemoryIOWriter writer; -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// len = index_blobs[i].Length(); -// assert(len != 0); -// writer(&len, sizeof(size_t), 1); -// writer(index_blobs[i].Data(), len, 1); -// len = 0; -// } -// writer(&length, sizeof(size_t), 1); -// writer(cstr, length, 1); -// auto data = std::make_shared(); -// data.reset(writer.data_); -// BinarySet binary_set; -// binary_set.Append("sptag", data, writer.total); - -// MemoryIOWriter writer; -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// if (i == 2) continue; -// len = index_blobs[i].Length(); -// assert(len != 0); -// writer(&len, sizeof(size_t), 1); -// writer(index_blobs[i].Data(), len, 1); -// len = 0; -// } -// writer(&length, sizeof(size_t), 1); -// writer(cstr, length, 1); -// auto data = std::make_shared(); -// data.reset(writer.data_); -// BinarySet binary_set; -// binary_set.Append("sptag", data, writer.total); -// auto graph = std::make_shared(); -// graph.reset(static_cast(index_blobs[2].Data())); -// binary_set.Append("graph", graph, index_blobs[2].Length()); - - return binary_set; +} + +BinarySet +CPUSPTAGRNG::Serialize() { + std::string index_config; + std::vector index_blobs; + + std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); + std::vector res(buffersize->size() + 1); + for (uint64_t i = 1; i < res.size(); i++) { + res[i] = new char[buffersize->at(i - 1)]; + auto ptr = &res[i][0]; + index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); } - void - CPUSPTAGRNG::Load(const BinarySet& binary_set) { - std::string index_config; - std::vector index_blobs; - - auto samples = binary_set.GetByName("samples"); - index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); - - auto tree = binary_set.GetByName("tree"); - index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); - - auto graph = binary_set.GetByName("graph"); - index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); - - auto deleteid = binary_set.GetByName("deleteid"); - index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); - - auto metadata1 = binary_set.GetByName("metadata1"); - index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); - - auto metadata2 = binary_set.GetByName("metadata2"); - index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); - - auto config = binary_set.GetByName("config"); - index_config = reinterpret_cast(config->data.get()); - -// std::vector index_blobs; -// auto data = binary_set.GetByName("sptag"); -// MemoryIOReader reader; -// reader.total = data->size; -// reader.data_ = data->data.get(); -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto binary = new uint8_t[len]; -// reader(binary, len, 1); -// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); -// len = 0; -// } -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto config = new char[len]; -// reader(config, len, 1); -// std::string index_config = config; -// delete[] config; - -// std::vector index_blobs; -// auto data = binary_set.GetByName("sptag"); -// MemoryIOReader reader; -// reader.total = data->size; -// reader.data_ = data->data.get(); -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// if (i == 2) { -// auto graph = binary_set.GetByName("graph"); -// index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); -// continue; -// } -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto binary = new uint8_t[len]; -// reader(binary, len, 1); -// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); -// len = 0; -// } -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto config = new char[len]; -// reader(config, len, 1); -// std::string index_config = config; -// delete[] config; - index_ptr_->LoadIndex(index_config, index_blobs); - } + index_ptr_->SaveIndex(index_config, index_blobs); + + size_t length = index_config.length(); + char* cstr = new char[length]; + snprintf(cstr, length, "%s", index_config.c_str()); + + BinarySet binary_set; + auto sample = std::make_shared(); + sample.reset(static_cast(index_blobs[0].Data())); + auto tree = std::make_shared(); + tree.reset(static_cast(index_blobs[1].Data())); + auto graph = std::make_shared(); + graph.reset(static_cast(index_blobs[2].Data())); + auto deleteid = std::make_shared(); + deleteid.reset(static_cast(index_blobs[3].Data())); + auto metadata1 = std::make_shared(); + metadata1.reset(static_cast(index_blobs[4].Data())); + auto metadata2 = std::make_shared(); + metadata2.reset(static_cast(index_blobs[5].Data())); + auto config = std::make_shared(); + config.reset(static_cast((void*)cstr)); + + binary_set.Append("samples", sample, index_blobs[0].Length()); + binary_set.Append("tree", tree, index_blobs[1].Length()); + binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); + binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); + binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); + binary_set.Append("config", config, length); + binary_set.Append("graph", graph, index_blobs[2].Length()); + + // MemoryIOWriter writer; + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // len = index_blobs[i].Length(); + // assert(len != 0); + // writer(&len, sizeof(size_t), 1); + // writer(index_blobs[i].Data(), len, 1); + // len = 0; + // } + // writer(&length, sizeof(size_t), 1); + // writer(cstr, length, 1); + // auto data = std::make_shared(); + // data.reset(writer.data_); + // BinarySet binary_set; + // binary_set.Append("sptag", data, writer.total); + + // MemoryIOWriter writer; + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // if (i == 2) continue; + // len = index_blobs[i].Length(); + // assert(len != 0); + // writer(&len, sizeof(size_t), 1); + // writer(index_blobs[i].Data(), len, 1); + // len = 0; + // } + // writer(&length, sizeof(size_t), 1); + // writer(cstr, length, 1); + // auto data = std::make_shared(); + // data.reset(writer.data_); + // BinarySet binary_set; + // binary_set.Append("sptag", data, writer.total); + // auto graph = std::make_shared(); + // graph.reset(static_cast(index_blobs[2].Data())); + // binary_set.Append("graph", graph, index_blobs[2].Length()); + + return binary_set; +} + +void +CPUSPTAGRNG::Load(const BinarySet& binary_set) { + std::string index_config; + std::vector index_blobs; + + auto samples = binary_set.GetByName("samples"); + index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); + + auto tree = binary_set.GetByName("tree"); + index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); + + auto graph = binary_set.GetByName("graph"); + index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + + auto deleteid = binary_set.GetByName("deleteid"); + index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); + + auto metadata1 = binary_set.GetByName("metadata1"); + index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); + + auto metadata2 = binary_set.GetByName("metadata2"); + index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); + + auto config = binary_set.GetByName("config"); + index_config = reinterpret_cast(config->data.get()); + + // std::vector index_blobs; + // auto data = binary_set.GetByName("sptag"); + // MemoryIOReader reader; + // reader.total = data->size; + // reader.data_ = data->data.get(); + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto binary = new uint8_t[len]; + // reader(binary, len, 1); + // index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); + // len = 0; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto config = new char[len]; + // reader(config, len, 1); + // std::string index_config = config; + // delete[] config; + + // std::vector index_blobs; + // auto data = binary_set.GetByName("sptag"); + // MemoryIOReader reader; + // reader.total = data->size; + // reader.data_ = data->data.get(); + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // if (i == 2) { + // auto graph = binary_set.GetByName("graph"); + // index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + // continue; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto binary = new uint8_t[len]; + // reader(binary, len, 1); + // index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); + // len = 0; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto config = new char[len]; + // reader(config, len, 1); + // std::string index_config = config; + // delete[] config; + index_ptr_->LoadIndex(index_config, index_blobs); +} // PreprocessorPtr // CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { // return std::make_shared(); //} - IndexModelPtr - CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { - SetParameters(train_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->BuildIndex(vectorset, metaset); - - // TODO: return IndexModelPtr - return nullptr; - } - - void - CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { - // SetParameters(add_config); - // DatasetPtr dataset = origin->Clone(); - // - // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // // && preprocessor_) { - // // preprocessor_->Preprocess(dataset); - // //} - // - // auto vectorset = ConvertToVectorSet(dataset); - // auto metaset = ConvertToMetadataSet(dataset); - // index_ptr_->AddIndex(vectorset, metaset); - } - - void - CPUSPTAGRNG::SetParameters(const Config& config) { +IndexModelPtr +CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { + SetParameters(train_config); + DatasetPtr dataset = origin->Clone(); + + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // && preprocessor_) { + // preprocessor_->Preprocess(dataset); + //} + + auto vectorset = ConvertToVectorSet(dataset); + auto metaset = ConvertToMetadataSet(dataset); + index_ptr_->BuildIndex(vectorset, metaset); + + // TODO: return IndexModelPtr + return nullptr; +} + +void +CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { + // SetParameters(add_config); + // DatasetPtr dataset = origin->Clone(); + // + // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // // && preprocessor_) { + // // preprocessor_->Preprocess(dataset); + // //} + // + // auto vectorset = ConvertToVectorSet(dataset); + // auto metaset = ConvertToMetadataSet(dataset); + // index_ptr_->AddIndex(vectorset, metaset); +} + +void +CPUSPTAGRNG::SetParameters(const Config& config) { #define Assign(param_name, str_name) \ conf->param_name == INVALID_VALUE ? index_ptr_->SetParameter(str_name, std::to_string(build_cfg->param_name)) \ : index_ptr_->SetParameter(str_name, std::to_string(conf->param_name)) - if (index_type_ == SPTAG::IndexAlgoType::KDT) { - auto conf = std::dynamic_pointer_cast(config); - auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); - - Assign(kdtnumber, "KDTNumber"); - Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); - Assign(samples, "Samples"); - Assign(tptnumber, "TPTNumber"); - Assign(tptleafsize, "TPTLeafSize"); - Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); - Assign(neighborhoodsize, "NeighborhoodSize"); - Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); - Assign(graphcefscale, "GraphCEFScale"); - Assign(refineiterations, "RefineIterations"); - Assign(cef, "CEF"); - Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); - Assign(numofthreads, "NumberOfThreads"); - Assign(maxcheck, "MaxCheck"); - Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); - Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); - Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); - } else { - auto conf = std::dynamic_pointer_cast(config); - auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); - - Assign(bktnumber, "BKTNumber"); - Assign(bktkmeansk, "BKTKMeansK"); - Assign(bktleafsize, "BKTLeafSize"); - Assign(samples, "Samples"); - Assign(tptnumber, "TPTNumber"); - Assign(tptleafsize, "TPTLeafSize"); - Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); - Assign(neighborhoodsize, "NeighborhoodSize"); - Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); - Assign(graphcefscale, "GraphCEFScale"); - Assign(refineiterations, "RefineIterations"); - Assign(cef, "CEF"); - Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); - Assign(numofthreads, "NumberOfThreads"); - Assign(maxcheck, "MaxCheck"); - Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); - Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); - Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); - } + if (index_type_ == SPTAG::IndexAlgoType::KDT) { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); + + Assign(kdtnumber, "KDTNumber"); + Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } else { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); + + Assign(bktnumber, "BKTNumber"); + Assign(bktkmeansk, "BKTKMeansK"); + Assign(bktleafsize, "BKTLeafSize"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); } - - DatasetPtr - CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { - SetParameters(config); - auto tensor = dataset->tensor()[0]; - auto p = (float*)tensor->raw_mutable_data(); - for (auto i = 0; i < 10; ++i) { - for (auto j = 0; j < 10; ++j) { - std::cout << p[i * 10 + j] << " "; - } - std::cout << std::endl; +} + +DatasetPtr +CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { + SetParameters(config); + auto tensor = dataset->tensor()[0]; + auto p = (float*)tensor->raw_mutable_data(); + for (auto i = 0; i < 10; ++i) { + for (auto j = 0; j < 10; ++j) { + std::cout << p[i * 10 + j] << " "; } - std::vector query_results = ConvertToQueryResult(dataset, config); - -#pragma omp parallel for - for (auto i = 0; i < query_results.size(); ++i) { - auto target = (float*)query_results[i].GetTarget(); - std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; - index_ptr_->SearchIndex(query_results[i]); - } - - return ConvertToDataset(query_results); - } - - int64_t - CPUSPTAGRNG::Count() { - return index_ptr_->GetNumSamples(); + std::cout << std::endl; } + std::vector query_results = ConvertToQueryResult(dataset, config); - int64_t - CPUSPTAGRNG::Dimension() { - return index_ptr_->GetFeatureDim(); - } - - VectorIndexPtr - CPUSPTAGRNG::Clone() { - KNOWHERE_THROW_MSG("not support"); - } - - void - CPUSPTAGRNG::Seal() { - return; // do nothing - } - - BinarySet - CPUSPTAGRNGIndexModel::Serialize() { - // KNOWHERE_THROW_MSG("not support"); // not support +#pragma omp parallel for + for (auto i = 0; i < query_results.size(); ++i) { + auto target = (float*)query_results[i].GetTarget(); + std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; + index_ptr_->SearchIndex(query_results[i]); } - void - CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { - // KNOWHERE_THROW_MSG("not support"); // not support - } + return ConvertToDataset(query_results); +} + +int64_t +CPUSPTAGRNG::Count() { + return index_ptr_->GetNumSamples(); +} + +int64_t +CPUSPTAGRNG::Dimension() { + return index_ptr_->GetFeatureDim(); +} + +VectorIndexPtr +CPUSPTAGRNG::Clone() { + KNOWHERE_THROW_MSG("not support"); +} + +void +CPUSPTAGRNG::Seal() { + return; // do nothing +} + +BinarySet +CPUSPTAGRNGIndexModel::Serialize() { + // KNOWHERE_THROW_MSG("not support"); // not support +} + +void +CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { + // KNOWHERE_THROW_MSG("not support"); // not support +} } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h index c78807a3ee0d843abb2c522b1c939bcf8377b4e4..01380ce943fe4b9b00fcc20644be3a30aa16205d 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h @@ -28,66 +28,66 @@ namespace knowhere { - class CPUSPTAGRNG : public VectorIndex { - public: - explicit CPUSPTAGRNG(const std::string& IndexType); +class CPUSPTAGRNG : public VectorIndex { + public: + explicit CPUSPTAGRNG(const std::string& IndexType); - public: - BinarySet - Serialize() override; + public: + BinarySet + Serialize() override; - VectorIndexPtr - Clone() override; + VectorIndexPtr + Clone() override; - void - Load(const BinarySet& index_array) override; + void + Load(const BinarySet& index_array) override; - public: - // PreprocessorPtr - // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; + public: + // PreprocessorPtr + // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; - int64_t - Count() override; + int64_t + Count() override; - int64_t - Dimension() override; + int64_t + Dimension() override; - IndexModelPtr - Train(const DatasetPtr& dataset, const Config& config) override; + IndexModelPtr + Train(const DatasetPtr& dataset, const Config& config) override; - void - Add(const DatasetPtr& dataset, const Config& config) override; + void + Add(const DatasetPtr& dataset, const Config& config) override; - DatasetPtr - Search(const DatasetPtr& dataset, const Config& config) override; + DatasetPtr + Search(const DatasetPtr& dataset, const Config& config) override; - void - Seal() override; + void + Seal() override; - private: - void - SetParameters(const Config& config); + private: + void + SetParameters(const Config& config); - private: - PreprocessorPtr preprocessor_; - std::shared_ptr index_ptr_; - SPTAG::IndexAlgoType index_type_; - }; + private: + PreprocessorPtr preprocessor_; + std::shared_ptr index_ptr_; + SPTAG::IndexAlgoType index_type_; +}; - using CPUSPTAGRNGPtr = std::shared_ptr; +using CPUSPTAGRNGPtr = std::shared_ptr; - class CPUSPTAGRNGIndexModel : public IndexModel { - public: - BinarySet - Serialize() override; +class CPUSPTAGRNGIndexModel : public IndexModel { + public: + BinarySet + Serialize() override; - void - Load(const BinarySet& binary) override; + void + Load(const BinarySet& binary) override; - private: - std::shared_ptr index_; - }; + private: + std::shared_ptr index_; +}; - using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; +using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 20f3388174a1572b4b35d82ab0ba451fa4c6f774..6e9ee55658ba484eeed4fea3d97d0e0326aef27c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } +std::stringstream +IVFCfg::DumpImpl() { + auto ss = Cfg::DumpImpl(); + ss << ", nlist: " << nlist << ", nprobe: " << nprobe; + return ss; +} + +std::stringstream +IVFSQCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", nbits: " << nbits; + return ss; +} + +std::stringstream +NSGCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree + << ", candidate: " << candidate_pool_size; + return ss; +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index 9bb3cac303b9fcec5f91a6aabc1dd776ed0ce7a1..e30088ecdf3fa091a0fdae74641d801de2f0f744 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -79,6 +79,9 @@ struct IVFCfg : public Cfg { IVFCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -95,6 +98,9 @@ struct IVFSQCfg : public IVFCfg { : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } + std::stringstream + DumpImpl() override; + IVFSQCfg() = default; bool @@ -145,6 +151,9 @@ struct NSGCfg : public IVFCfg { NSGCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp index f6e1409abdfce9fd38e02518261fb968c22c19a1..836f204c77c88cea8864133a33a6e1c551a169bb 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp @@ -21,55 +21,55 @@ namespace knowhere { - const KDTConfig& - SPTAGParameterMgr::GetKDTParameters() { - return kdt_config_; - } +const KDTConfig& +SPTAGParameterMgr::GetKDTParameters() { + return kdt_config_; +} - const BKTConfig& - SPTAGParameterMgr::GetBKTParameters() { - return bkt_config_; - } +const BKTConfig& +SPTAGParameterMgr::GetBKTParameters() { + return bkt_config_; +} - SPTAGParameterMgr::SPTAGParameterMgr() { - kdt_config_ = std::make_shared(); - kdt_config_->kdtnumber = 1; - kdt_config_->numtopdimensionkdtsplit = 5; - kdt_config_->samples = 100; - kdt_config_->tptnumber = 1; - kdt_config_->tptleafsize = 2000; - kdt_config_->numtopdimensiontptsplit = 5; - kdt_config_->neighborhoodsize = 32; - kdt_config_->graphneighborhoodscale = 2; - kdt_config_->graphcefscale = 2; - kdt_config_->refineiterations = 0; - kdt_config_->cef = 1000; - kdt_config_->maxcheckforrefinegraph = 10000; - kdt_config_->numofthreads = 1; - kdt_config_->maxcheck = 8192; - kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; - kdt_config_->numberofinitialdynamicpivots = 50; - kdt_config_->numberofotherdynamicpivots = 4; +SPTAGParameterMgr::SPTAGParameterMgr() { + kdt_config_ = std::make_shared(); + kdt_config_->kdtnumber = 1; + kdt_config_->numtopdimensionkdtsplit = 5; + kdt_config_->samples = 100; + kdt_config_->tptnumber = 1; + kdt_config_->tptleafsize = 2000; + kdt_config_->numtopdimensiontptsplit = 5; + kdt_config_->neighborhoodsize = 32; + kdt_config_->graphneighborhoodscale = 2; + kdt_config_->graphcefscale = 2; + kdt_config_->refineiterations = 0; + kdt_config_->cef = 1000; + kdt_config_->maxcheckforrefinegraph = 10000; + kdt_config_->numofthreads = 1; + kdt_config_->maxcheck = 8192; + kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + kdt_config_->numberofinitialdynamicpivots = 50; + kdt_config_->numberofotherdynamicpivots = 4; - bkt_config_ = std::make_shared(); - bkt_config_->bktnumber = 1; - bkt_config_->bktkmeansk = 32; - bkt_config_->bktleafsize = 8; - bkt_config_->samples = 100; - bkt_config_->tptnumber = 1; - bkt_config_->tptleafsize = 2000; - bkt_config_->numtopdimensiontptsplit = 5; - bkt_config_->neighborhoodsize = 32; - bkt_config_->graphneighborhoodscale = 2; - bkt_config_->graphcefscale = 2; - bkt_config_->refineiterations = 0; - bkt_config_->cef = 1000; - bkt_config_->maxcheckforrefinegraph = 10000; - bkt_config_->numofthreads = 1; - bkt_config_->maxcheck = 8192; - bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; - bkt_config_->numberofinitialdynamicpivots = 50; - bkt_config_->numberofotherdynamicpivots = 4; - } + bkt_config_ = std::make_shared(); + bkt_config_->bktnumber = 1; + bkt_config_->bktkmeansk = 32; + bkt_config_->bktleafsize = 8; + bkt_config_->samples = 100; + bkt_config_->tptnumber = 1; + bkt_config_->tptleafsize = 2000; + bkt_config_->numtopdimensiontptsplit = 5; + bkt_config_->neighborhoodsize = 32; + bkt_config_->graphneighborhoodscale = 2; + bkt_config_->graphcefscale = 2; + bkt_config_->refineiterations = 0; + bkt_config_->cef = 1000; + bkt_config_->maxcheckforrefinegraph = 10000; + bkt_config_->numofthreads = 1; + bkt_config_->maxcheck = 8192; + bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + bkt_config_->numberofinitialdynamicpivots = 50; + bkt_config_->numberofotherdynamicpivots = 4; +} -} // namespace knowhere \ No newline at end of file +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h index d215d9e2750b746b6ad6f081caffc911842ebace..6a6f7c48d15f5a5733b9888444b1c49e50485ae8 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h @@ -27,35 +27,35 @@ namespace knowhere { - using KDTConfig = std::shared_ptr; - using BKTConfig = std::shared_ptr; +using KDTConfig = std::shared_ptr; +using BKTConfig = std::shared_ptr; - class SPTAGParameterMgr { - public: - const KDTConfig& - GetKDTParameters(); +class SPTAGParameterMgr { + public: + const KDTConfig& + GetKDTParameters(); - const BKTConfig& - GetBKTParameters(); + const BKTConfig& + GetBKTParameters(); - public: - static SPTAGParameterMgr& - GetInstance() { - static SPTAGParameterMgr instance; - return instance; - } + public: + static SPTAGParameterMgr& + GetInstance() { + static SPTAGParameterMgr instance; + return instance; + } - SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; + SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; - SPTAGParameterMgr& - operator=(const SPTAGParameterMgr&) = delete; + SPTAGParameterMgr& + operator=(const SPTAGParameterMgr&) = delete; - private: - SPTAGParameterMgr(); + private: + SPTAGParameterMgr(); - private: - KDTConfig kdt_config_; - BKTConfig bkt_config_; - }; + private: + KDTConfig kdt_config_; + BKTConfig bkt_config_; +}; } // namespace knowhere diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp index a0898a619ba01fe38704bcf1b021946651e8e3fe..d472dab0d7119db2d9e17a74bb915f9f3023baf2 100644 --- a/core/src/index/unittest/test_sptag.cpp +++ b/core/src/index/unittest/test_sptag.cpp @@ -76,8 +76,8 @@ TEST_P(SPTAGTest, sptag_basic) { AssertAnns(result, nq, k); { - //auto ids = result->array()[0]; - //auto dists = result->array()[1]; + // auto ids = result->array()[0]; + // auto dists = result->array()[1]; auto ids = result->ids(); auto dists = result->dist(); @@ -85,8 +85,8 @@ TEST_P(SPTAGTest, sptag_basic) { std::stringstream ss_dist; for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - //ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - //ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; ss_id << *((int64_t*)(ids) + i * k + j) << " "; ss_dist << *((float*)(dists) + i * k + j) << " "; } diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index 110d8fbf922f83a00364d651d76eb12aedc7958c..11dad4a8b9fefdc6c812dcd7611c016c70d0fad0 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -153,7 +153,7 @@ void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } @@ -167,8 +167,8 @@ PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { std::stringstream ss_dist; for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - //ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; - //ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + // ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; + // ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; ss_id << *((int64_t*)(ids) + i * k + j) << " "; ss_dist << *((float*)(dists) + i * k + j) << " "; } diff --git a/core/src/main.cpp b/core/src/main.cpp index 401736c34fe9b83bc291ce944d617ed7a88db56d..b39ba87997fc637753724fe882d65c583d5d8953 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -25,7 +25,7 @@ #include "easyloggingpp/easylogging++.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "src/config.h" +#include "src/version.h" #include "utils/CommonUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 794f6a0f37984071e3dcb40b71d7c17dcbe3c4e6..76c07fe4596beb67ce070e5773112e661822f226 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -85,7 +85,7 @@ JobMgr::worker_function() { } for (auto& task : tasks) { - calculate_path(task); + calculate_path(res_mgr_, task); } // disk resources NEVER be empty. @@ -103,8 +103,8 @@ JobMgr::build_task(const JobPtr& job) { } void -JobMgr::calculate_path(const TaskPtr& task) { - if (task->type_ != TaskType::SearchTask) { +JobMgr::calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task) { + if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { return; } @@ -114,9 +114,9 @@ JobMgr::calculate_path(const TaskPtr& task) { std::vector path; auto spec_label = std::static_pointer_cast(task->label()); - auto src = res_mgr_->GetDiskResources()[0]; + auto src = res_mgr->GetDiskResources()[0]; auto dest = spec_label->resource(); - ShortestPath(src.lock(), dest.lock(), res_mgr_, path); + ShortestPath(src.lock(), dest.lock(), res_mgr, path); task->path() = Path(path, path.size() - 1); } diff --git a/core/src/scheduler/JobMgr.h b/core/src/scheduler/JobMgr.h index fbd6c0ee45cec8625c3bf7a91bd6dfc00aedb6b0..af072614b52331cd370ebac2c4789057e149d614 100644 --- a/core/src/scheduler/JobMgr.h +++ b/core/src/scheduler/JobMgr.h @@ -59,8 +59,9 @@ class JobMgr : public interface::dumpable { static std::vector build_task(const JobPtr& job); - void - calculate_path(const TaskPtr& task); + public: + static void + calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task); private: bool running_ = false; diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 61e0c0975926043ffecd27e70d633b40550d3dc2..69d293f986891844f953b4ed9f8291e15783e6d7 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -45,18 +45,6 @@ std::mutex BuildMgrInst::mutex_; void load_simple_config() { - server::Config& config = server::Config::GetInstance(); - std::string mode; - config.GetResourceConfigMode(mode); - std::vector pool; - config.GetResourceConfigSearchResources(pool); - - // get resources - auto gpu_ids = get_gpu_pool(); - - int32_t build_gpu_id; - config.GetResourceConfigIndexBuildDevice(build_gpu_id); - // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); @@ -64,26 +52,46 @@ load_simple_config() { ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); ResMgrInst::GetInstance()->Connect("disk", "cpu", io); + // get resources +#ifdef MILVUS_GPU_VERSION + server::Config& config = server::Config::GetInstance(); + std::vector gpu_ids; + config.GetGpuResourceConfigSearchResources(gpu_ids); + std::vector build_gpu_ids; + config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); - bool find_build_gpu_id = false; + + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); + } + } + for (auto& gpu_id : gpu_ids) { ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - if (build_gpu_id == gpu_id) { - find_build_gpu_id = true; - } } - if (not find_build_gpu_id) { + for (auto& not_find_id : not_find_build_ids) { ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); } +#endif } void StartSchedulerService() { load_simple_config(); + OptimizerInst::GetInstance()->Init(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index a3048069f9febe9ec7ea9b5112b3b7b6fab537f9..dc2d5ade355131d3b6a481bc11f936ef475a8cbd 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,10 +21,13 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" -#include "optimizer/HybridPass.h" -#include "optimizer/LargeSQ8HPass.h" -#include "optimizer/OnlyCPUPass.h" -#include "optimizer/OnlyGPUPass.h" +#include "Utils.h" +#include "optimizer/BuildIndexPass.h" +#include "optimizer/FaissFlatPass.h" +#include "optimizer/FaissIVFFlatPass.h" +#include "optimizer/FaissIVFSQ8HPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FallbackPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" @@ -97,21 +100,15 @@ class OptimizerInst { if (instance == nullptr) { std::lock_guard lock(mutex_); if (instance == nullptr) { - server::Config& config = server::Config::GetInstance(); - std::vector search_resources; - bool has_cpu = false; - config.GetResourceConfigSearchResources(search_resources); - for (auto& resource : search_resources) { - if (resource == "cpu") { - has_cpu = true; - } - } - std::vector pass_list; - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared(has_cpu)); +#ifdef MILVUS_GPU_VERSION + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); +#endif + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/Scheduler.cpp b/core/src/scheduler/Scheduler.cpp index 8d2d4406f8ccc3e1d8f7a69f2426818ead8c6c30..68d7457aa950adf294cb73e5357b2a5cd8f86221 100644 --- a/core/src/scheduler/Scheduler.cpp +++ b/core/src/scheduler/Scheduler.cpp @@ -108,10 +108,6 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { auto task_table_type = load_completed_event->task_table_item_->task->label()->Type(); switch (task_table_type) { - case TaskLabelType::DEFAULT: { - Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event); - break; - } case TaskLabelType::SPECIFIED_RESOURCE: { Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; diff --git a/core/src/scheduler/TaskCreator.cpp b/core/src/scheduler/TaskCreator.cpp index 40cfa9aac694649d07a25ab59d134d2981fd1ea8..30b76cc5bb282d10631ab723656bf9d77e5baab4 100644 --- a/core/src/scheduler/TaskCreator.cpp +++ b/core/src/scheduler/TaskCreator.cpp @@ -18,7 +18,6 @@ #include "scheduler/TaskCreator.h" #include "SchedInst.h" #include "tasklabel/BroadcastLabel.h" -#include "tasklabel/DefaultLabel.h" #include "tasklabel/SpecResLabel.h" namespace milvus { @@ -47,8 +46,7 @@ std::vector TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; for (auto& index_file : job->index_files()) { - auto label = std::make_shared(); - auto task = std::make_shared(index_file.second, label); + auto task = std::make_shared(index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } @@ -70,12 +68,8 @@ TaskCreator::Create(const DeleteJobPtr& job) { std::vector TaskCreator::Create(const BuildIndexJobPtr& job) { std::vector tasks; - // TODO(yukun): remove "disk" hardcode here - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("disk"); - for (auto& to_index_file : job->to_index_files()) { - auto label = std::make_shared(std::weak_ptr(res_ptr)); - auto task = std::make_shared(to_index_file.second, label); + auto task = std::make_shared(to_index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 2fd573e47a3af61d7ae2905689b5999e8cf86393..527b9adf0323ac2029d099fc081b55359b02a699 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -16,8 +16,6 @@ // under the License. #include "scheduler/Utils.h" -#include "server/Config.h" -#include "utils/Log.h" #ifdef MILVUS_GPU_VERSION #include @@ -46,42 +44,5 @@ get_num_gpu() { return n_devices; } -std::vector -get_gpu_pool() { - std::vector gpu_pool; - - server::Config& config = server::Config::GetInstance(); - std::vector pool; - Status s = config.GetResourceConfigSearchResources(pool); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } - - std::set gpu_ids; - - for (auto& resource : pool) { - if (resource == "cpu") { - continue; - } else { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // error - exit(-1); - } - auto gpu_id = std::stoi(resource.substr(3)); - if (gpu_id >= scheduler::get_num_gpu()) { - // error - exit(-1); - } - gpu_ids.insert(gpu_id); - } - } - - for (auto& gpu_id : gpu_ids) { - gpu_pool.push_back(gpu_id); - } - - return gpu_pool; -} - } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/Utils.h b/core/src/scheduler/Utils.h index 24876eeb965f8d9a05e7d9c5dc122206e048c928..bf88cf0345faa7a7a435f873b3ed4c7426cab871 100644 --- a/core/src/scheduler/Utils.h +++ b/core/src/scheduler/Utils.h @@ -27,8 +27,5 @@ get_current_timestamp(); uint64_t get_num_gpu(); -std::vector -get_gpu_pool(); - } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/action/Action.h b/core/src/scheduler/action/Action.h index f5f828cbf6b3632e7bcc59f848d6c05840562bd9..7391d287a8a0ec0516f4902b6ccc807e6018d827 100644 --- a/core/src/scheduler/action/Action.h +++ b/core/src/scheduler/action/Action.h @@ -36,10 +36,6 @@ class Action { static void PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest); - static void - DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event); - static void SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event); diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index b8a4a1164b255585d57d0cf992aa2ac4c0f8a5e3..f49f1d871f40c5efa91d2866a6977ece46656a93 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -101,110 +101,46 @@ Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) dest->task_table().Put(task_item->task, task_item); } -void -Action::DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event) { - if (not resource->HasExecutor() && event->task_table_item_->Move()) { - auto task_item = event->task_table_item_; - auto task = event->task_table_item_->task; - auto search_task = std::static_pointer_cast(task); - bool moved = false; - - // to support test task, REFACTOR - if (resource->type() == ResourceType::CPU) { - if (auto index_engine = search_task->index_engine_) { - auto location = index_engine->GetLocation(); - - for (auto i = 0; i < res_mgr->GetNumGpuResource(); ++i) { - auto index = milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); - if (index != nullptr) { - moved = true; - auto dest_resource = res_mgr->GetResource(ResourceType::GPU, i); - PushTaskToResource(event->task_table_item_, dest_resource); - break; - } - } - } - } - - if (not moved) { - PushTaskToNeighbourRandomly(task_item, resource); - } - } -} - void Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event) { auto task_item = event->task_table_item_; auto task = event->task_table_item_->task; - if (resource->type() == ResourceType::DISK) { - // step 1: calculate shortest path per resource, from disk to compute resource - auto compute_resources = res_mgr->GetComputeResources(); - std::vector> paths; - std::vector transport_costs; - for (auto& res : compute_resources) { - std::vector path; - uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); - transport_costs.push_back(transport_cost); - paths.emplace_back(path); - } - // if (task->job_.lock()->type() == JobType::SEARCH) { - // auto label = task->label(); - // auto spec_label = std::static_pointer_cast(label); - // if (spec_label->resource().lock()->type() == ResourceType::CPU) { - // std::vector spec_path; - // spec_path.push_back(spec_label->resource().lock()->name()); - // spec_path.push_back(resource->name()); - // task->path() = Path(spec_path, spec_path.size() - 1); - // } else { - // // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost - // uint64_t min_cost = std::numeric_limits::max(); - // uint64_t min_cost_idx = 0; - // for (uint64_t i = 0; i < compute_resources.size(); ++i) { - // if (compute_resources[i]->TotalTasks() == 0) { - // min_cost_idx = i; - // break; - // } - // uint64_t cost = compute_resources[i]->TaskAvgCost() * - // compute_resources[i]->NumOfTaskToExec() + - // transport_costs[i]; - // if (min_cost > cost) { - // min_cost = cost; - // min_cost_idx = i; - // } - // } - // - // // step 3: set path in task - // Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); - // task->path() = task_path; - // } - // - // } else - if (task->job_.lock()->type() == JobType::BUILD) { - // step2: Read device id in config - // get build index gpu resource - server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu; - Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); - - bool find_gpu_res = false; - if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (compute_resources[i]->name() == - res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { - find_gpu_res = true; - Path task_path(paths[i], paths[i].size() - 1); - task->path() = task_path; - break; - } - } - } - if (not find_gpu_res) { - task->path() = Path(paths[0], paths[0].size() - 1); - } - } - } + // if (resource->type() == ResourceType::DISK) { + // // step 1: calculate shortest path per resource, from disk to compute resource + // auto compute_resources = res_mgr->GetComputeResources(); + // std::vector> paths; + // std::vector transport_costs; + // for (auto& res : compute_resources) { + // std::vector path; + // uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); + // transport_costs.push_back(transport_cost); + // paths.emplace_back(path); + // } + // if (task->job_.lock()->type() == JobType::BUILD) { + // // step2: Read device id in config + // // get build index gpu resource + // server::Config& config = server::Config::GetInstance(); + // int32_t build_index_gpu; + // Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + // + // bool find_gpu_res = false; + // if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + // for (uint64_t i = 0; i < compute_resources.size(); ++i) { + // if (compute_resources[i]->name() == + // res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { + // find_gpu_res = true; + // Path task_path(paths[i], paths[i].size() - 1); + // task->path() = task_path; + // break; + // } + // } + // } + // if (not find_gpu_res) { + // task->path() = Path(paths[0], paths[0].size() - 1); + // } + // } + // } if (resource->name() == task->path().Last()) { resource->WakeupExecutor(); diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp similarity index 65% rename from core/src/scheduler/optimizer/OnlyCPUPass.cpp rename to core/src/scheduler/optimizer/BuildIndexPass.cpp index 238a91a82c77c05fe74576e2fedf33123214dbfc..d535b9675fa74a0935b9263c850962cff79b2d80 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -15,32 +15,38 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyCPUPass.h" +#include "scheduler/optimizer/BuildIndexPass.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" -#include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { +void +BuildIndexPass::Init() { + server::Config& config = server::Config::GetInstance(); + std::vector build_resources; + Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + if (!s.ok()) { + throw; + } +} + bool -OnlyCPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask) +BuildIndexPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::BuildIndexTask) return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { - return false; - } - auto gpu_id = get_gpu_pool(); - if (not gpu_id.empty()) + if (build_gpu_ids_.empty()) return false; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + ResourcePtr res_ptr; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; + + specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); return true; } diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h similarity index 86% rename from core/src/scheduler/optimizer/OnlyGPUPass.h rename to core/src/scheduler/optimizer/BuildIndexPass.h index 10d909d30e1888d5025df8a80ee899bc348eafc9..4f7117fc4e07e84314ee0b6a58018b33524ba46f 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -32,20 +32,23 @@ namespace milvus { namespace scheduler { -class OnlyGPUPass : public Pass { +class BuildIndexPass : public Pass { public: - explicit OnlyGPUPass(bool has_cpu); + BuildIndexPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; private: uint64_t specified_gpu_id_ = 0; - bool has_cpu_ = false; + std::vector build_gpu_ids_; }; -using OnlyGPUPassPtr = std::shared_ptr; +using BuildIndexPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp similarity index 59% rename from core/src/scheduler/optimizer/LargeSQ8HPass.cpp rename to core/src/scheduler/optimizer/FaissFlatPass.cpp index b9784e3c0a69b667d1e8d5068bb3ab6afa424a2d..61ca1b9ec98d639f8754df8ee047965d4bf021fa 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/LargeSQ8HPass.h" +#include "scheduler/optimizer/FaissFlatPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" @@ -27,57 +27,41 @@ namespace milvus { namespace scheduler { -LargeSQ8HPass::LargeSQ8HPass() { +void +FaissFlatPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } } bool -LargeSQ8HPass::Run(const TaskPtr& task) { +FaissFlatPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) { return false; } auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { return false; } auto search_job = std::static_pointer_cast(search_task->job_.lock()); - - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - + ResourcePtr res_ptr; if (search_job->nq() < threshold_) { - return false; + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } - - std::vector gpus = scheduler::get_gpu_pool(); - // std::vector all_free_mem; - // for (auto& gpu : gpus) { - // auto cache = cache::GpuCacheMgr::GetInstance(gpu); - // auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); - // all_free_mem.push_back(free_mem); - // } - // - // auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); - // auto best_index = std::distance(all_free_mem.begin(), max_e); - // auto best_device_id = gpus[best_index]; - auto best_device_id = count_ % gpus.size(); - count_++; - - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); - if (not res_ptr) { - SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; - // TODO: throw critical error and exit - return false; - } - - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto label = std::make_shared(res_ptr); task->label() = label; - return true; } diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h similarity index 83% rename from core/src/scheduler/optimizer/LargeSQ8HPass.h rename to core/src/scheduler/optimizer/FaissFlatPass.h index 9d135d413ae3038ca38596d6d90a0e68839335aa..f219bebdf3713809418035af44e0ad223c290f11 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -33,20 +33,24 @@ namespace milvus { namespace scheduler { -class LargeSQ8HPass : public Pass { +class FaissFlatPass : public Pass { public: - LargeSQ8HPass(); + FaissFlatPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; + std::vector gpus; }; -using LargeSQ8HPassPtr = std::shared_ptr; +using FaissFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f1efb374b78d65217d5cd1be6b4600f8ffe3f2f --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFFlatPass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h similarity index 79% rename from core/src/scheduler/optimizer/HybridPass.h rename to core/src/scheduler/optimizer/FaissIVFFlatPass.h index 0d02a8bda9926211a43d92fe30ccc17996f97c8a..2d15539014245afecc7c590190d12f7892675e92 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,16 +33,24 @@ namespace milvus { namespace scheduler { -class HybridPass : public Pass { +class FaissIVFFlatPass : public Pass { public: - HybridPass() = default; + FaissIVFFlatPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using HybridPassPtr = std::shared_ptr; +using FaissIVFFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp similarity index 55% rename from core/src/scheduler/optimizer/OnlyGPUPass.cpp rename to core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index 2a72f9757e9b10b4b08d09597d5454af2ca00cee..a99e861e036edf1b13ef6f76700b3c751d998945 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -15,39 +15,50 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyGPUPass.h" +#include "scheduler/optimizer/FaissIVFSQ8HPass.h" +#include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" namespace milvus { namespace scheduler { -OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) { +void +FaissIVFSQ8HPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); } bool -OnlyGPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask || has_cpu_) +FaissIVFSQ8HPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { return false; + } auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { return false; } - auto gpu_id = get_gpu_pool(); - if (gpu_id.empty()) - return false; - - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); task->label() = label; - - specified_gpu_id_ = specified_gpu_id_++ % gpu_id.size(); return true; } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h new file mode 100644 index 0000000000000000000000000000000000000000..0d2892809fd1ba606b3e655e95040c328c84d53c --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FaissIVFSQ8HPass : public Pass { + public: + FaissIVFSQ8HPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; +}; + +using FaissIVFSQ8HPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp new file mode 100644 index 0000000000000000000000000000000000000000..30dd306b3bde35346f7f44888e0b3838942ad969 --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFSQ8Pass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFSQ8Pass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFSQ8Pass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h similarity index 79% rename from core/src/scheduler/optimizer/OnlyCPUPass.h rename to core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index 76b42e376609977b6e6b4cf852e32f93fbb8f648..e92ea2fe4e7ca0454ade7018c875b3e3ea3f6190 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,16 +33,24 @@ namespace milvus { namespace scheduler { -class OnlyCPUPass : public Pass { +class FaissIVFSQ8Pass : public Pass { public: - OnlyCPUPass() = default; + FaissIVFSQ8Pass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using OnlyCPUPassPtr = std::shared_ptr; +using FaissIVFSQ8PassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2e275ede4bc6b2f14022ea5fb17c2b3fbcdc9041 --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FallbackPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +void +FallbackPass::Init() { +} + +bool +FallbackPass::Run(const TaskPtr& task) { + auto task_type = task->Type(); + if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) { + return false; + } + // NEVER be empty + auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; + auto label = std::make_shared(cpu); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/tasklabel/DefaultLabel.h b/core/src/scheduler/optimizer/FallbackPass.h similarity index 82% rename from core/src/scheduler/tasklabel/DefaultLabel.h rename to core/src/scheduler/optimizer/FallbackPass.h index c2157435757cbb41392a498e9dc82964f7506206..728740d53a7d96307974a0da3df291b248409f40 100644 --- a/core/src/scheduler/tasklabel/DefaultLabel.h +++ b/core/src/scheduler/optimizer/FallbackPass.h @@ -14,23 +14,27 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - #pragma once -#include "TaskLabel.h" - +#include #include +#include "Pass.h" + namespace milvus { namespace scheduler { -class DefaultLabel : public TaskLabel { +class FallbackPass : public Pass { public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { - } -}; + FallbackPass() = default; -using DefaultLabelPtr = std::shared_ptr; + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; +}; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp deleted file mode 100644 index d63fc2e819de149b5d41def74a6df22e606de318..0000000000000000000000000000000000000000 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "scheduler/optimizer/HybridPass.h" -#include "scheduler/SchedInst.h" -#include "scheduler/task/SearchTask.h" -#include "scheduler/tasklabel/SpecResLabel.h" - -namespace milvus { -namespace scheduler { - -bool -HybridPass::Run(const TaskPtr& task) { - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ == (int)engine::EngineType::FAISS_IVFSQ8H) { - // TODO: remove "cpu" hardcode - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - return true; - } - return false; -} - -} // namespace scheduler -} // namespace milvus diff --git a/core/src/scheduler/optimizer/Optimizer.cpp b/core/src/scheduler/optimizer/Optimizer.cpp index 46f24ea712b7d08dfb5010a914dcad2b0f3430fa..c5fa311a27279f528e702ed01cee0be995e0703e 100644 --- a/core/src/scheduler/optimizer/Optimizer.cpp +++ b/core/src/scheduler/optimizer/Optimizer.cpp @@ -20,12 +20,12 @@ namespace milvus { namespace scheduler { -// void -// Optimizer::Init() { -// for (auto& pass : pass_list_) { -// pass->Init(); -// } -// } +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} bool Optimizer::Run(const TaskPtr& task) { diff --git a/core/src/scheduler/optimizer/Optimizer.h b/core/src/scheduler/optimizer/Optimizer.h index bfabbf7de3ef9331405475144c43f6fadb90787c..68b519e115cbf8769ff1339f5e4b84030c4a807c 100644 --- a/core/src/scheduler/optimizer/Optimizer.h +++ b/core/src/scheduler/optimizer/Optimizer.h @@ -38,8 +38,8 @@ class Optimizer { explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { } - // void - // Init(); + void + Init(); bool Run(const TaskPtr& task); diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 016b05e4570f958bedb54a215d1aeb099b735332..36a36a1df592cdf47ef8da099467eb7489eaac99 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,8 @@ namespace scheduler { class Pass { public: - // virtual void - // Init() { - // } + virtual void + Init() = 0; virtual bool Run(const TaskPtr& task) = 0; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index d8602c141e166510ffe9f1c8a9a9bd83470b6c39..f561fa947d670a08c536375c59afaa18336d1790 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -146,8 +146,7 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" - << std::endl; + ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); @@ -179,8 +178,8 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); diff --git a/core/src/scheduler/tasklabel/TaskLabel.h b/core/src/scheduler/tasklabel/TaskLabel.h index d35ce409ffbcbd4503418d9f321525c8b7f8dfe3..33e6eb6e575a09b4e765ea56ef0ccf5c8ed05f60 100644 --- a/core/src/scheduler/tasklabel/TaskLabel.h +++ b/core/src/scheduler/tasklabel/TaskLabel.h @@ -23,7 +23,6 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource SPECIFIED_RESOURCE, // means must executing in special resource BROADCAST, // means all enable-executor resource must execute task }; diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index d9dd9507107a47fc27f2b0e5de4f3db13d613c2b..da5e854e9bb2a4907a24ec882c5d18f8590c4bd4 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -167,7 +167,7 @@ Utils::PrintSearchResult(const std::vector index++; std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { - size_t idx = i * nq + j; + size_t idx = i * topk + j; std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl; } } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98040420a8ba5455ddc74b00efd73bb1..4a9c319b4d431e1441a39c7ac306513ab3586469 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -17,7 +17,7 @@ #include "sdk/grpc/ClientProxy.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "src/config.h" +#include "src/version.h" #include #include @@ -204,9 +204,8 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ if (!id_array.empty()) { /* set user's ids */ auto row_ids = insert_param.mutable_row_id_array(); - row_ids->Reserve(static_cast(id_array.size())); + row_ids->Resize(static_cast(id_array.size()), -1); memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); - client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85784769d39badcc9ba5b9d186a5855b..f3efcff0cc23bf51b4d38518ec245fe6eccb9439 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -163,63 +163,63 @@ Config::ValidateConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity; - s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); + bool cache_insert_data; + s = GetCacheConfigCacheInsertData(cache_insert_data); if (!s.ok()) { return s; } - float cache_gpu_cache_threshold; - s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); + /* engine config */ + int64_t engine_use_blas_threshold; + s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } -#endif - bool cache_insert_data; - s = GetCacheConfigCacheInsertData(cache_insert_data); + int64_t engine_omp_thread_num; + s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - /* engine config */ - int32_t engine_use_blas_threshold; - s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); + int64_t engine_gpu_search_threshold; + s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; - s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; - s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); if (!s.ok()) { return s; } - /* resource config */ - std::string resource_mode; - s = GetResourceConfigMode(resource_mode); + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); if (!s.ok()) { return s; } - std::vector search_resources; - s = GetResourceConfigSearchResources(search_resources); + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); if (!s.ok()) { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); if (!s.ok()) { return s; } +#endif return Status::OK(); } @@ -307,54 +307,54 @@ Config::ResetDefaultConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); + s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!s.ok()) { return s; } - s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); + /* engine config */ + s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } -#endif - s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); if (!s.ok()) { return s; } - /* engine config */ - s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); + s = SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } - s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } - s = SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); + s = SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); if (!s.ok()) { return s; } - /* resource config */ - s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + s = SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + s = SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); if (!s.ok()) { return s; } +#endif return Status::OK(); } @@ -377,7 +377,7 @@ Config::PrintAll() { PrintConfigSection(CONFIG_CACHE); PrintConfigSection(CONFIG_METRIC); PrintConfigSection(CONFIG_ENGINE); - PrintConfigSection(CONFIG_RESOURCE); + PrintConfigSection(CONFIG_GPU_RESOURCE); } //////////////////////////////////////////////////////////////////////////////// @@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -591,52 +591,6 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); - if (!s.ok()) { - return s; - } - - size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; - } - } - return Status::OK(); -} - -Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { - if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - float gpu_cache_threshold = std::stof(value); - if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - return Status::OK(); -} - Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -665,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -687,56 +641,99 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { - if (value != "simple") { - std::string msg = "Invalid resource mode: " + value + ". Possible reason: resource_config.mode is invalid."; +Config::CheckGpuResourceConfigEnable(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + std::string msg = + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status -CheckResource(const std::string& value) { +Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; + Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); + if (!s.ok()) { + return s; + } + + for (int64_t gpu_id : gpu_ids) { + size_t gpu_memory; + if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } + } + } + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + float gpu_cache_threshold = std::stof(value); + if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + +Status +CheckGpuResource(const std::string& value) { std::string s = value; std::transform(s.begin(), s.end(), s.begin(), ::tolower); -#ifdef MILVUS_CPU_VERSION - if (s != "cpu") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s); - } -#else - const std::regex pat("cpu|gpu(\\d+)"); + const std::regex pat("gpu(\\d+)"); std::smatch m; if (!std::regex_match(s, m, pat)) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources is not in the format of cpux or gpux"; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } if (s.compare(0, 3, "gpu") == 0) { int32_t gpu_index = std::stoi(s.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config does not match with the hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } -#endif return Status::OK(); } Status -Config::CheckResourceConfigSearchResources(const std::vector& value) { +Config::CheckGpuResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { std::string msg = - "Invalid search resource. " - "Possible reason: resource_config.search_resources is empty."; + "Invalid gpu search resource. " + "Possible reason: gpu_resource_config.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } for (auto& resource : value) { - auto status = CheckResource(resource); + auto status = CheckGpuResource(resource); if (!status.ok()) { return Status(SERVER_INVALID_ARGUMENT, status.message()); } @@ -745,11 +742,21 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckGpuResourceConfigBuildIndexResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid gpu build index resource. " + "Possible reason: gpu_resource_config.build_index_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + for (auto& resource : value) { + auto status = CheckGpuResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } } + return Status::OK(); } @@ -848,40 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -898,7 +902,6 @@ Config::GetMetricConfigEnableMonitor(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -924,8 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -937,119 +939,160 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { if (!s.ok()) { return s; } - value = std::stof(str); return Status::OK(); } Status -Config::GetCacheConfigGpuCacheCapacity(int64_t& value) { +Config::GetCacheConfigCacheInsertData(bool& value) { std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - Status s = CheckCacheConfigGpuCacheCapacity(str); + GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); } Status -Config::GetCacheConfigGpuCacheThreshold(float& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - Status s = CheckCacheConfigGpuCacheThreshold(str); + GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); + Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - - value = std::stof(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetCacheConfigCacheInsertData(bool& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - Status s = CheckCacheConfigCacheInsertData(str); +Config::GetEngineConfigOmpThreadNum(int64_t& value) { + std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = - GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); - Status s = CheckEngineConfigUseBlasThreshold(str); + GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); + Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - Status s = CheckEngineConfigOmpThreadNum(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); - Status s = CheckEngineConfigGpuSearchThreshold(str); +Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - - value = std::stoi(str); + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, + CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); + s = CheckGpuResourceConfigCacheCapacity(str); + if (!s.ok()) { + return s; + } + value = std::stoll(str); return Status::OK(); } Status -Config::GetResourceConfigMode(std::string& value) { - value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return CheckResourceConfigMode(value); +Config::GetGpuResourceConfigCacheThreshold(float& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, + CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); + s = CheckGpuResourceConfigCacheThreshold(str); + if (!s.ok()) { + return s; + } + value = std::stof(str); + return Status::OK(); } Status -Config::GetResourceConfigSearchResources(std::vector& value) { - std::string str = - GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); - return CheckResourceConfigSearchResources(value); +Config::GetGpuResourceConfigSearchResources(std::vector& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoll(res.substr(3))); + } + return Status::OK(); } Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); + GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigBuildIndexResources(res_vec); if (!s.ok()) { return s; } - - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); + for (std::string& res : res_vec) { + value.push_back(std::stoll(res.substr(3))); } - return Status::OK(); } @@ -1061,7 +1104,6 @@ Config::SetServerConfigAddress(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); return Status::OK(); } @@ -1072,7 +1114,6 @@ Config::SetServerConfigPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); return Status::OK(); } @@ -1083,7 +1124,6 @@ Config::SetServerConfigDeployMode(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -1094,7 +1134,6 @@ Config::SetServerConfigTimeZone(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); return Status::OK(); } @@ -1106,7 +1145,6 @@ Config::SetDBConfigPrimaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } @@ -1117,7 +1155,6 @@ Config::SetDBConfigSecondaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -1128,7 +1165,6 @@ Config::SetDBConfigBackendUrl(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); return Status::OK(); } @@ -1139,7 +1175,6 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); return Status::OK(); } @@ -1150,7 +1185,6 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); return Status::OK(); } @@ -1161,7 +1195,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -1173,7 +1206,6 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1184,7 +1216,6 @@ Config::SetMetricConfigCollector(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1195,7 +1226,6 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1207,7 +1237,6 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1218,40 +1247,16 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } -Status -Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { - Status s = CheckCacheConfigGpuCacheCapacity(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - return Status::OK(); -} - -Status -Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { - Status s = CheckCacheConfigGpuCacheThreshold(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - return Status::OK(); -} - Status Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1263,7 +1268,6 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1274,7 +1278,6 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1285,47 +1288,64 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); return Status::OK(); } -/* resource config */ +/* gpu resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { - Status s = CheckResourceConfigMode(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } Status -Config::SetResourceConfigSearchResources(const std::string& value) { - std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); - - Status s = CheckResourceConfigSearchResources(res_vec); +Config::SetGpuResourceConfigCacheCapacity(const std::string& value) { + Status s = CheckGpuResourceConfigCacheCapacity(value); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); return Status::OK(); } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { + Status s = CheckGpuResourceConfigCacheThreshold(value); if (!s.ok()) { return s; } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); + return Status::OK(); +} - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); +Status +Config::SetGpuResourceConfigSearchResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); return Status::OK(); } +Status +Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigBuildIndexResources(res_vec); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); + return Status::OK(); +} // namespace server + } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd80534f88990245cd30eb736f786404bcfc..0907080a6fa0c6528e0305f3834cae599728e5b5 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -59,12 +59,8 @@ static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "4"; -static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -87,26 +83,23 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; -/* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; - -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; -#else -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; -#endif - -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; +/* gpu resource config */ +static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; +#ifdef MILVUS_GPU_VERSION +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif -const int32_t CPU_DEVICE_ID = -1; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_GPU_RESOURCE_DELIMITER = ","; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT = "gpu0"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_resources"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT = "gpu0"; class Config { public: @@ -170,10 +163,6 @@ class Config { Status CheckCacheConfigCpuCacheThreshold(const std::string& value); Status - CheckCacheConfigGpuCacheCapacity(const std::string& value); - Status - CheckCacheConfigGpuCacheThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -184,13 +173,17 @@ class Config { Status CheckEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ + Status + CheckGpuResourceConfigEnable(const std::string& value); + Status + CheckGpuResourceConfigCacheCapacity(const std::string& value); Status - CheckResourceConfigMode(const std::string& value); + CheckGpuResourceConfigCacheThreshold(const std::string& value); Status - CheckResourceConfigSearchResources(const std::vector& value); + CheckGpuResourceConfigSearchResources(const std::vector& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckGpuResourceConfigBuildIndexResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -217,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -239,27 +232,27 @@ class Config { Status GetCacheConfigCpuCacheThreshold(float& value); Status - GetCacheConfigGpuCacheCapacity(int64_t& value); - Status - GetCacheConfigGpuCacheThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); - /* resource config */ + /* gpu resource config */ + Status + GetGpuResourceConfigEnable(bool& value); + Status + GetGpuResourceConfigCacheCapacity(int64_t& value); Status - GetResourceConfigMode(std::string& value); + GetGpuResourceConfigCacheThreshold(float& value); Status - GetResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigSearchResources(std::vector& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ @@ -300,10 +293,6 @@ class Config { Status SetCacheConfigCpuCacheThreshold(const std::string& value); Status - SetCacheConfigGpuCacheCapacity(const std::string& value); - Status - SetCacheConfigGpuCacheThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -314,13 +303,17 @@ class Config { Status SetEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ + Status + SetGpuResourceConfigEnable(const std::string& value); + Status + SetGpuResourceConfigCacheCapacity(const std::string& value); Status - SetResourceConfigMode(const std::string& value); + SetGpuResourceConfigCacheThreshold(const std::string& value); Status - SetResourceConfigSearchResources(const std::string& value); + SetGpuResourceConfigSearchResources(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetGpuResourceConfigBuildIndexResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc6937f12a213bf569e65b6292c3636e09..f7a43d671c44d73512daba82471292d70acaa395 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index eb2b07782968014141204b0a1d7160256567669c..567650472287c46da8ed1a6532f857931eb75c77 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -25,7 +25,7 @@ #include "server/DBWrapper.h" #include "server/Server.h" #include "server/grpc_impl/GrpcServer.h" -#include "src/config.h" +#include "src/version.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.cpp b/core/src/server/grpc_impl/GrpcRequestHandler.cpp index a6680745ad460aeb748062b81b187aa1cd530b48..014a0e2180f5ea979da075089e4f2bbff23360de 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -16,7 +16,24 @@ // under the License. #include "server/grpc_impl/GrpcRequestHandler.h" -#include "server/grpc_impl/GrpcRequestTask.h" +#include "server/grpc_impl/GrpcRequestScheduler.h" +#include "server/grpc_impl/request/CmdRequest.h" +#include "server/grpc_impl/request/CountTableRequest.h" +#include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/grpc_impl/request/CreatePartitionRequest.h" +#include "server/grpc_impl/request/CreateTableRequest.h" +#include "server/grpc_impl/request/DeleteByDateRequest.h" +#include "server/grpc_impl/request/DescribeIndexRequest.h" +#include "server/grpc_impl/request/DescribeTableRequest.h" +#include "server/grpc_impl/request/DropIndexRequest.h" +#include "server/grpc_impl/request/DropPartitionRequest.h" +#include "server/grpc_impl/request/DropTableRequest.h" +#include "server/grpc_impl/request/HasTableRequest.h" +#include "server/grpc_impl/request/InsertRequest.h" +#include "server/grpc_impl/request/PreloadTableRequest.h" +#include "server/grpc_impl/request/SearchRequest.h" +#include "server/grpc_impl/request/ShowPartitionsRequest.h" +#include "server/grpc_impl/request/ShowTablesRequest.h" #include "utils/TimeRecorder.h" #include @@ -28,8 +45,8 @@ namespace grpc { ::grpc::Status GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreateTableTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreateTableRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } @@ -37,9 +54,9 @@ GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus:: GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response) { bool has_table = false; - BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); + BaseRequestPtr request_ptr = HasTableRequest::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_bool_reply(has_table); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -49,25 +66,25 @@ GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grp ::grpc::Status GrpcRequestHandler::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name()); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = DropTableRequest::Create(request->table_name()); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreateIndexTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreateIndexRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) { - BaseTaskPtr task_ptr = InsertTask::Create(request, response); + BaseRequestPtr request_ptr = InsertRequest::Create(request, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -77,9 +94,9 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc: GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) { std::vector file_id_array; - BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); + BaseRequestPtr request_ptr = SearchRequest::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -93,9 +110,10 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus file_id_array.push_back(request->file_id_array(i)); } ::milvus::grpc::SearchInFilesParam* request_mutable = const_cast<::milvus::grpc::SearchInFilesParam*>(request); - BaseTaskPtr task_ptr = SearchTask::Create(request_mutable->mutable_search_param(), file_id_array, response); + BaseRequestPtr request_ptr = + SearchRequest::Create(request_mutable->mutable_search_param(), file_id_array, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -104,9 +122,9 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) { - BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = DescribeTableRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -116,9 +134,9 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response) { int64_t row_count = 0; - BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); + BaseRequestPtr request_ptr = CountTableRequest::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_table_row_count(row_count); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -128,9 +146,9 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::g ::grpc::Status GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) { - BaseTaskPtr task_ptr = ShowTablesTask::Create(response); + BaseRequestPtr request_ptr = ShowTablesRequest::Create(response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -140,9 +158,9 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::g GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) { std::string result; - BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); + BaseRequestPtr request_ptr = CmdRequest::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_string_reply(result); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -152,9 +170,9 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Co ::grpc::Status GrpcRequestHandler::DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DeleteByDateTask::Create(request); + BaseRequestPtr request_ptr = DeleteByDateRequest::Create(request); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_error_code(grpc_status.error_code()); response->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -163,9 +181,9 @@ GrpcRequestHandler::DeleteByDate(::grpc::ServerContext* context, const ::milvus: ::grpc::Status GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); + BaseRequestPtr request_ptr = PreloadTableRequest::Create(request->table_name()); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -174,9 +192,9 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus: ::grpc::Status GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response) { - BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = DescribeIndexRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -185,9 +203,9 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); + BaseRequestPtr request_ptr = DropIndexRequest::Create(request->table_name()); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -196,17 +214,17 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::gr ::grpc::Status GrpcRequestHandler::CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreatePartitionTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreatePartitionRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response) { - BaseTaskPtr task_ptr = ShowPartitionsTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = ShowPartitionsRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -215,9 +233,9 @@ GrpcRequestHandler::ShowPartitions(::grpc::ServerContext* context, const ::milvu ::grpc::Status GrpcRequestHandler::DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropPartitionTask::Create(request); + BaseRequestPtr request_ptr = DropPartitionRequest::Create(request); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; diff --git a/core/src/server/grpc_impl/GrpcRequestScheduler.cpp b/core/src/server/grpc_impl/GrpcRequestScheduler.cpp index ac35f82947a88ae7eaf89449d6a7aa4c5e28c350..9755678ddcf6fd142943410a5d259e6bfbb05cd0 100644 --- a/core/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -70,43 +70,6 @@ ErrorMap(ErrorCode code) { } } // namespace -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcBaseTask::GrpcBaseTask(const std::string& task_group, bool async) - : task_group_(task_group), async_(async), done_(false) { -} - -GrpcBaseTask::~GrpcBaseTask() { - WaitToFinish(); -} - -Status -GrpcBaseTask::Execute() { - status_ = OnExecute(); - Done(); - return status_; -} - -void -GrpcBaseTask::Done() { - done_ = true; - finish_cond_.notify_all(); -} - -Status -GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string& error_msg) { - status_ = Status(error_code, error_msg); - SERVER_LOG_ERROR << error_msg; - return status_; -} - -Status -GrpcBaseTask::WaitToFinish() { - std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { return done_; }); - - return status_; -} - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcRequestScheduler::GrpcRequestScheduler() : stopped_(false) { Start(); @@ -117,17 +80,17 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { } void -GrpcRequestScheduler::ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status) { - if (task_ptr == nullptr) { +GrpcRequestScheduler::ExecRequest(BaseRequestPtr& request_ptr, ::milvus::grpc::Status* grpc_status) { + if (request_ptr == nullptr) { return; } GrpcRequestScheduler& scheduler = GrpcRequestScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + scheduler.ExecuteRequest(request_ptr); - if (!task_ptr->IsAsync()) { - task_ptr->WaitToFinish(); - const Status& status = task_ptr->status(); + if (!request_ptr->IsAsync()) { + request_ptr->WaitToFinish(); + const Status& status = request_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -153,7 +116,7 @@ GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler gonna stop..."; { std::lock_guard lock(queue_mtx_); - for (auto iter : task_groups_) { + for (auto iter : request_groups_) { if (iter.second != nullptr) { iter.second->Put(nullptr); } @@ -171,64 +134,64 @@ GrpcRequestScheduler::Stop() { } Status -GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) { - if (task_ptr == nullptr) { +GrpcRequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) { + if (request_ptr == nullptr) { return Status::OK(); } - auto status = PutTaskToQueue(task_ptr); + auto status = PutToQueue(request_ptr); if (!status.ok()) { - SERVER_LOG_ERROR << "Put task to queue failed with code: " << status.ToString(); + SERVER_LOG_ERROR << "Put request to queue failed with code: " << status.ToString(); return status; } - if (task_ptr->IsAsync()) { + if (request_ptr->IsAsync()) { return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere } - return task_ptr->WaitToFinish(); // sync execution + return request_ptr->WaitToFinish(); // sync execution } void -GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { - if (task_queue == nullptr) { +GrpcRequestScheduler::TakeToExecute(RequestQueuePtr request_queue) { + if (request_queue == nullptr) { return; } while (true) { - BaseTaskPtr task = task_queue->Take(); - if (task == nullptr) { - SERVER_LOG_ERROR << "Take null from task queue, stop thread"; + BaseRequestPtr request = request_queue->Take(); + if (request == nullptr) { + SERVER_LOG_ERROR << "Take null from request queue, stop thread"; break; // stop the thread } try { - auto status = task->Execute(); + auto status = request->Execute(); if (!status.ok()) { - SERVER_LOG_ERROR << "Task failed with code: " << status.ToString(); + SERVER_LOG_ERROR << "Request failed with code: " << status.ToString(); } } catch (std::exception& ex) { - SERVER_LOG_ERROR << "Task failed to execute: " << ex.what(); + SERVER_LOG_ERROR << "Request failed to execute: " << ex.what(); } } } Status -GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) { +GrpcRequestScheduler::PutToQueue(const BaseRequestPtr& request_ptr) { std::lock_guard lock(queue_mtx_); - std::string group_name = task_ptr->TaskGroup(); - if (task_groups_.count(group_name) > 0) { - task_groups_[group_name]->Put(task_ptr); + std::string group_name = request_ptr->RequestGroup(); + if (request_groups_.count(group_name) > 0) { + request_groups_[group_name]->Put(request_ptr); } else { - TaskQueuePtr queue = std::make_shared(); - queue->Put(task_ptr); - task_groups_.insert(std::make_pair(group_name, queue)); + RequestQueuePtr queue = std::make_shared(); + queue->Put(request_ptr); + request_groups_.insert(std::make_pair(group_name, queue)); // start a thread - ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeTaskToExecute, this, queue); + ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeToExecute, this, queue); execute_threads_.push_back(thread); - SERVER_LOG_INFO << "Create new thread for task group: " << group_name; + SERVER_LOG_INFO << "Create new thread for request group: " << group_name; } return Status::OK(); diff --git a/core/src/server/grpc_impl/GrpcRequestScheduler.h b/core/src/server/grpc_impl/GrpcRequestScheduler.h index 802d247fb5bbfb9d4becdaf3018b17f3e2cef49a..73f3a6e2c7894ac56a728287de7ae6741f4bf5b2 100644 --- a/core/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/core/src/server/grpc_impl/GrpcRequestScheduler.h @@ -19,6 +19,7 @@ #include "grpc/gen-status/status.grpc.pb.h" #include "grpc/gen-status/status.pb.h" +#include "server/grpc_impl/request/GrpcBaseRequest.h" #include "utils/BlockingQueue.h" #include "utils/Status.h" @@ -32,57 +33,8 @@ namespace milvus { namespace server { namespace grpc { -class GrpcBaseTask { - protected: - explicit GrpcBaseTask(const std::string& task_group, bool async = false); - - virtual ~GrpcBaseTask(); - - public: - Status - Execute(); - - void - Done(); - - Status - WaitToFinish(); - - std::string - TaskGroup() const { - return task_group_; - } - - const Status& - status() const { - return status_; - } - - bool - IsAsync() const { - return async_; - } - - protected: - virtual Status - OnExecute() = 0; - - Status - SetStatus(ErrorCode error_code, const std::string& error_msg); - - protected: - mutable std::mutex finish_mtx_; - std::condition_variable finish_cond_; - - std::string task_group_; - bool async_; - bool done_; - Status status_; -}; - -using BaseTaskPtr = std::shared_ptr; -using TaskQueue = BlockingQueue; -using TaskQueuePtr = std::shared_ptr; +using RequestQueue = BlockingQueue; +using RequestQueuePtr = std::shared_ptr; using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { @@ -100,10 +52,10 @@ class GrpcRequestScheduler { Stop(); Status - ExecuteTask(const BaseTaskPtr& task_ptr); + ExecuteRequest(const BaseRequestPtr& request_ptr); static void - ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status); + ExecRequest(BaseRequestPtr& request_ptr, ::milvus::grpc::Status* grpc_status); protected: GrpcRequestScheduler(); @@ -111,15 +63,15 @@ class GrpcRequestScheduler { virtual ~GrpcRequestScheduler(); void - TakeTaskToExecute(TaskQueuePtr task_queue); + TakeToExecute(RequestQueuePtr request_queue); Status - PutTaskToQueue(const BaseTaskPtr& task_ptr); + PutToQueue(const BaseRequestPtr& request_ptr); private: mutable std::mutex queue_mtx_; - std::map task_groups_; + std::map request_groups_; std::vector execute_threads_; diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp deleted file mode 100644 index b8a9dbccbf8226f19a19edf7b4ca45d22e1551e6..0000000000000000000000000000000000000000 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ /dev/null @@ -1,1069 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "server/grpc_impl/GrpcRequestTask.h" - -#include -#include -#include -#include -#include -//#include - -#include "GrpcServer.h" -#include "db/Utils.h" -#include "scheduler/SchedInst.h" -#include "server/DBWrapper.h" -#include "server/Server.h" -#include "src/config.h" -#include "utils/CommonUtil.h" -#include "utils/Log.h" -#include "utils/TimeRecorder.h" -#include "utils/ValidationUtil.h" - -namespace milvus { -namespace server { -namespace grpc { - -static const char* DQL_TASK_GROUP = "dql"; -static const char* DDL_DML_TASK_GROUP = "ddl_dml"; -static const char* INFO_TASK_GROUP = "info"; - -constexpr int64_t DAY_SECONDS = 24 * 60 * 60; - -using DB_META = milvus::engine::meta::Meta; -using DB_DATE = milvus::engine::meta::DateT; - -namespace { -engine::EngineType -EngineType(int type) { - static std::map map_type = { - {0, engine::EngineType::INVALID}, - {1, engine::EngineType::FAISS_IDMAP}, - {2, engine::EngineType::FAISS_IVFFLAT}, - {3, engine::EngineType::FAISS_IVFSQ8}, - }; - - if (map_type.find(type) == map_type.end()) { - return engine::EngineType::INVALID; - } - - return map_type[type]; -} - -int -IndexType(engine::EngineType type) { - static std::map map_type = { - {engine::EngineType::INVALID, 0}, - {engine::EngineType::FAISS_IDMAP, 1}, - {engine::EngineType::FAISS_IVFFLAT, 2}, - {engine::EngineType::FAISS_IVFSQ8, 3}, - }; - - if (map_type.find(type) == map_type.end()) { - return 0; - } - - return map_type[type]; -} - -Status -ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { - dates.clear(); - for (auto& range : range_array) { - time_t tt_start, tt_end; - tm tm_start, tm_end; - if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - int64_t days = (tt_end - tt_start) / DAY_SECONDS; - if (days <= 0) { - return Status(SERVER_INVALID_TIME_RANGE, - "Invalid time range: The start-date should be smaller than end-date!"); - } - - // range: [start_day, end_day) - for (int64_t i = 0; i < days; i++) { - time_t tt_day = tt_start + DAY_SECONDS * i; - tm tm_day; - CommonUtil::ConvertTime(tt_day, tm_day); - - int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic - dates.push_back(date); - } - } - - return Status::OK(); -} - -std::string -TableNotExistMsg(const std::string& table_name) { - return "Table " + table_name + - " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " - "exists."; -} - -} // namespace - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema* schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), schema_(schema) { -} - -BaseTaskPtr -CreateTableTask::Create(const ::milvus::grpc::TableSchema* schema) { - if (schema == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreateTableTask(schema)); -} - -Status -CreateTableTask::OnExecute() { - TimeRecorder rc("CreateTableTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(schema_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableDimension(schema_->dimension()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); - if (!status.ok()) { - return status; - } - - // step 2: construct table schema - engine::meta::TableSchema table_info; - table_info.table_id_ = schema_->table_name(); - table_info.dimension_ = static_cast(schema_->dimension()); - table_info.index_file_size_ = schema_->index_file_size(); - table_info.metric_type_ = schema_->metric_type(); - - // step 3: create table - status = DBWrapper::DB()->CreateTable(table_info); - if (!status.ok()) { - // table could exist - if (status.code() == DB_ALREADY_EXIST) { - return Status(SERVER_INVALID_TABLE_NAME, status.message()); - } - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeTableTask::DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), schema_(schema) { -} - -BaseTaskPtr -DescribeTableTask::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { - return std::shared_ptr(new DescribeTableTask(table_name, schema)); -} - -Status -DescribeTableTask::OnExecute() { - TimeRecorder rc("DescribeTableTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: get table info - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - return status; - } - - schema_->set_table_name(table_info.table_id_); - schema_->set_dimension(table_info.dimension_); - schema_->set_index_file_size(table_info.index_file_size_); - schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam* index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), index_param_(index_param) { -} - -BaseTaskPtr -CreateIndexTask::Create(const ::milvus::grpc::IndexParam* index_param) { - if (index_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreateIndexTask(index_param)); -} - -Status -CreateIndexTask::OnExecute() { - try { - TimeRecorder rc("CreateIndexTask"); - - // step 1: check arguments - std::string table_name_ = index_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - bool has_table = false; - status = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!status.ok()) { - return status; - } - - if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } - - auto& grpc_index = index_param_->index(); - status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::TableIndex index; - index.engine_type_ = grpc_index.index_type(); - index.nlist_ = grpc_index.nlist(); - status = DBWrapper::DB()->CreateIndex(table_name_, index); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), has_table_(has_table) { -} - -BaseTaskPtr -HasTableTask::Create(const std::string& table_name, bool& has_table) { - return std::shared_ptr(new HasTableTask(table_name, has_table)); -} - -Status -HasTableTask::OnExecute() { - try { - TimeRecorder rc("HasTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - status = DBWrapper::DB()->HasTable(table_name_, has_table_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropTableTask::DropTableTask(const std::string& table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -DropTableTask::Create(const std::string& table_name) { - return std::shared_ptr(new DropTableTask(table_name)); -} - -Status -DropTableTask::OnExecute() { - try { - TimeRecorder rc("DropTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - rc.ElapseFromBegin("check validation"); - - // step 3: Drop table - std::vector dates; - status = DBWrapper::DB()->DropTable(table_name_, dates); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList* table_name_list) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_list_(table_name_list) { -} - -BaseTaskPtr -ShowTablesTask::Create(::milvus::grpc::TableNameList* table_name_list) { - return std::shared_ptr(new ShowTablesTask(table_name_list)); -} - -Status -ShowTablesTask::OnExecute() { - std::vector schema_array; - auto statuts = DBWrapper::DB()->AllTables(schema_array); - if (!statuts.ok()) { - return statuts; - } - - for (auto& schema : schema_array) { - table_name_list_->add_table_names(schema.table_id_); - } - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -InsertTask::InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) - : GrpcBaseTask(DDL_DML_TASK_GROUP), insert_param_(insert_param), record_ids_(record_ids) { -} - -BaseTaskPtr -InsertTask::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { - if (insert_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new InsertTask(insert_param, record_ids)); -} - -Status -InsertTask::OnExecute() { - try { - TimeRecorder rc("InsertVectorTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); - if (!status.ok()) { - return status; - } - if (insert_param_->row_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector array is empty. Make sure you have entered vector records."); - } - - if (!insert_param_->row_id_array().empty()) { - if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "The size of vector ID array must be equal to the size of the vector."); - } - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = insert_param_->table_name(); - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); - } else { - return status; - } - } - - // step 3: check table flag - // all user provide id, or all internal id - bool user_provide_ids = !insert_param_->row_id_array().empty(); - // user already provided id before, all insert action require user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); - } - - // user didn't provided id before, no need to provide user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { - return Status( - SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); - } - - rc.RecordSection("check validation"); - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = - "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - - // step 4: prepare float data - std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); - - // TODO(yk): change to one dimension array or use multiple-thread to copy the data - for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { - if (insert_param_->row_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector dimension must be equal to the table dimension."); - } - uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); - if (vec_dim != table_info.dimension_) { - ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "The vector dimension must be equal to the table dimension."; - return Status(error_code, error_msg); - } - memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), - table_info.dimension_ * sizeof(float)); - } - - rc.ElapseFromBegin("prepare vectors data"); - - // step 5: insert vectors - auto vec_count = static_cast(insert_param_->row_record_array_size()); - std::vector vec_ids(insert_param_->row_id_array_size(), 0); - if (!insert_param_->row_id_array().empty()) { - const int64_t* src_data = insert_param_->row_id_array().data(); - int64_t* target_data = vec_ids.data(); - memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); - } - - status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, - vec_f.data(), vec_ids); - rc.ElapseFromBegin("add vectors to engine"); - if (!status.ok()) { - return status; - } - for (int64_t id : vec_ids) { - record_ids_->add_vector_id_array(id); - } - - auto ids_size = record_ids_->vector_id_array_size(); - if (ids_size != vec_count) { - std::string msg = - "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; - return Status(SERVER_ILLEGAL_VECTOR_ID, msg); - } - - // step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; - status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); - -#ifdef MILVUS_ENABLE_PROFILING - ProfilerStop(); -#endif - - rc.RecordSection("add vectors to engine"); - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchTask::SearchTask(const ::milvus::grpc::SearchParam* search_vector_infos, - const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) - : GrpcBaseTask(DQL_TASK_GROUP), - search_param_(search_vector_infos), - file_id_array_(file_id_array), - topk_result_(response) { -} - -BaseTaskPtr -SearchTask::Create(const ::milvus::grpc::SearchParam* search_vector_infos, - const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) { - if (search_vector_infos == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, response)); -} - -Status -SearchTask::OnExecute() { - try { - int64_t top_k = search_param_->topk(); - int64_t nprobe = search_param_->nprobe(); - - std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; - TimeRecorder rc(hdr); - - // step 1: check table name - std::string table_name_ = search_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - // step 3: check search parameter - status = ValidationUtil::ValidateSearchTopk(top_k, table_info); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); - if (!status.ok()) { - return status; - } - - if (search_param_->query_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector array is empty. Make sure you have entered vector records."); - } - - // step 4: check date range, and convert to db dates - std::vector dates; - std::vector<::milvus::grpc::Range> range_array; - for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { - range_array.emplace_back(search_param_->query_range_array(i)); - } - - status = ConvertTimeRangeToDBDates(range_array, dates); - if (!status.ok()) { - return status; - } - - rc.RecordSection("check validation"); - - // step 5: prepare float data - auto record_array_size = search_param_->query_record_array_size(); - std::vector vec_f(record_array_size * table_info.dimension_, 0); - for (size_t i = 0; i < record_array_size; i++) { - if (search_param_->query_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector dimension must be equal to the table dimension."); - } - uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); - if (query_vec_dim != table_info.dimension_) { - ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "The vector dimension must be equal to the table dimension."; - return Status(error_code, error_msg); - } - - memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), - table_info.dimension_ * sizeof(float)); - } - rc.RecordSection("prepare vector data"); - - // step 6: search vectors - engine::ResultIds result_ids; - engine::ResultDistances result_distances; - auto record_count = (uint64_t)search_param_->query_record_array().size(); - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = - "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - - if (file_id_array_.empty()) { - std::vector partition_tags; - for (size_t i = 0; i < search_param_->partition_tag_array_size(); i++) { - partition_tags.emplace_back(search_param_->partition_tag_array(i)); - } - - status = ValidationUtil::ValidatePartitionTags(partition_tags); - if (!status.ok()) { - return status; - } - - status = DBWrapper::DB()->Query(table_name_, partition_tags, (size_t)top_k, record_count, nprobe, - vec_f.data(), dates, result_ids, result_distances); - } else { - status = DBWrapper::DB()->QueryByFileID(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, - vec_f.data(), dates, result_ids, result_distances); - } - -#ifdef MILVUS_ENABLE_PROFILING - ProfilerStop(); -#endif - - rc.RecordSection("search vectors from engine"); - if (!status.ok()) { - return status; - } - - if (result_ids.empty()) { - return Status::OK(); // empty table - } - - // step 7: construct result array - topk_result_->set_row_num(record_count); - topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), 0); - memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); - topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); - memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), - result_distances.size() * sizeof(float)); - - // step 8: print time cost percent - rc.RecordSection("construct result and send"); - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CountTableTask::CountTableTask(const std::string& table_name, int64_t& row_count) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), row_count_(row_count) { -} - -BaseTaskPtr -CountTableTask::Create(const std::string& table_name, int64_t& row_count) { - return std::shared_ptr(new CountTableTask(table_name, row_count)); -} - -Status -CountTableTask::OnExecute() { - try { - TimeRecorder rc("GetTableRowCountTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: get row count - uint64_t row_count = 0; - status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); - if (!status.ok()) { - if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - row_count_ = static_cast(row_count); - - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CmdTask::CmdTask(const std::string& cmd, std::string& result) - : GrpcBaseTask(INFO_TASK_GROUP), cmd_(cmd), result_(result) { -} - -BaseTaskPtr -CmdTask::Create(const std::string& cmd, std::string& result) { - return std::shared_ptr(new CmdTask(cmd, result)); -} - -Status -CmdTask::OnExecute() { - if (cmd_ == "version") { - result_ = MILVUS_VERSION; - } else if (cmd_ == "tasktable") { - result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); - } else { - result_ = "OK"; - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DeleteByDateTask::DeleteByDateTask(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), delete_by_range_param_(delete_by_range_param) { -} - -BaseTaskPtr -DeleteByDateTask::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) { - if (delete_by_range_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - - return std::shared_ptr(new DeleteByDateTask(delete_by_range_param)); -} - -Status -DeleteByDateTask::OnExecute() { - try { - TimeRecorder rc("DeleteByRangeTask"); - - // step 1: check arguments - std::string table_name = delete_by_range_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); - } else { - return status; - } - } - - rc.ElapseFromBegin("check validation"); - - // step 3: check date range, and convert to db dates - std::vector dates; - ErrorCode error_code = SERVER_SUCCESS; - std::string error_msg; - - std::vector<::milvus::grpc::Range> range_array; - range_array.emplace_back(delete_by_range_param_->range()); - status = ConvertTimeRangeToDBDates(range_array, dates); - if (!status.ok()) { - return status; - } - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - status = DBWrapper::DB()->DropTable(table_name, dates); - if (!status.ok()) { - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -PreloadTableTask::PreloadTableTask(const std::string& table_name) - : GrpcBaseTask(DQL_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -PreloadTableTask::Create(const std::string& table_name) { - return std::shared_ptr(new PreloadTableTask(table_name)); -} - -Status -PreloadTableTask::OnExecute() { - try { - TimeRecorder rc("PreloadTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - status = DBWrapper::DB()->PreloadTable(table_name_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeIndexTask::DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), index_param_(index_param) { -} - -BaseTaskPtr -DescribeIndexTask::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { - return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); -} - -Status -DescribeIndexTask::OnExecute() { - try { - TimeRecorder rc("DescribeIndexTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::TableIndex index; - status = DBWrapper::DB()->DescribeIndex(table_name_, index); - if (!status.ok()) { - return status; - } - - index_param_->set_table_name(table_name_); - index_param_->mutable_index()->set_index_type(index.engine_type_); - index_param_->mutable_index()->set_nlist(index.nlist_); - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropIndexTask::DropIndexTask(const std::string& table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -DropIndexTask::Create(const std::string& table_name) { - return std::shared_ptr(new DropIndexTask(table_name)); -} - -Status -DropIndexTask::OnExecute() { - try { - TimeRecorder rc("DropIndexTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - bool has_table = false; - status = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!status.ok()) { - return status; - } - - if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } - - // step 2: check table existence - status = DBWrapper::DB()->DropIndex(table_name_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreatePartitionTask::CreatePartitionTask(const ::milvus::grpc::PartitionParam* partition_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), partition_param_(partition_param) { -} - -BaseTaskPtr -CreatePartitionTask::Create(const ::milvus::grpc::PartitionParam* partition_param) { - if (partition_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreatePartitionTask(partition_param)); -} - -Status -CreatePartitionTask::OnExecute() { - TimeRecorder rc("CreatePartitionTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); - if (!status.ok()) { - return status; - } - - // step 2: create partition - status = DBWrapper::DB()->CreatePartition(partition_param_->table_name(), partition_param_->partition_name(), - partition_param_->tag()); - if (!status.ok()) { - // partition could exist - if (status.code() == DB_ALREADY_EXIST) { - return Status(SERVER_INVALID_TABLE_NAME, status.message()); - } - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowPartitionsTask::ShowPartitionsTask(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), partition_list_(partition_list) { -} - -BaseTaskPtr -ShowPartitionsTask::Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) { - return std::shared_ptr(new ShowPartitionsTask(table_name, partition_list)); -} - -Status -ShowPartitionsTask::OnExecute() { - std::vector schema_array; - auto statuts = DBWrapper::DB()->ShowPartitions(table_name_, schema_array); - if (!statuts.ok()) { - return statuts; - } - - for (auto& schema : schema_array) { - ::milvus::grpc::PartitionParam* param = partition_list_->add_partition_array(); - param->set_table_name(schema.owner_table_); - param->set_partition_name(schema.table_id_); - param->set_tag(schema.partition_tag_); - } - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropPartitionTask::DropPartitionTask(const ::milvus::grpc::PartitionParam* partition_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), partition_param_(partition_param) { -} - -BaseTaskPtr -DropPartitionTask::Create(const ::milvus::grpc::PartitionParam* partition_param) { - return std::shared_ptr(new DropPartitionTask(partition_param)); -} - -Status -DropPartitionTask::OnExecute() { - if (!partition_param_->partition_name().empty()) { - auto status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); - if (!status.ok()) { - return status; - } - return DBWrapper::DB()->DropPartition(partition_param_->partition_name()); - } else { - auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); - if (!status.ok()) { - return status; - } - return DBWrapper::DB()->DropPartitionByTag(partition_param_->table_name(), partition_param_->tag()); - } -} - -} // namespace grpc -} // namespace server -} // namespace milvus diff --git a/core/src/server/grpc_impl/GrpcRequestTask.h b/core/src/server/grpc_impl/GrpcRequestTask.h deleted file mode 100644 index 087c71aaaf23e3c806e41befaffbf59048eb6f80..0000000000000000000000000000000000000000 --- a/core/src/server/grpc_impl/GrpcRequestTask.h +++ /dev/null @@ -1,321 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "db/Types.h" -#include "server/grpc_impl/GrpcRequestScheduler.h" -#include "utils/Status.h" - -#include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "grpc/gen-status/status.pb.h" - -#include -#include -#include -#include - -namespace milvus { -namespace server { -namespace grpc { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::TableSchema* schema); - - protected: - explicit CreateTableTask(const ::milvus::grpc::TableSchema* schema); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::TableSchema* schema_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class HasTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, bool& has_table); - - protected: - HasTableTask(const std::string& table_name, bool& has_table); - - Status - OnExecute() override; - - private: - std::string table_name_; - bool& has_table_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DescribeTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); - - protected: - DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::TableSchema* schema_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit DropTableTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::IndexParam* index_param); - - protected: - explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::IndexParam* index_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class ShowTablesTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(::milvus::grpc::TableNameList* table_name_list); - - protected: - explicit ShowTablesTask(::milvus::grpc::TableNameList* table_name_list); - - Status - OnExecute() override; - - private: - ::milvus::grpc::TableNameList* table_name_list_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class InsertTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); - - protected: - InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::InsertParam* insert_param_; - ::milvus::grpc::VectorIds* record_ids_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SearchTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, - ::milvus::grpc::TopKQueryResult* response); - - protected: - SearchTask(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, - ::milvus::grpc::TopKQueryResult* response); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::SearchParam* search_param_; - std::vector file_id_array_; - ::milvus::grpc::TopKQueryResult* topk_result_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CountTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, int64_t& row_count); - - protected: - CountTableTask(const std::string& table_name, int64_t& row_count); - - Status - OnExecute() override; - - private: - std::string table_name_; - int64_t& row_count_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CmdTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& cmd, std::string& result); - - protected: - CmdTask(const std::string& cmd, std::string& result); - - Status - OnExecute() override; - - private: - std::string cmd_; - std::string& result_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DeleteByDateTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); - - protected: - explicit DeleteByDateTask(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::DeleteByDateParam* delete_by_range_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class PreloadTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit PreloadTableTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DescribeIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); - - protected: - DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::IndexParam* index_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit DropIndexTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreatePartitionTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::PartitionParam* partition_param); - - protected: - explicit CreatePartitionTask(const ::milvus::grpc::PartitionParam* partition_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::PartitionParam* partition_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class ShowPartitionsTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); - - protected: - ShowPartitionsTask(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::PartitionList* partition_list_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropPartitionTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::PartitionParam* partition_param); - - protected: - explicit DropPartitionTask(const ::milvus::grpc::PartitionParam* partition_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::PartitionParam* partition_param_; -}; - -} // namespace grpc -} // namespace server -} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b215f94d3168299d108ef054881565ca67c09fc1 --- /dev/null +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CmdRequest.h" +#include "scheduler/SchedInst.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CmdRequest::CmdRequest(const std::string& cmd, std::string& result) + : GrpcBaseRequest(INFO_REQUEST_GROUP), cmd_(cmd), result_(result) { +} + +BaseRequestPtr +CmdRequest::Create(const std::string& cmd, std::string& result) { + return std::shared_ptr(new CmdRequest(cmd, result)); +} + +Status +CmdRequest::OnExecute() { + if (cmd_ == "version") { + result_ = MILVUS_VERSION; + } else if (cmd_ == "tasktable") { + result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); + } else { + result_ = "OK"; + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CmdRequest.h b/core/src/server/grpc_impl/request/CmdRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..960b17d436ac97bd63d0f8ef05b9de127e79b651 --- /dev/null +++ b/core/src/server/grpc_impl/request/CmdRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class CmdRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& cmd, std::string& result); + + protected: + CmdRequest(const std::string& cmd, std::string& result); + + Status + OnExecute() override; + + private: + std::string cmd_; + std::string& result_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CountTableRequest.cpp b/core/src/server/grpc_impl/request/CountTableRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8559890ad656079cd6a5d89adcab7b29b87a1169 --- /dev/null +++ b/core/src/server/grpc_impl/request/CountTableRequest.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CountTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CountTableRequest::CountTableRequest(const std::string& table_name, int64_t& row_count) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), row_count_(row_count) { +} + +BaseRequestPtr +CountTableRequest::Create(const std::string& table_name, int64_t& row_count) { + return std::shared_ptr(new CountTableRequest(table_name, row_count)); +} + +Status +CountTableRequest::OnExecute() { + try { + TimeRecorder rc("CountTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: get row count + uint64_t row_count = 0; + status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + row_count_ = static_cast(row_count); + + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CountTableRequest.h b/core/src/server/grpc_impl/request/CountTableRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..80f8fcfcf5380b4b4e7b525bb7a6ecc11cbccb6a --- /dev/null +++ b/core/src/server/grpc_impl/request/CountTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class CountTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, int64_t& row_count); + + protected: + CountTableRequest(const std::string& table_name, int64_t& row_count); + + Status + OnExecute() override; + + private: + std::string table_name_; + int64_t& row_count_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..72678aee87e89f861835ca06dcd239e551664dd1 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreateIndexRequest::CreateIndexRequest(const ::milvus::grpc::IndexParam* index_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), index_param_(index_param) { +} + +BaseRequestPtr +CreateIndexRequest::Create(const ::milvus::grpc::IndexParam* index_param) { + if (index_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreateIndexRequest(index_param)); +} + +Status +CreateIndexRequest::OnExecute() { + try { + TimeRecorder rc("CreateIndexRequest"); + + // step 1: check arguments + std::string table_name_ = index_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + bool has_table = false; + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; + } + + if (!has_table) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } + + auto& grpc_index = index_param_->index(); + status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::TableIndex index; + index.engine_type_ = grpc_index.index_type(); + index.nlist_ = grpc_index.nlist(); + status = DBWrapper::DB()->CreateIndex(table_name_, index); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.h b/core/src/server/grpc_impl/request/CreateIndexRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..9503cf0d0624a88f5b06eb7b1b3a79f88743b4dd --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreateIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::IndexParam* index_param); + + protected: + explicit CreateIndexRequest(const ::milvus::grpc::IndexParam* index_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::IndexParam* index_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3bd4a86ef68d5c019993f9898052d5f15fa3564c --- /dev/null +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreatePartitionRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreatePartitionRequest::CreatePartitionRequest(const ::milvus::grpc::PartitionParam* partition_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), partition_param_(partition_param) { +} + +BaseRequestPtr +CreatePartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_param) { + if (partition_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreatePartitionRequest(partition_param)); +} + +Status +CreatePartitionRequest::OnExecute() { + TimeRecorder rc("CreatePartitionRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionName(partition_param_->partition_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); + if (!status.ok()) { + return status; + } + + // step 2: create partition + status = DBWrapper::DB()->CreatePartition(partition_param_->table_name(), partition_param_->partition_name(), + partition_param_->tag()); + if (!status.ok()) { + // partition could exist + if (status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); + } + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.h b/core/src/server/grpc_impl/request/CreatePartitionRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..70a284bc16adc52f313d0a53cf54cd04ca97c0cc --- /dev/null +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreatePartitionRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::PartitionParam* partition_param); + + protected: + explicit CreatePartitionRequest(const ::milvus::grpc::PartitionParam* partition_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::PartitionParam* partition_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.cpp b/core/src/server/grpc_impl/request/CreateTableRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..67a3eaa8779afaf419dbb9b8509196abfd39f4a7 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateTableRequest.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreateTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreateTableRequest::CreateTableRequest(const ::milvus::grpc::TableSchema* schema) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), schema_(schema) { +} + +BaseRequestPtr +CreateTableRequest::Create(const ::milvus::grpc::TableSchema* schema) { + if (schema == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreateTableRequest(schema)); +} + +Status +CreateTableRequest::OnExecute() { + TimeRecorder rc("CreateTableRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(schema_->table_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableDimension(schema_->dimension()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); + if (!status.ok()) { + return status; + } + + // step 2: construct table schema + engine::meta::TableSchema table_info; + table_info.table_id_ = schema_->table_name(); + table_info.dimension_ = static_cast(schema_->dimension()); + table_info.index_file_size_ = schema_->index_file_size(); + table_info.metric_type_ = schema_->metric_type(); + + // step 3: create table + status = DBWrapper::DB()->CreateTable(table_info); + if (!status.ok()) { + // table could exist + if (status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); + } + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.h b/core/src/server/grpc_impl/request/CreateTableRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..3a07aae1b7e24cf7137bf16828f9dc3ce3952984 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateTableRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreateTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::TableSchema* schema); + + protected: + explicit CreateTableRequest(const ::milvus::grpc::TableSchema* schema); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::TableSchema* schema_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4a4d414803bc72a2d226ef528fd3663c1a2927d3 --- /dev/null +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DeleteByDateRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DeleteByDateRequest::DeleteByDateRequest(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), delete_by_range_param_(delete_by_range_param) { +} + +BaseRequestPtr +DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) { + if (delete_by_range_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + + return std::shared_ptr(new DeleteByDateRequest(delete_by_range_param)); +} + +Status +DeleteByDateRequest::OnExecute() { + try { + TimeRecorder rc("DeleteByDateRequest"); + + // step 1: check arguments + std::string table_name = delete_by_range_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); + } else { + return status; + } + } + + rc.ElapseFromBegin("check validation"); + + // step 3: check date range, and convert to db dates + std::vector dates; + ErrorCode error_code = SERVER_SUCCESS; + std::string error_msg; + + std::vector<::milvus::grpc::Range> range_array; + range_array.emplace_back(delete_by_range_param_->range()); + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; + } + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + status = DBWrapper::DB()->DropTable(table_name, dates); + if (!status.ok()) { + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.h b/core/src/server/grpc_impl/request/DeleteByDateRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..b3e04c2850b632e46310337085ad71569ed1b3be --- /dev/null +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class DeleteByDateRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); + + protected: + explicit DeleteByDateRequest(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::DeleteByDateParam* delete_by_range_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3a987c6b03a244bc4dcc7765c8422f575b4cdad --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DescribeIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DescribeIndexRequest::DescribeIndexRequest(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), index_param_(index_param) { +} + +BaseRequestPtr +DescribeIndexRequest::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { + return std::shared_ptr(new DescribeIndexRequest(table_name, index_param)); +} + +Status +DescribeIndexRequest::OnExecute() { + try { + TimeRecorder rc("DescribeIndexRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::TableIndex index; + status = DBWrapper::DB()->DescribeIndex(table_name_, index); + if (!status.ok()) { + return status; + } + + index_param_->set_table_name(table_name_); + index_param_->mutable_index()->set_index_type(index.engine_type_); + index_param_->mutable_index()->set_nlist(index.nlist_); + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.h b/core/src/server/grpc_impl/request/DescribeIndexRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..fa0d72152f459062e1bcf7c6e34bc17b926b79cb --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DescribeIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); + + protected: + DescribeIndexRequest(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::IndexParam* index_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..28a5f327c5a0149cd72b046c0e2fe6744a90e9c1 --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DescribeTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DescribeTableRequest::DescribeTableRequest(const std::string& table_name, ::milvus::grpc::TableSchema* schema) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), schema_(schema) { +} + +BaseRequestPtr +DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { + return std::shared_ptr(new DescribeTableRequest(table_name, schema)); +} + +Status +DescribeTableRequest::OnExecute() { + TimeRecorder rc("DescribeTableRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: get table info + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + return status; + } + + schema_->set_table_name(table_info.table_id_); + schema_->set_dimension(table_info.dimension_); + schema_->set_index_file_size(table_info.index_file_size_); + schema_->set_metric_type(table_info.metric_type_); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.h b/core/src/server/grpc_impl/request/DescribeTableRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..0d903fedbdd83f533797f867180392ad548fde6f --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DescribeTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); + + protected: + DescribeTableRequest(const std::string& table_name, ::milvus::grpc::TableSchema* schema); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::TableSchema* schema_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.cpp b/core/src/server/grpc_impl/request/DropIndexRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ab5c83b0e5fa52f0b845250589abeabb94dde98f --- /dev/null +++ b/core/src/server/grpc_impl/request/DropIndexRequest.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropIndexRequest::DropIndexRequest(const std::string& table_name) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +DropIndexRequest::Create(const std::string& table_name) { + return std::shared_ptr(new DropIndexRequest(table_name)); +} + +Status +DropIndexRequest::OnExecute() { + try { + TimeRecorder rc("DropIndexRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + bool has_table = false; + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; + } + + if (!has_table) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } + + // step 2: check table existence + status = DBWrapper::DB()->DropIndex(table_name_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.h b/core/src/server/grpc_impl/request/DropIndexRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..c7cd51fcd7ee0d9f14a03f09299daa5f79a73839 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropIndexRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DropIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit DropIndexRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0e29b6abe8d458a7cbbba87ec74d8d11db809196 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropPartitionRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropPartitionRequest::DropPartitionRequest(const ::milvus::grpc::PartitionParam* partition_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), partition_param_(partition_param) { +} + +BaseRequestPtr +DropPartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_param) { + return std::shared_ptr(new DropPartitionRequest(partition_param)); +} + +Status +DropPartitionRequest::OnExecute() { + std::string table_name = partition_param_->table_name(); + std::string partition_name = partition_param_->partition_name(); + std::string partition_tag = partition_param_->tag(); + if (!partition_name.empty()) { + auto status = ValidationUtil::ValidateTableName(partition_name); + if (!status.ok()) { + return status; + } + + // check partition existence + engine::meta::TableSchema table_info; + table_info.table_id_ = partition_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, + "Table " + table_name + "'s partition " + partition_name + " not found"); + } else { + return status; + } + } + + return DBWrapper::DB()->DropPartition(partition_name); + } else { + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionTags({partition_tag}); + if (!status.ok()) { + return status; + } + return DBWrapper::DB()->DropPartitionByTag(table_name, partition_tag); + } +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.h b/core/src/server/grpc_impl/request/DropPartitionRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..4ac57435551441ef7f2259003ac5f23926571a2c --- /dev/null +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class DropPartitionRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::PartitionParam* partition_param); + + protected: + explicit DropPartitionRequest(const ::milvus::grpc::PartitionParam* partition_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::PartitionParam* partition_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropTableRequest.cpp b/core/src/server/grpc_impl/request/DropTableRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a678e5a6f6c4a9b8aa69bbd0625b9f7a1f6b96bb --- /dev/null +++ b/core/src/server/grpc_impl/request/DropTableRequest.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropTableRequest::DropTableRequest(const std::string& table_name) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +DropTableRequest::Create(const std::string& table_name) { + return std::shared_ptr(new DropTableRequest(table_name)); +} + +Status +DropTableRequest::OnExecute() { + try { + TimeRecorder rc("DropTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + rc.ElapseFromBegin("check validation"); + + // step 3: Drop table + std::vector dates; + status = DBWrapper::DB()->DropTable(table_name_, dates); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropTableRequest.h b/core/src/server/grpc_impl/request/DropTableRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..7dd7eca209fbeff280e936b1d63fdeb92afb5735 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropTableRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DropTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit DropTableRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e1eb07af5c2af2dddff553c69720ea51cf4c0443 --- /dev/null +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/GrpcBaseRequest.h" +#include "utils/CommonUtil.h" +#include "utils/Log.h" + +namespace milvus { +namespace server { +namespace grpc { + +constexpr int64_t DAY_SECONDS = 24 * 60 * 60; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { + dates.clear(); + for (auto& range : range_array) { + time_t tt_start, tt_end; + tm tm_start, tm_end; + if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + int64_t days = (tt_end - tt_start) / DAY_SECONDS; + if (days <= 0) { + return Status(SERVER_INVALID_TIME_RANGE, + "Invalid time range: The start-date should be smaller than end-date!"); + } + + // range: [start_day, end_day) + for (int64_t i = 0; i < days; i++) { + time_t tt_day = tt_start + DAY_SECONDS * i; + tm tm_day; + CommonUtil::ConvertTime(tt_day, tm_day); + + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic + dates.push_back(date); + } + } + + return Status::OK(); +} + +GrpcBaseRequest::GrpcBaseRequest(const std::string& request_group, bool async) + : request_group_(request_group), async_(async), done_(false) { +} + +GrpcBaseRequest::~GrpcBaseRequest() { + WaitToFinish(); +} + +Status +GrpcBaseRequest::Execute() { + status_ = OnExecute(); + Done(); + return status_; +} + +void +GrpcBaseRequest::Done() { + done_ = true; + finish_cond_.notify_all(); +} + +Status +GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) { + status_ = Status(error_code, error_msg); + SERVER_LOG_ERROR << error_msg; + return status_; +} + +std::string +GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) { + return "Table " + table_name + + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " + "exists."; +} + +Status +GrpcBaseRequest::WaitToFinish() { + std::unique_lock lock(finish_mtx_); + finish_cond_.wait(lock, [this] { return done_; }); + + return status_; +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.h b/core/src/server/grpc_impl/request/GrpcBaseRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..b64662b2cf063b8923ec42130bed192a1359dfbf --- /dev/null +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "db/meta/MetaTypes.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" +#include "utils/Status.h" + +#include +//#include +#include +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +static const char* DQL_REQUEST_GROUP = "dql"; +static const char* DDL_DML_REQUEST_GROUP = "ddl_dml"; +static const char* INFO_REQUEST_GROUP = "info"; + +using DB_DATE = milvus::engine::meta::DateT; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates); + +class GrpcBaseRequest { + protected: + explicit GrpcBaseRequest(const std::string& request_group, bool async = false); + + virtual ~GrpcBaseRequest(); + + public: + Status + Execute(); + + void + Done(); + + Status + WaitToFinish(); + + std::string + RequestGroup() const { + return request_group_; + } + + const Status& + status() const { + return status_; + } + + bool + IsAsync() const { + return async_; + } + + protected: + virtual Status + OnExecute() = 0; + + Status + SetStatus(ErrorCode error_code, const std::string& error_msg); + + std::string + TableNotExistMsg(const std::string& table_name); + + protected: + mutable std::mutex finish_mtx_; + std::condition_variable finish_cond_; + + std::string request_group_; + bool async_; + bool done_; + Status status_; +}; + +using BaseRequestPtr = std::shared_ptr; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/HasTableRequest.cpp b/core/src/server/grpc_impl/request/HasTableRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2909c9305624ef88b9fdd8dd88aade23a18b20b7 --- /dev/null +++ b/core/src/server/grpc_impl/request/HasTableRequest.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/HasTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +HasTableRequest::HasTableRequest(const std::string& table_name, bool& has_table) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), has_table_(has_table) { +} + +BaseRequestPtr +HasTableRequest::Create(const std::string& table_name, bool& has_table) { + return std::shared_ptr(new HasTableRequest(table_name, has_table)); +} + +Status +HasTableRequest::OnExecute() { + try { + TimeRecorder rc("HasTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + status = DBWrapper::DB()->HasTable(table_name_, has_table_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/HasTableRequest.h b/core/src/server/grpc_impl/request/HasTableRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..85fa2b345bf90ffd658101c89751b7234be6a7c8 --- /dev/null +++ b/core/src/server/grpc_impl/request/HasTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class HasTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, bool& has_table); + + protected: + HasTableRequest(const std::string& table_name, bool& has_table); + + Status + OnExecute() override; + + private: + std::string table_name_; + bool& has_table_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f436db074e4280546f78f8249f282e56bf09b0d8 --- /dev/null +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/InsertRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +InsertRequest::InsertRequest(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), insert_param_(insert_param), record_ids_(record_ids) { +} + +BaseRequestPtr +InsertRequest::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { + if (insert_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new InsertRequest(insert_param, record_ids)); +} + +Status +InsertRequest::OnExecute() { + try { + TimeRecorder rc("InsertRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); + if (!status.ok()) { + return status; + } + if (insert_param_->row_record_array().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); + } + + if (!insert_param_->row_id_array().empty()) { + if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "The size of vector ID array must be equal to the size of the vector."); + } + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = insert_param_->table_name(); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); + } else { + return status; + } + } + + // step 3: check table flag + // all user provide id, or all internal id + bool user_provide_ids = !insert_param_->row_id_array().empty(); + // user already provided id before, all insert action require user id + if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); + } + + // user didn't provided id before, no need to provide user id + if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { + return Status( + SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); + } + + rc.RecordSection("check validation"); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = + "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + + // step 4: prepare float data + std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); + + // TODO(yk): change to one dimension array or use multiple-thread to copy the data + for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { + if (insert_param_->row_record_array(i).vector_data().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); + } + uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); + if (vec_dim != table_info.dimension_) { + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; + std::string error_msg = "The vector dimension must be equal to the table dimension."; + return Status(error_code, error_msg); + } + memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), + table_info.dimension_ * sizeof(float)); + } + + rc.ElapseFromBegin("prepare vectors data"); + + // step 5: insert vectors + auto vec_count = static_cast(insert_param_->row_record_array_size()); + std::vector vec_ids(insert_param_->row_id_array_size(), 0); + if (!insert_param_->row_id_array().empty()) { + const int64_t* src_data = insert_param_->row_id_array().data(); + int64_t* target_data = vec_ids.data(); + memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); + } + + status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, + vec_f.data(), vec_ids); + rc.ElapseFromBegin("add vectors to engine"); + if (!status.ok()) { + return status; + } + for (int64_t id : vec_ids) { + record_ids_->add_vector_id_array(id); + } + + auto ids_size = record_ids_->vector_id_array_size(); + if (ids_size != vec_count) { + std::string msg = + "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; + return Status(SERVER_ILLEGAL_VECTOR_ID, msg); + } + + // step 6: update table flag + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("add vectors to engine"); + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/InsertRequest.h b/core/src/server/grpc_impl/request/InsertRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..1582615d2e4b0dfa91a3306a4baef2fe7c58cb5d --- /dev/null +++ b/core/src/server/grpc_impl/request/InsertRequest.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class InsertRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); + + protected: + InsertRequest(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::InsertParam* insert_param_; + ::milvus::grpc::VectorIds* record_ids_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e26aa8a8776547980f733571d81f4100694a46e8 --- /dev/null +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/PreloadTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +PreloadTableRequest::PreloadTableRequest(const std::string& table_name) + : GrpcBaseRequest(DQL_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +PreloadTableRequest::Create(const std::string& table_name) { + return std::shared_ptr(new PreloadTableRequest(table_name)); +} + +Status +PreloadTableRequest::OnExecute() { + try { + TimeRecorder rc("PreloadTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + status = DBWrapper::DB()->PreloadTable(table_name_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.h b/core/src/server/grpc_impl/request/PreloadTableRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..208c7477ad1687a605c738bea20fe10f4b8707cc --- /dev/null +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class PreloadTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit PreloadTableRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..28f4ff723e4fe6415c6181be03d55666e8fec3ca --- /dev/null +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/SearchRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +SearchRequest::SearchRequest(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) + : GrpcBaseRequest(DQL_REQUEST_GROUP), + search_param_(search_vector_infos), + file_id_array_(file_id_array), + topk_result_(response) { +} + +BaseRequestPtr +SearchRequest::Create(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) { + if (search_vector_infos == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new SearchRequest(search_vector_infos, file_id_array, response)); +} + +Status +SearchRequest::OnExecute() { + try { + int64_t top_k = search_param_->topk(); + int64_t nprobe = search_param_->nprobe(); + + std::string hdr = "SearchRequest(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; + TimeRecorder rc(hdr); + + // step 1: check table name + std::string table_name_ = search_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + // step 3: check search parameter + status = ValidationUtil::ValidateSearchTopk(top_k, table_info); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); + if (!status.ok()) { + return status; + } + + if (search_param_->query_record_array().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); + } + + // step 4: check date range, and convert to db dates + std::vector dates; + std::vector<::milvus::grpc::Range> range_array; + for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { + range_array.emplace_back(search_param_->query_range_array(i)); + } + + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; + } + + rc.RecordSection("check validation"); + + // step 5: prepare float data + auto record_array_size = search_param_->query_record_array_size(); + std::vector vec_f(record_array_size * table_info.dimension_, 0); + for (size_t i = 0; i < record_array_size; i++) { + if (search_param_->query_record_array(i).vector_data().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); + } + uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); + if (query_vec_dim != table_info.dimension_) { + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; + std::string error_msg = "The vector dimension must be equal to the table dimension."; + return Status(error_code, error_msg); + } + + memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), + table_info.dimension_ * sizeof(float)); + } + rc.RecordSection("prepare vector data"); + + // step 6: search vectors + engine::ResultIds result_ids; + engine::ResultDistances result_distances; + auto record_count = (uint64_t)search_param_->query_record_array().size(); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = + "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + + if (file_id_array_.empty()) { + std::vector partition_tags; + for (size_t i = 0; i < search_param_->partition_tag_array_size(); i++) { + partition_tags.emplace_back(search_param_->partition_tag_array(i)); + } + + status = ValidationUtil::ValidatePartitionTags(partition_tags); + if (!status.ok()) { + return status; + } + + status = DBWrapper::DB()->Query(table_name_, partition_tags, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, result_ids, result_distances); + } else { + status = DBWrapper::DB()->QueryByFileID(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, result_ids, result_distances); + } + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("search vectors from engine"); + if (!status.ok()) { + return status; + } + + if (result_ids.empty()) { + return Status::OK(); // empty table + } + + // step 7: construct result array + topk_result_->set_row_num(record_count); + topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), -1); + memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); + topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); + memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), + result_distances.size() * sizeof(float)); + + // step 8: print time cost percent + rc.RecordSection("construct result and send"); + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/SearchRequest.h b/core/src/server/grpc_impl/request/SearchRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..667c7e62ee3a48fe5ae8ee898b27b7d9286d9dd3 --- /dev/null +++ b/core/src/server/grpc_impl/request/SearchRequest.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +class SearchRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResult* response); + + protected: + SearchRequest(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResult* response); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::SearchParam* search_param_; + std::vector file_id_array_; + ::milvus::grpc::TopKQueryResult* topk_result_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..32fa0672c513d690bf105aac9b4679e77e6c3fba --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/ShowPartitionsRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +ShowPartitionsRequest::ShowPartitionsRequest(const std::string& table_name, + ::milvus::grpc::PartitionList* partition_list) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), partition_list_(partition_list) { +} + +BaseRequestPtr +ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) { + return std::shared_ptr(new ShowPartitionsRequest(table_name, partition_list)); +} + +Status +ShowPartitionsRequest::OnExecute() { + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + std::vector schema_array; + auto statuts = DBWrapper::DB()->ShowPartitions(table_name_, schema_array); + if (!statuts.ok()) { + return statuts; + } + + for (auto& schema : schema_array) { + ::milvus::grpc::PartitionParam* param = partition_list_->add_partition_array(); + param->set_table_name(schema.owner_table_); + param->set_partition_name(schema.table_id_); + param->set_tag(schema.partition_tag_); + } + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.h b/core/src/server/grpc_impl/request/ShowPartitionsRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..dd9768b9cb43c9233df19cf536cfc66cde8d080b --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class ShowPartitionsRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); + + protected: + ShowPartitionsRequest(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::PartitionList* partition_list_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..90dbad981fa5242cf409956e1b16af210883e0ca --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/ShowTablesRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +ShowTablesRequest::ShowTablesRequest(::milvus::grpc::TableNameList* table_name_list) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_list_(table_name_list) { +} + +BaseRequestPtr +ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) { + return std::shared_ptr(new ShowTablesRequest(table_name_list)); +} + +Status +ShowTablesRequest::OnExecute() { + std::vector schema_array; + auto statuts = DBWrapper::DB()->AllTables(schema_array); + if (!statuts.ok()) { + return statuts; + } + + for (auto& schema : schema_array) { + table_name_list_->add_table_names(schema.table_id_); + } + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.h b/core/src/server/grpc_impl/request/ShowTablesRequest.h new file mode 100644 index 0000000000000000000000000000000000000000..4818f8bb9da42809d6e0270f5d261f8815b98768 --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class ShowTablesRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(::milvus::grpc::TableNameList* table_name_list); + + protected: + explicit ShowTablesRequest(::milvus::grpc::TableNameList* table_name_list); + + Status + OnExecute() override; + + private: + ::milvus::grpc::TableNameList* table_name_list_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb29b80e6e228fc322ab09f8b5d1df94..26e43619fb405136b5d9ac706a8d3375d621cee0 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31e8ee46ead21b70bafbc930c17be9d8..121196986a31797654da4a256335be3db0232e09 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0a1c27c736581ed3fad49137fc29b47..12b2372fc5b837c2b1dcd341715e42af37284bf3 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -18,6 +18,7 @@ #include "utils/ValidationUtil.h" #include "Log.h" #include "db/engine/ExecutionEngine.h" +#include "utils/StringHelpFunctions.h" #include #ifdef MILVUS_GPU_VERSION @@ -168,11 +169,26 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc return Status::OK(); } +Status +ValidationUtil::ValidatePartitionName(const std::string& partition_name) { + if (partition_name.empty()) { + std::string msg = "Partition name should not be empty."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); + } + + return ValidateTableName(partition_name); +} + Status ValidationUtil::ValidatePartitionTags(const std::vector& partition_tags) { - for (auto& tag : partition_tags) { - if (tag.empty()) { - std::string msg = "Invalid partition tag: " + tag + ". " + "Partition tag should not be empty."; + for (const std::string& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + StringHelpFunctions::TrimStringBlank(valid_tag); + if (valid_tag.empty()) { + std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } @@ -182,7 +198,7 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { #ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -203,7 +219,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { #ifdef MILVUS_GPU_VERSION cudaDeviceProp deviceProp; diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a0714e5815393e9e0cc90d4d149ab84..ab32c35c408a4ced5d8be07c613645bd1a392366 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -55,14 +55,17 @@ class ValidationUtil { static Status ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + static Status + ValidatePartitionName(const std::string& partition_name); + static Status ValidatePartitionTags(const std::vector& partition_tags); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidateGpuIndex(int32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/src/config.h.in b/core/src/version.h.in similarity index 100% rename from core/src/config.h.in rename to core/src/version.h.in diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 4114eb2af40ce173f43562519410f18ca0b3257c..aa4b3c12b8a98ff1e1ac04ef815fdcd24e7b6b77 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; MatchBase(conf); return conf; } @@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; MatchBase(conf); return conf; } @@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; MatchBase(conf); return conf; @@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; if (!(conf->d % 4)) @@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; - double factor = metaconf.size / TYPICAL_COUNT; auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1; - conf->knng = (100 + 100 * scale_factor) * factor; - conf->search_length = (40 + 5 * scale_factor) * factor; - conf->out_degree = (50 + 5 * scale_factor) * factor; - conf->candidate_pool_size = (200 + 100 * scale_factor) * factor; + conf->nprobe = int64_t(conf->nlist * 0.01); + conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); - - // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist - // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d - // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 600dd3a517aed1d5a8ea20011c21e979f1ca7fb0..46fc27eb3b2fbc5a93161090ac90cd2699c81f66 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -46,9 +46,6 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf& metaconf, const IndexType& type); - // virtual void - // Dump(){} - protected: static void MatchBase(knowhere::Config conf); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 8ed19232e947455abe2116ef27302942a1c7c151..5a2296b16e95dfa2c2f4194b1badf313237619ff 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -19,6 +19,8 @@ #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #endif + +#include "scheduler/Utils.h" #include "server/Config.h" #include @@ -35,7 +37,6 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { #ifdef MILVUS_GPU_VERSION - struct GpuResourceSetting { int64_t pinned_memory = 300 * M_BYTE; int64_t temp_memory = 300 * M_BYTE; @@ -47,27 +48,22 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - - int32_t build_index_gpu; - s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + std::vector build_index_gpus; + s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; - gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); + for (auto gpu_id : build_index_gpus) { + gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); + } // get search gpu resource - std::vector pool; - s = config.GetResourceConfigSearchResources(pool); + std::vector search_gpus; + s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; - std::set gpu_ids; - for (auto& resource : pool) { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // invalid - continue; - } - auto gpu_id = std::stoi(resource.substr(3)); + for (auto& gpu_id : search_gpus) { gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); } diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index e7967cbf59b52cddc74a54572160f0adcdb76231..6d681f7600ea0b6e30cdf90107899d3b702c3cbf 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -21,17 +21,19 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "utils/Log.h" #include "wrapper/WrapperException.h" +#include "wrapper/gpu/GPUVecImpl.h" #ifdef MILVUS_GPU_VERSION -#include -#include +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #endif /* * no parameter check in this layer. - * only responible for index combination + * only responsible for index combination */ namespace milvus { @@ -246,5 +248,130 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const return Status::OK(); } +// TODO(linxj): add lock here. +Status +IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) { + try { + dim = cfg->d; + auto dataset = GenDatasetWithIds(nb, dim, xb, ids); + + auto preprocessor = index_->BuildPreprocessor(dataset, cfg); + index_->set_preprocessor(preprocessor); + auto model = index_->Train(dataset, cfg); + index_->set_index_model(model); + index_->Add(dataset, cfg); + + if (auto device_index = std::dynamic_pointer_cast(index_)) { + auto host_index = device_index->CopyGpuToCpu(Config()); + index_ = host_index; + type = ConvertToCpuIndexType(type); + } else { + WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; + return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +Status +IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { + index_->Load(index_binary); + dim = Dimension(); + return Status::OK(); +} + +knowhere::QuantizerPtr +IVFHybridIndex::LoadQuantizer(const Config& conf) { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + return new_idx->LoadQuantizer(conf); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + } +} + +Status +IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + new_idx->SetQuantizer(q); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +Status +IVFHybridIndex::UnsetQuantizer() { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + new_idx->UnsetQuantizer(); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +VecIndexPtr +IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + return std::make_shared(new_idx->LoadData(q, conf), type); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + } + return nullptr; +} + +std::pair +IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) { + try { + // TODO(linxj): Hardcode here + if (auto hybrid_idx = std::dynamic_pointer_cast(index_)) { + auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg); + auto new_idx = std::make_shared(pair.first, type); + return std::make_pair(new_idx, pair.second); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + } + return std::make_pair(nullptr, nullptr); +} + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index 9f68cac0a1bd189460ce2453bdbadd8e22256c45..75f75fb9830d4a12bac0564125fcac52c1dfcc1e 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -22,8 +22,8 @@ #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexSPTAG.h" #include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" #include "utils/Log.h" #ifdef MILVUS_GPU_VERSION diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 911046aa1fad48eb9a3b567c6ab9c89d76a2fc4d..6f3fabc084b0ff274e8dc040e21a2c720dbb442e 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -9,4 +9,6 @@ sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev -sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then + sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +fi diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 01e1054f7e454ee74ed6878e45497dcac3fb1b57..a4d68e74230ce553ff894cccd85908769b83c547 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -66,7 +66,13 @@ set(thirdparty_files ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl/request grpc_request_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_impl_files) +set(grpc_server_files + ${grpc_request_files} + ${grpc_impl_files} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index d8614dd5d14b580de1751db78a3489cdc5a38d87..343e924e8ed2478487fd311b14cb3877e2a41c0d 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -461,6 +461,13 @@ TEST_F(DBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); std::vector xb; BuildVectors(INSERT_BATCH, xb); diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index f8284318380e54357b58bd0ca3157affb9c62f63..93eea2fd2722b349b1e92d4f90324fecf65120e0 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -297,6 +297,14 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); + std::vector xb; BuildVectors(INSERT_BATCH, xb); diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index f87371fe6bd37e1702a9d73013b6220a633ab228..afa1d39006d0306210affb941f8220ae377c0da3 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -155,12 +155,14 @@ DBTest::SetUp() { res_mgr->Clear(); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); - res_mgr->Connect("cpu", "gtx1660", PCIE); +#ifdef MILVUS_GPU_VERSION + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0, true, true)); + res_mgr->Connect("cpu", "0", PCIE); +#endif res_mgr->Start(); milvus::scheduler::SchedInst::GetInstance()->Start(); diff --git a/core/unittest/scheduler/CMakeLists.txt b/core/unittest/scheduler/CMakeLists.txt index 0148441c2d86be314241a8a9ce83f96e5b3352eb..878f0f23a94b028cc0c7f26af6e8b8abd6d89d82 100644 --- a/core/unittest/scheduler/CMakeLists.txt +++ b/core/unittest/scheduler/CMakeLists.txt @@ -21,7 +21,6 @@ set(test_files ${CMAKE_CURRENT_SOURCE_DIR}/test_algorithm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_node.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_normal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_factory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp diff --git a/core/unittest/scheduler/test_normal.cpp b/core/unittest/scheduler/test_normal.cpp deleted file mode 100644 index 20cd39d0bf7ef0fd171407e0af102cb44ba25f20..0000000000000000000000000000000000000000 --- a/core/unittest/scheduler/test_normal.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include "scheduler/ResourceFactory.h" -#include "scheduler/ResourceMgr.h" -#include "scheduler/SchedInst.h" -#include "scheduler/Scheduler.h" -#include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" -#include "utils/Log.h" - -namespace { - -namespace ms = milvus::scheduler; - -} // namespace - -TEST(NormalTest, INST_TEST) { - // ResourceMgr only compose resources, provide unified event - auto res_mgr = ms::ResMgrInst::GetInstance(); - - res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - - auto IO = ms::Connection("IO", 500.0); - res_mgr->Connect("disk", "cpu", IO); - - auto scheduler = ms::SchedInst::GetInstance(); - - res_mgr->Start(); - scheduler->Start(); - - const uint64_t NUM_TASK = 2; - std::vector> tasks; - ms::TableFileSchemaPtr dummy = nullptr; - - auto disks = res_mgr->GetDiskResources(); - ASSERT_FALSE(disks.empty()); - if (auto observe = disks[0].lock()) { - for (uint64_t i = 0; i < NUM_TASK; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - observe->task_table().Put(task); - } - } - - for (auto& task : tasks) { - task->Wait(); - ASSERT_EQ(task->load_count_, 1); - ASSERT_EQ(task->exec_count_, 1); - } - - scheduler->Stop(); - res_mgr->Stop(); -} diff --git a/core/unittest/scheduler/test_resource.cpp b/core/unittest/scheduler/test_resource.cpp index 6fe85ec5588c3127636deebdf6bd80a1bc26fedb..101dcd397605e16e7214cb6916eb3a7358757b7d 100644 --- a/core/unittest/scheduler/test_resource.cpp +++ b/core/unittest/scheduler/test_resource.cpp @@ -24,7 +24,7 @@ #include "scheduler/resource/TestResource.h" #include "scheduler/task/Task.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" +#include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { @@ -182,8 +182,10 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(disk_resource_); auto task = std::make_shared(dummy, label); + std::vector path{disk_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); disk_resource_->task_table().Put(task); } @@ -208,8 +210,10 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(cpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{cpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); cpu_resource_->task_table().Put(task); } @@ -234,8 +238,10 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(gpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{gpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); gpu_resource_->task_table().Put(task); } @@ -260,8 +266,10 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(test_resource_); auto task = std::make_shared(dummy, label); + std::vector path{test_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); test_resource_->task_table().Put(task); } diff --git a/core/unittest/scheduler/test_resource_mgr.cpp b/core/unittest/scheduler/test_resource_mgr.cpp index b9127060bdbb5341877c8f57e26e756172419299..d6495971c45712e1249de2a581696468024e673e 100644 --- a/core/unittest/scheduler/test_resource_mgr.cpp +++ b/core/unittest/scheduler/test_resource_mgr.cpp @@ -22,7 +22,6 @@ #include "scheduler/resource/GpuResource.h" #include "scheduler/resource/TestResource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" namespace milvus { namespace scheduler { @@ -187,8 +186,7 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { auto callback = [&](EventPtr event) { flag = true; }; mgr1_->RegisterSubscriber(callback); TableFileSchemaPtr dummy = nullptr; - auto label = std::make_shared(); - disk_res->task_table().Put(std::make_shared(dummy, label)); + disk_res->task_table().Put(std::make_shared(dummy, nullptr)); sleep(1); ASSERT_TRUE(flag); } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index 22fb9be7235514e7a866edb04a4e249a2f1f54b5..72538113c3ad8d7098d7b543724366efa94324c3 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -23,7 +23,6 @@ #include "scheduler/Scheduler.h" #include "scheduler/resource/Resource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" #include "scheduler/tasklabel/SpecResLabel.h" #include "utils/Error.h" #include "wrapper/VecIndex.h" @@ -150,46 +149,6 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) { cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj); } -TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy = std::make_shared(); - dummy->location_ = "location"; - - insert_dummy_index_into_gpu_cache(1); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().size(), NUM); -} - -TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy1 = std::make_shared(); - dummy1->location_ = "location"; - - tasks.clear(); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy1, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - // ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM); -} - class SchedulerTest2 : public testing::Test { protected: void diff --git a/core/unittest/scheduler/test_tasktable.cpp b/core/unittest/scheduler/test_tasktable.cpp index 28a2e29c98433501967d45d4f3d163e02374e545..3fa1beabebf15d953a3496192fdc4d97976aefbb 100644 --- a/core/unittest/scheduler/test_tasktable.cpp +++ b/core/unittest/scheduler/test_tasktable.cpp @@ -18,7 +18,6 @@ #include #include "scheduler/TaskTable.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" /************ TaskTableBaseTest ************/ @@ -162,9 +161,8 @@ class TaskTableBaseTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; - auto label = std::make_shared(); - task1_ = std::make_shared(dummy, label); - task2_ = std::make_shared(dummy, label); + task1_ = std::make_shared(dummy, nullptr); + task2_ = std::make_shared(dummy, nullptr); } milvus::scheduler::TaskPtr invalid_task_; @@ -320,8 +318,7 @@ class TaskTableAdvanceTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); + auto task = std::make_shared(dummy, nullptr); table1_.Put(task); } diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732dfb2a74cb437cef6803342ccc5c9fd8..664a08d63105f54d32271462464f1ada41a9dcbf 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; std::string str_val; - int32_t int32_val; int64_t int64_val; float float_val; bool bool_val; @@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(str_val == db_backend_url); - int32_t db_archive_disk_threshold = 100; + int64_t db_archive_disk_threshold = 100; s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int32_val); + s = config.GetDBConfigArchiveDiskThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_disk_threshold); + ASSERT_TRUE(int64_val == db_archive_disk_threshold); - int32_t db_archive_days_threshold = 365; + int64_t db_archive_days_threshold = 365; s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int32_val); + s = config.GetDBConfigArchiveDaysThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_days_threshold); + ASSERT_TRUE(int64_val == db_archive_days_threshold); - int32_t db_insert_buffer_size = 2; + int64_t db_insert_buffer_size = 2; s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int32_val); + s = config.GetDBConfigInsertBufferSize(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_insert_buffer_size); + ASSERT_TRUE(int64_val == db_insert_buffer_size); /* metric config */ bool metric_enable_monitor = false; @@ -216,21 +215,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetCacheConfigCpuCacheThreshold(float_val); ASSERT_TRUE(float_val == cache_cpu_cache_threshold); -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity = 1; - s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheCapacity(int64_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int64_val == cache_gpu_cache_capacity); - - float cache_gpu_cache_threshold = 0.2; - s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheThreshold(float_val); - ASSERT_TRUE(float_val == cache_gpu_cache_threshold); -#endif - bool cache_insert_data = true; s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)); ASSERT_TRUE(s.ok()); @@ -238,63 +222,75 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ - int32_t engine_use_blas_threshold = 50; + int64_t engine_use_blas_threshold = 50; s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int32_val); + s = config.GetEngineConfigUseBlasThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_use_blas_threshold); + ASSERT_TRUE(int64_val == engine_use_blas_threshold); - int32_t engine_omp_thread_num = 8; + int64_t engine_omp_thread_num = 8; s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int32_val); + s = config.GetEngineConfigOmpThreadNum(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_omp_thread_num); + ASSERT_TRUE(int64_val == engine_omp_thread_num); - int32_t engine_gpu_search_threshold = 800; + int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int32_val); + s = config.GetEngineConfigGpuSearchThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + ASSERT_TRUE(int64_val == engine_gpu_search_threshold); - /* resource config */ - std::string resource_mode = "simple"; - s = config.SetResourceConfigMode(resource_mode); + /* gpu resource config */ + bool resource_enable_gpu = true; + s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigMode(str_val); + s = config.GetGpuResourceConfigEnable(bool_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(str_val == resource_mode); + ASSERT_TRUE(bool_val == resource_enable_gpu); -#ifdef MILVUS_CPU_VERSION - std::vector search_resources = {"cpu"}; -#else - std::vector search_resources = {"cpu", "gpu0"}; -#endif - std::vector res_vec; - std::string res_str; +#ifdef MILVUS_GPU_VERSION + int64_t gpu_cache_capacity = 1; + s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigCacheCapacity(int64_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int64_val == gpu_cache_capacity); + + float gpu_cache_threshold = 0.2; + s = config.SetGpuResourceConfigCacheThreshold(std::to_string(gpu_cache_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigCacheThreshold(float_val); + ASSERT_TRUE(float_val == gpu_cache_threshold); + + std::vector search_resources = {"gpu0"}; + std::vector search_res_vec; + std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( - search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); - s = config.SetResourceConfigSearchResources(res_str); + search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); + s = config.SetGpuResourceConfigSearchResources(search_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigSearchResources(res_vec); + s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(search_resources[i] == res_vec[i]); + ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } -#ifdef MILVUS_CPU_VERSION - int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; - s = config.SetResourceConfigIndexBuildDevice("cpu"); -#else - int32_t resource_index_build_device = 0; - s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); -#endif + std::vector build_index_resources = {"gpu0"}; + std::vector build_index_res_vec; + std::string build_index_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); + s = config.SetGpuResourceConfigBuildIndexResources(build_index_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); + s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == resource_index_build_device); + for (size_t i = 0; i < build_index_resources.size(); i++) { + ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + } +#endif } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -381,18 +377,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetCacheConfigCpuCacheThreshold("1.0"); ASSERT_FALSE(s.ok()); -#ifdef MILVUS_GPU_VERSION - s = config.SetCacheConfigGpuCacheCapacity("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheCapacity("128"); - ASSERT_FALSE(s.ok()); - - s = config.SetCacheConfigGpuCacheThreshold("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheThreshold("1.0"); - ASSERT_FALSE(s.ok()); -#endif - s = config.SetCacheConfigCacheInsertData("N"); ASSERT_FALSE(s.ok()); @@ -408,20 +392,29 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetEngineConfigGpuSearchThreshold("-1"); ASSERT_FALSE(s.ok()); - /* resource config */ - s = config.SetResourceConfigMode("default"); + /* gpu resource config */ + s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("gpu10"); +#ifdef MILVUS_GPU_VERSION + s = config.SetGpuResourceConfigCacheCapacity("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheCapacity("128"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("cpu"); - ASSERT_TRUE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("1.0"); + ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gup2"); + s = config.SetGpuResourceConfigSearchResources("gpu10"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gpu16"); + + s = config.SetGpuResourceConfigBuildIndexResources("gup2"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigBuildIndexResources("gpu16"); ASSERT_FALSE(s.ok()); +#endif } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { @@ -438,4 +431,3 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) { s = config.ResetDefaultConfig(); ASSERT_TRUE(s.ok()); } - diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 76c10769a77dc39b54ae1607146e6bee42f8a95a..5753c684223c24a6b434351b06387ed8198386f8 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -22,8 +22,8 @@ #include "server/Server.h" #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestScheduler.h" -#include "server/grpc_impl/GrpcRequestTask.h" -#include "src/config.h" +#include "server/grpc_impl/request/GrpcBaseRequest.h" +#include "src/version.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" @@ -85,7 +85,6 @@ class RpcHandlerTest : public testing::Test { // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - milvus::server::Config::GetInstance().SetResourceConfigMode("single"); milvus::server::DBWrapper::GetInstance().StartService(); // initialize handler, create table @@ -410,7 +409,7 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) { partition_parm.set_partition_name(partition_name); handler->DropPartition(&context, &partition_parm, &response); - ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code()); + ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code()); } TEST_F(RpcHandlerTest, CMD_TEST) { @@ -453,20 +452,20 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) { ////////////////////////////////////////////////////////////////////// namespace { -class DummyTask : public milvus::server::grpc::GrpcBaseTask { +class DummyRequest : public milvus::server::grpc::GrpcBaseRequest { public: milvus::Status OnExecute() override { return milvus::Status::OK(); } - static milvus::server::grpc::BaseTaskPtr + static milvus::server::grpc::BaseRequestPtr Create(std::string& dummy) { - return std::shared_ptr(new DummyTask(dummy)); + return std::shared_ptr(new DummyRequest(dummy)); } public: - explicit DummyTask(std::string& dummy) : GrpcBaseTask(dummy) { + explicit DummyRequest(std::string& dummy) : GrpcBaseRequest(dummy) { } }; @@ -475,27 +474,27 @@ class RpcSchedulerTest : public testing::Test { void SetUp() override { std::string dummy = "dql"; - task_ptr = std::make_shared(dummy); + request_ptr = std::make_shared(dummy); } - std::shared_ptr task_ptr; + std::shared_ptr request_ptr; }; } // namespace TEST_F(RpcSchedulerTest, BASE_TASK_TEST) { - auto status = task_ptr->Execute(); + auto status = request_ptr->Execute(); ASSERT_TRUE(status.ok()); milvus::server::grpc::GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; std::string dummy = "dql"; - milvus::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); - milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); + milvus::server::grpc::BaseRequestPtr base_task_ptr = DummyRequest::Create(dummy); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecRequest(base_task_ptr, &grpc_status); - milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); - task_ptr = nullptr; - milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteRequest(request_ptr); + request_ptr = nullptr; + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteRequest(request_ptr); milvus::server::grpc::GrpcRequestScheduler::GetInstance().Stop(); } diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 68400f2454b58b1a1c7c6e2fb582ab16563a9ccd..e5884cac6546d32b796b0fbb92cd69e30eae1f6a 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - uint32_t thread_cnt = 0; + int64_t thread_cnt = 0; milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 413db60e37688c989cde742c26665b8e6715f195..4019c0f63c82750f9bf482e634524a20c11063e0 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -16,13 +16,13 @@ // under the License. #include "easyloggingpp/easylogging++.h" -#include "wrapper/VecIndex.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #endif #include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include @@ -57,10 +57,6 @@ class KnowhereWrapperTest index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); - - // conf->k = k; - // conf->d = dim; - // conf->gpu_id = DEVICEID; } void @@ -80,8 +76,7 @@ class KnowhereWrapperTest INSTANTIATE_TEST_CASE_P( WrapperParam, KnowhereWrapperTest, Values( -//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] - + //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] #ifdef MILVUS_GPU_VERSION std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10), @@ -98,6 +93,7 @@ INSTANTIATE_TEST_CASE_P( TEST_P(KnowhereWrapperTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); + // conf->Dump(); auto elems = nq * k; std::vector res_ids(elems); @@ -192,3 +188,119 @@ TEST(whatever, test_config) { auto pq_conf = std::make_shared(); pq_conf->Match(conf); } + +// #include "knowhere/index/vector_index/IndexIDMAP.h" +// #include "src/wrapper/VecImpl.h" +// #include "src/index/unittest/utils.h" +// The two case below prove NSG is concern with data distribution +// Further work: 1. Use right basedata and pass it by milvus +// a. batch size is 100000 [Pass] +// b. transfer all at once [Pass] +// 2. Use SIFT1M in test and check time cost [] +// TEST_P(KnowhereWrapperTest, nsgwithidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// auto ori_ids = ids; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); +// assert(xb.size() == nb*dim); +// //#define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// assert(temp_xb.size() == 1000000*128); +// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf); +// #endif +// } + +// TEST_P(KnowhereWrapperTest, nsgwithsidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// #define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf); +// #endif + +// // The code use to store raw base data +// FileIOWriter writer("/tmp/newraw"); +// ori_xb.shrink_to_fit(); +// std::cout << "size" << ori_xb.size(); +// writer(static_cast(ori_xb.data()), ori_xb.size()* sizeof(float)); +// std::cout << "Finish!" << std::endl; +// } + +// void load_data(char* filename, float*& data, unsigned& num, +// unsigned& dim) { // load data with sift10K pattern +// std::ifstream in(filename, std::ios::binary); +// if (!in.is_open()) { +// std::cout << "open file error" << std::endl; +// exit(-1); +// } +// in.read((char*)&dim, 4); +// in.seekg(0, std::ios::end); +// std::ios::pos_type ss = in.tellg(); +// size_t fsize = (size_t)ss; +// num = (unsigned)(fsize / (dim + 1) / 4); +// data = new float[(size_t)num * (size_t)dim]; + +// in.seekg(0, std::ios::beg); +// for (size_t i = 0; i < num; i++) { +// in.seekg(4, std::ios::cur); +// in.read((char*)(data + i * dim), dim * 4); +// } +// in.close(); +// } + +// TEST_P(KnowhereWrapperTest, Sift1M) { +// float* data = nullptr; +// unsigned points_num, dim; +// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num, +// dim); std::cout << points_num << " " << dim << std::endl; + +// index_->BuildAll(points_num, data, ids.data(), conf); +// } diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index 00a1db2c88351cd2faad1bba9a99d7ae723fff86..96b9e643f5d07251362f9ed72b392ad09e297461 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -117,6 +117,11 @@ void DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis) { + xb.clear(); + xq.clear(); + ids.clear(); + gt_ids.clear(); + gt_dis.clear(); xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index 1eaa7ae4ec7d70557acf8ceabbd3d85c8a1bb51a..05dc92f2d69180c3fc2d521dddd1f33248114407 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -25,7 +25,6 @@ #include #include - #include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -90,17 +89,20 @@ class ParamGenerator { return instance; } - knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); } - knowhere::Config Gen(const milvus::engine::IndexType& type) { + knowhere::Config + Gen(const milvus::engine::IndexType& type) { switch (type) { case milvus::engine::IndexType::FAISS_IDMAP: { auto tempconf = std::make_shared(); diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..64409d82407aceb06c34fa51b8202013accc110c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,11 @@ +# Docs + +This repository contains test reports on the search performance of different index types on standalone Milvus. + +The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provide results on the following measures: + +- Query Elapsed Time: Time cost (in seconds) to run a query. + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved. + +Test variables are `nq` and `topk`. diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md new file mode 100644 index 0000000000000000000000000000000000000000..7bea764df03aed0d7f0aa71cee9a1dcb9b0b3f10 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -0,0 +1,206 @@ +# milvus_ivfsq8_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8 index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + + > Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 200, 400, 600, 800, 1000], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 17ms in GPU Mode. + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 27ms in CPU Mode. + + + +**Conclusion** + +The query elapsed time in CPU Mode increases quickly with nq, while in GPU Mode query elapsed time increases much slower. When nq is small, CPU Mode consumes less time than GPU Mode. However, as nq becomes larger, GPU Mode shows its advantage against CPU Mode. + +The query elapsed time in GPU Mode consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is smaller than 500, index CPU-to-GPU copy time cannot be amortized efficiently, CPU Mode is a better choice; when nq is larger than 500, choosing GPU Mode is better. + +Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md new file mode 100644 index 0000000000000000000000000000000000000000..a6e5e75ea46f8b0e1dcef7afc375c3b57f564307 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -0,0 +1,207 @@ +# milvus_ivfsq8_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +内存: 503GB + +Docker版本: 18.09 + +NVIDIA Driver版本: 430.34 + +Milvus版本: 0.5.3 + +SDK接口: Python 3.6.8 + +pymilvus版本: 0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源: sift1b +- 数据类型: hdf5 + +关于该数据集的详细信息请参考 : http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量: + + - nq (被查询向量的数量) + + > 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组。 + +- Recall: 实际返回的正确结果占总数之比 . 影响Recall的变量: + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 200, 400, 600, 800, 1000]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约17毫秒。 + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约27毫秒。 + + + +**总结** + +在CPU模式下查询耗时随nq的增长快速增大，而在GPU模式下查询耗时的增大则缓慢许多。当nq较小时，CPU模式比GPU模式耗时更少。但当nq足够大时，GPU模式则更具有优势。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。 + +和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。 + diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md new file mode 100755 index 0000000000000000000000000000000000000000..042d059684ae756211a41dc955536a28050d8fe7 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md @@ -0,0 +1,179 @@ +# milvus_ivfsq8h_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8H index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, please check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + +> Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]. + > + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +When nq is 1800, the query time cost of a 128-dimension vector is around 11ms. + + + +**Conclusion** + +When nq < 1200, the query elapsed time increases quickly with nq; when nq > 1200, the query elapsed time increases much slower. It is because gpu_search_threshold is set to 1200, when nq < 1200, CPU is chosen to do the query, otherwise GPU is chosen. Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + +The query elapsed time consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is larger enough, index CPU-to-GPU copy time can be amortized efficiently. So Milvus performs well through setting suitable gpu_search_threshold. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. The usage of CPU or GPU and different topk are not related to recall. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md new file mode 100644 index 0000000000000000000000000000000000000000..b50d00f9bdbdbd7f1c423c8dde489563b8a8310a --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -0,0 +1,180 @@ +# milvus_ivfsq8h_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8h索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统：CentOS Linux release 7.6.1810 (Core) + +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0：GeForce GTX 1080 + +GPU1：GeForce GTX 1080 + +内存：503GB + +Docker版本：18.09 + +NVIDIA Driver版本：430.34 + +Milvus版本：0.5.3 + +SDK接口：Python 3.6.8 + +pymilvus版本：0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源：sift1b +- 数据类型：hdf5 + +关于该数据集的详细信息请参考：http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： + + - nq (被查询向量的数量) + +> 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组。 + > + +- Recall: 实际返回的正确结果占总数之比。影响Recall的变量： + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +当nq为1800时，查询一条128维向量需要耗时约11毫秒。 + + + +**总结** + +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。CPU/GPU的使用以及topk的值与召回率的大小无关。 + diff --git a/install.md b/install.md index 6711b41f76011f1193cab429d34750ca839b7b1d..f53586af36831e7cde470b6c9dc65dc5c3d915e4 100644 --- a/install.md +++ b/install.md @@ -3,6 +3,9 @@ ## Software requirements - Ubuntu 18.04 or higher + + If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + - CMake 3.12 or higher ##### For GPU version, you will also need: diff --git a/shards/mishards/__init__.py b/shards/mishards/__init__.py index a3c55c4ae38469f596af29f3e03be211da1335d9..55594220d3aae9b33635ceda6bf3ca51ec07a833 100644 --- a/shards/mishards/__init__.py +++ b/shards/mishards/__init__.py @@ -11,7 +11,9 @@ grpc_server = Server() def create_app(testing_config=None): config = testing_config if testing_config else settings.DefaultConfig - db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO) + db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO, pool_size=config.SQL_POOL_SIZE, + pool_recycle=config.SQL_POOL_RECYCLE, pool_timeout=config.SQL_POOL_TIMEOUT, + pool_pre_ping=config.SQL_POOL_PRE_PING, max_overflow=config.SQL_MAX_OVERFLOW) from mishards.connections import ConnectionMgr connect_mgr = ConnectionMgr() diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099df98d3d600eca5b20d2fe9a3275420..50e214ec9aa0726d4653a3d093a7ea623da2fbc3 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/db_base.py b/shards/mishards/db_base.py index 5f2eee9ba15bde17a53c30e8640fbae21918d645..e55c330352488ea27cf13d2ffb961f958cb70247 100644 --- a/shards/mishards/db_base.py +++ b/shards/mishards/db_base.py @@ -23,15 +23,12 @@ class DB: uri and self.init_db(uri, echo) self.session_factory = scoped_session(sessionmaker(class_=LocalSession, db=self)) - def init_db(self, uri, echo=False): + def init_db(self, uri, echo=False, pool_size=100, pool_recycle=5, pool_timeout=30, pool_pre_ping=True, max_overflow=0): url = make_url(uri) if url.get_backend_name() == 'sqlite': self.engine = create_engine(url) else: - self.engine = create_engine(uri, pool_size=100, pool_recycle=5, pool_timeout=30, - pool_pre_ping=True, - echo=echo, - max_overflow=0) + self.engine = create_engine(uri, pool_size, pool_recycle, pool_timeout, pool_pre_ping, echo, max_overflow) self.uri = uri self.url = url diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae60eb60be610fba14dd0729151113556..fc0ee0fa2b660a50fd92d1aee21eb62e5eb5be83 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,71 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + source_diss.extend(diss) + diss_t = enumerate(source_diss) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk, may need to compare with topk + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +141,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +273,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +307,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method diff --git a/shards/mishards/settings.py b/shards/mishards/settings.py index 8d7361dddcfbe3371df25456e12eba00e33ce111..832f1639ea720f861a5615b3667c87a3211bf230 100644 --- a/shards/mishards/settings.py +++ b/shards/mishards/settings.py @@ -50,10 +50,16 @@ class TracingConfig: } } + max_overflow=0 class DefaultConfig: SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_URI') SQL_ECHO = env.bool('SQL_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_PLUGIN_PATH = env.str('TRACER_PLUGIN_PATH', '') TRACER_CLASS_NAME = env.str('TRACER_CLASS_NAME', '') ROUTER_PLUGIN_PATH = env.str('ROUTER_PLUGIN_PATH', '') @@ -65,5 +71,10 @@ class DefaultConfig: class TestingConfig(DefaultConfig): SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_TEST_URI', '') SQL_ECHO = env.bool('SQL_TEST_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_CLASS_NAME = env.str('TRACER_CLASS_TEST_NAME', '') ROUTER_CLASS_NAME = env.str('ROUTER_CLASS_TEST_NAME', 'FileBasedHashRingRouter') diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a0628b18b78b058386523fed2a78a7747..426ee0b7040a0ae2e2ecb451d366e6b21b9dba6d 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -14,8 +14,7 @@ py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 -#pymilvus==0.2.0 +pymilvus==0.2.5 pyparsing==2.4.0 pytest==4.6.3 pytest-level==0.1.1 diff --git a/tests/milvus_python_test/conftest.py b/tests/milvus_python_test/conftest.py index 8bab824606685920d4f28758352d1eb5472f11b6..dc58063a451c382503e3a9ae78c955df3f5682cc 100644 --- a/tests/milvus_python_test/conftest.py +++ b/tests/milvus_python_test/conftest.py @@ -42,16 +42,22 @@ def connect(request): port = request.config.getoption("--port") milvus = Milvus() try: - milvus.connect(host=ip, port=port) - except: + status = milvus.connect(host=ip, port=port) + logging.getLogger().info(status) + if not status.OK(): + # try again + logging.getLogger().info("------------------------------------") + logging.getLogger().info("Try to connect again") + logging.getLogger().info("------------------------------------") + res = milvus.connect(host=ip, port=port) + except Exception as e: + logging.getLogger().error(str(e)) pytest.exit("Milvus server can not connected, exit pytest ...") - def fin(): try: milvus.disconnect() except: pass - request.addfinalizer(fin) return milvus @@ -129,4 +135,4 @@ def ip_table(request, connect): request.addfinalizer(teardown) - return table_name \ No newline at end of file + return table_name diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index 96ce1d3bdfc85b352b9162b1f7c4d65dbed79b39..dd7e80c1f9682c7dcd864a22a0ed61ab6c0532ef 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -249,8 +249,8 @@ class TestConnect: ''' milvus = Milvus() uri_value = "tcp://%s:19540" % args["ip"] - milvus.connect(host=args["ip"], port="", uri=uri_value) - assert milvus.connected() + with pytest.raises(Exception) as e: + milvus.connect(host=args["ip"], port="", uri=uri_value) def test_connect_param_priority_uri(self, args): '''