Merge remote-tracking branch 'main/0.5.1' into 0.5.1

Former-commit-id: 42769ae0088a9f79eb3050992fcb9eb7063f9cf4

CHANGELOG.md

@@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#104 - test_scheduler core dump
 - \#115 - Using new structure for tasktable
 - \#139 - New config opion use_gpu_threshold
+- \#146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT
 
 ## Improvement
 - \#64 - Improvement dump function in scheduler
@@ -23,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#122 - Add unique id for Job
 - \#130 - Set task state MOVED after resource copy it completed
 - \#149 - Improve large query optimizer pass
+- \#156 - Not return error when search_resources and index_build_device set cpu
 
 ## Task
 

ci/jenkins/Jenkinsfile

+#!/usr/bin/env groovy
+
 String cron_timezone = "TZ=Asia/Shanghai"
 String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : ""
 cron_string =  BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string
@@ -16,7 +18,6 @@ pipeline {
 
     parameters{
         choice choices: ['Release', 'Debug'], description: '', name: 'BUILD_TYPE'
-        string defaultValue: 'cf1434e7-5a4b-4d25-82e8-88d667aef9e5', description: 'GIT CREDENTIALS ID', name: 'GIT_CREDENTIALS_ID', trim: true
         string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true
         string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true
         string defaultValue: 'http://192.168.1.202/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true
@@ -56,7 +57,7 @@ pipeline {
                             steps {
                                 container('milvus-build-env') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
                                     }
                                 }
                             }
@@ -65,7 +66,7 @@ pipeline {
                             steps {
                                 container('milvus-build-env') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/coverage.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
                                     }
                                 }
                             }
@@ -74,7 +75,7 @@ pipeline {
                             steps {
                                 container('milvus-build-env') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/package.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
                                     }
                                 }
                             }
@@ -96,7 +97,7 @@ pipeline {
                             steps {
                                 container('publish-images'){
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/publishImages.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
                                     }
                                 }
                             }
@@ -118,7 +119,7 @@ pipeline {
                             steps {
                                 container('milvus-test-env') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
                                     }
                                 }
                             }
@@ -130,9 +131,9 @@ pipeline {
                                     script {
                                         boolean isNightlyTest = isTimeTriggeredBuild()
                                         if (isNightlyTest) {
-                                            load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy"
+                                            load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
                                         } else {
-                                            load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy"
+                                            load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
                                         }
                                     }
                                 }
@@ -143,7 +144,7 @@ pipeline {
                             steps {
                                 container('milvus-test-env') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
                                     }
                                 }
                             }
@@ -153,7 +154,7 @@ pipeline {
                         unsuccessful {
                             container('milvus-test-env') {
                                 script {
-                                    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+                                    load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
                                 }
                             }
                         }

ci/jenkins/jenkinsfile/deploySingle2Dev.groovy → ci/jenkins/step/deploySingle2Dev.groovy

 sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
 sh 'helm repo update'
 dir ('milvus-helm') {
-    checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+    checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
     dir ("milvus-gpu") {
         sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
     }

ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy → ci/jenkins/step/singleDevNightlyTest.groovy

@@ -8,7 +8,7 @@ timeout(time: 90, unit: 'MINUTES') {
 
     if (!fileExists('milvus-helm')) {
         dir ("milvus-helm") {
-            checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+            checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
         }
     }
     dir ("milvus-helm") {

ci/jenkins/jenkinsfile/singleDevTest.groovy → ci/jenkins/step/singleDevTest.groovy

@@ -10,7 +10,7 @@ timeout(time: 60, unit: 'MINUTES') {
 
     // if (!fileExists('milvus-helm')) {
     //     dir ("milvus-helm") {
-    //         checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+    //         checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
     //     }
     // }
     // dir ("milvus-helm") {

core/src/scheduler/SchedInst.h

@@ -23,10 +23,14 @@
 #include "Scheduler.h"
 #include "optimizer/HybridPass.h"
 #include "optimizer/LargeSQ8HPass.h"
+#include "optimizer/OnlyCPUPass.h"
+#include "optimizer/OnlyGPUPass.h"
 #include "optimizer/Optimizer.h"
+#include "server/Config.h"
 
 #include <memory>
 #include <mutex>
+#include <string>
 #include <vector>
 
 namespace milvus {
@@ -93,9 +97,21 @@ class OptimizerInst {
         if (instance == nullptr) {
             std::lock_guard<std::mutex> lock(mutex_);
             if (instance == nullptr) {
+                server::Config& config = server::Config::GetInstance();
+                std::vector<std::string> search_resources;
+                bool has_cpu = false;
+                config.GetResourceConfigSearchResources(search_resources);
+                for (auto& resource : search_resources) {
+                    if (resource == "cpu") {
+                        has_cpu = true;
+                    }
+                }
+
                 std::vector<PassPtr> pass_list;
                 pass_list.push_back(std::make_shared<LargeSQ8HPass>());
                 pass_list.push_back(std::make_shared<HybridPass>());
+                pass_list.push_back(std::make_shared<OnlyCPUPass>());
+                pass_list.push_back(std::make_shared<OnlyGPUPass>(has_cpu));
                 instance = std::make_shared<Optimizer>(pass_list);
             }
         }

core/src/scheduler/optimizer/OnlyCPUPass.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "scheduler/optimizer/OnlyCPUPass.h"
+#include "scheduler/SchedInst.h"
+#include "scheduler/Utils.h"
+#include "scheduler/task/SearchTask.h"
+#include "scheduler/tasklabel/SpecResLabel.h"
+
+namespace milvus {
+namespace scheduler {
+
+bool
+OnlyCPUPass::Run(const TaskPtr& task) {
+    if (task->Type() != TaskType::SearchTask)
+        return false;
+    auto search_task = std::static_pointer_cast<XSearchTask>(task);
+    if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 &&
+        search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) {
+        return false;
+    }
+
+    auto gpu_id = get_gpu_pool();
+    if (not gpu_id.empty())
+        return false;
+
+    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
+    auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));
+    task->label() = label;
+    return true;
+}
+
+}  // namespace scheduler
+}  // namespace milvus

core/src/scheduler/optimizer/OnlyCPUPass.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <condition_variable>
+#include <deque>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#include "Pass.h"
+
+namespace milvus {
+namespace scheduler {
+
+class OnlyCPUPass : public Pass {
+ public:
+    OnlyCPUPass() = default;
+
+ public:
+    bool
+    Run(const TaskPtr& task) override;
+};
+
+using OnlyCPUPassPtr = std::shared_ptr<OnlyCPUPass>;
+
+}  // namespace scheduler
+}  // namespace milvus

core/src/scheduler/optimizer/OnlyGPUPass.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "scheduler/optimizer/OnlyGPUPass.h"
+#include "scheduler/SchedInst.h"
+#include "scheduler/Utils.h"
+#include "scheduler/task/SearchTask.h"
+#include "scheduler/tasklabel/SpecResLabel.h"
+
+namespace milvus {
+namespace scheduler {
+
+OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) {
+}
+
+bool
+OnlyGPUPass::Run(const TaskPtr& task) {
+    if (task->Type() != TaskType::SearchTask || has_cpu_)
+        return false;
+
+    auto search_task = std::static_pointer_cast<XSearchTask>(task);
+    if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 &&
+        search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) {
+        return false;
+    }
+
+    auto gpu_id = get_gpu_pool();
+    if (gpu_id.empty())
+        return false;
+
+    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]);
+    auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));
+    task->label() = label;
+
+    specified_gpu_id_ = specified_gpu_id_++ % gpu_id.size();
+    return true;
+}
+
+}  // namespace scheduler
+}  // namespace milvus

core/src/scheduler/optimizer/OnlyGPUPass.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <condition_variable>
+#include <deque>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#include "Pass.h"
+
+namespace milvus {
+namespace scheduler {
+
+class OnlyGPUPass : public Pass {
+ public:
+    explicit OnlyGPUPass(bool has_cpu);
+
+ public:
+    bool
+    Run(const TaskPtr& task) override;
+
+ private:
+    uint64_t specified_gpu_id_ = 0;
+    bool has_cpu_ = false;
+};
+
+using OnlyGPUPassPtr = std::shared_ptr<OnlyGPUPass>;
+
+}  // namespace scheduler
+}  // namespace milvus

core/src/server/Config.cpp

@@ -714,9 +714,12 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
         return Status(SERVER_INVALID_ARGUMENT, msg);
     }
 
-    for (auto& gpu_device : value) {
-        if (!CheckGpuDevice(gpu_device).ok()) {
-            std::string msg = "Invalid search resource: " + gpu_device +
+    for (auto& device : value) {
+        if (device == "cpu") {
+            continue;
+        }
+        if (!CheckGpuDevice(device).ok()) {
+            std::string msg = "Invalid search resource: " + device +
                               ". Possible reason: resource_config.search_resources does not match your hardware.";
             return Status(SERVER_INVALID_ARGUMENT, msg);
         }
@@ -726,6 +729,9 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
 
 Status
 Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
+    if (value == "cpu") {
+        return Status::OK();
+    }
     if (!CheckGpuDevice(value).ok()) {
         std::string msg = "Invalid index build device: " + value +
                           ". Possible reason: resource_config.index_build_device does not match your hardware.";