Merge remote-tracking branch 'source/0.6.0' into 0.6.0

CHANGELOG.md

@@ -30,6 +30,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#322 - Add option to enable / disable prometheus
 - \#358 - Add more information in build.sh and install.md
 - \#255 - Add ivfsq8 test report detailed version
+- \#404 - Add virtual method Init() in Pass abstract class
+- \#409 - Add a Fallback pass in optimizer
 
 ## Task
 

ci/jenkins/step/deploySingle2Dev.groovy

@@ -3,11 +3,7 @@ sh 'helm repo update'
 dir ('milvus-helm') {
     checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]])
     dir ("milvus") {
-    	if ("${env.BINRARY_VERSION}" == "gpu") {
-        	sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
-    	} else {
-    		sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ."
-    	}
+        sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
     }
 }
 

ci/jenkins/step/singleDevNightlyTest.groovy

@@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') {
     }
     dir ("milvus-helm") {
         dir ("milvus") {
-            if ("${env.BINRARY_VERSION}" == "gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
-            } else {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
-            }
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
         }
     }
     dir ("tests/milvus_python_test") {

ci/jenkins/step/singleDevTest.groovy

@@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') {
     // }
     // dir ("milvus-helm") {
     //     dir ("milvus") {
-    //         if ("${env.BINRARY_VERSION}" == "gpu") {
-    //             sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
-    //         } else {
-    //             sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
-    //         }
+    //         sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
     //     }
     // }
     // dir ("tests/milvus_python_test") {

core/src/scheduler/SchedInst.cpp

@@ -84,6 +84,7 @@ load_simple_config() {
 void
 StartSchedulerService() {
     load_simple_config();
+    OptimizerInst::GetInstance()->Init();
     ResMgrInst::GetInstance()->Start();
     SchedInst::GetInstance()->Start();
     JobMgrInst::GetInstance()->Start();

core/src/scheduler/SchedInst.h

@@ -21,6 +21,7 @@
 #include "JobMgr.h"
 #include "ResourceMgr.h"
 #include "Scheduler.h"
+#include "optimizer/FallbackPass.h"
 #include "optimizer/HybridPass.h"
 #include "optimizer/LargeSQ8HPass.h"
 #include "optimizer/OnlyCPUPass.h"
@@ -112,6 +113,7 @@ class OptimizerInst {
                 pass_list.push_back(std::make_shared<HybridPass>());
                 pass_list.push_back(std::make_shared<OnlyCPUPass>());
                 pass_list.push_back(std::make_shared<OnlyGPUPass>(has_cpu));
+                pass_list.push_back(std::make_shared<FallbackPass>());
                 instance = std::make_shared<Optimizer>(pass_list);
             }
         }

core/src/scheduler/optimizer/FallbackPass.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "scheduler/optimizer/FallbackPass.h"
+#include "scheduler/SchedInst.h"
+#include "scheduler/tasklabel/SpecResLabel.h"
+
+namespace milvus {
+namespace scheduler {
+
+void
+FallbackPass::Init() {
+}
+
+bool
+FallbackPass::Run(const TaskPtr& task) {
+    auto task_type = task->Type();
+    if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) {
+        return false;
+    }
+    // NEVER be empty
+    auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0];
+    auto label = std::make_shared<SpecResLabel>(cpu);
+    task->label() = label;
+    return true;
+}
+
+}  // namespace scheduler
+}  // namespace milvus

core/src/scheduler/optimizer/FallbackPass.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <bits/stdc++.h>
+#include <memory>
+
+#include "Pass.h"
+
+namespace milvus {
+namespace scheduler {
+
+class FallbackPass : public Pass {
+ public:
+    FallbackPass() = default;
+
+ public:
+    void
+    Init() override;
+
+    bool
+    Run(const TaskPtr& task) override;
+};
+
+}  // namespace scheduler
+}  // namespace milvus

core/src/scheduler/optimizer/HybridPass.cpp

@@ -23,6 +23,10 @@
 namespace milvus {
 namespace scheduler {
 
+void
+HybridPass::Init() {
+}
+
 bool
 HybridPass::Run(const TaskPtr& task) {
     // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu

core/src/scheduler/optimizer/HybridPass.h

@@ -37,6 +37,9 @@ class HybridPass : public Pass {
     HybridPass() = default;
 
  public:
+    void
+    Init() override;
+
     bool
     Run(const TaskPtr& task) override;
 };

core/src/scheduler/optimizer/LargeSQ8HPass.cpp

@@ -27,7 +27,8 @@
 namespace milvus {
 namespace scheduler {
 
-LargeSQ8HPass::LargeSQ8HPass() {
+void
+LargeSQ8HPass::Init() {
     server::Config& config = server::Config::GetInstance();
     Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
     if (!s.ok()) {

core/src/scheduler/optimizer/LargeSQ8HPass.h

@@ -35,9 +35,12 @@ namespace scheduler {
 
 class LargeSQ8HPass : public Pass {
  public:
-    LargeSQ8HPass();
+    LargeSQ8HPass() = default;
 
  public:
+    void
+    Init() override;
+
     bool
     Run(const TaskPtr& task) override;
 

core/src/scheduler/optimizer/OnlyCPUPass.cpp

@@ -24,6 +24,10 @@
 namespace milvus {
 namespace scheduler {
 
+void
+OnlyCPUPass::Init() {
+}
+
 bool
 OnlyCPUPass::Run(const TaskPtr& task) {
     if (task->Type() != TaskType::SearchTask)

core/src/scheduler/optimizer/OnlyCPUPass.h

@@ -37,6 +37,9 @@ class OnlyCPUPass : public Pass {
     OnlyCPUPass() = default;
 
  public:
+    void
+    Init() override;
+
     bool
     Run(const TaskPtr& task) override;
 };

core/src/scheduler/optimizer/OnlyGPUPass.cpp

@@ -27,6 +27,10 @@ namespace scheduler {
 OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) {
 }
 
+void
+OnlyGPUPass::Init() {
+}
+
 bool
 OnlyGPUPass::Run(const TaskPtr& task) {
     if (task->Type() != TaskType::SearchTask || has_cpu_)

core/src/scheduler/optimizer/OnlyGPUPass.h

@@ -37,6 +37,9 @@ class OnlyGPUPass : public Pass {
     explicit OnlyGPUPass(bool has_cpu);
 
  public:
+    void
+    Init() override;
+
     bool
     Run(const TaskPtr& task) override;
 

core/src/scheduler/optimizer/Optimizer.cpp

@@ -20,12 +20,12 @@
 namespace milvus {
 namespace scheduler {
 
-// void
-// Optimizer::Init() {
-//    for (auto& pass : pass_list_) {
-//        pass->Init();
-//    }
-// }
+void
+Optimizer::Init() {
+    for (auto& pass : pass_list_) {
+        pass->Init();
+    }
+}
 
 bool
 Optimizer::Run(const TaskPtr& task) {

core/src/scheduler/optimizer/Optimizer.h

@@ -38,8 +38,8 @@ class Optimizer {
     explicit Optimizer(std::vector<PassPtr> pass_list) : pass_list_(std::move(pass_list)) {
     }
 
-    //    void
-    //    Init();
+    void
+    Init();
 
     bool
     Run(const TaskPtr& task);

core/src/scheduler/optimizer/Pass.h

@@ -34,9 +34,8 @@ namespace scheduler {
 
 class Pass {
  public:
-    //    virtual void
-    //    Init() {
-    //    }
+    virtual void
+    Init() = 0;
 
     virtual bool
     Run(const TaskPtr& task) = 0;

docs/test_report/milvus_ivfsq8_test_report_detailed_version.md

@@ -28,13 +28,13 @@ Memory: 503GB
 
 Docker version: 18.09
 
-Nvidia Driver version: 430.34
+NVIDIA Driver version: 430.34
 
 Milvus version: 0.5.3
 
 SDK interface: Python 3.6.8
 
-Pymilvus version: 0.2.5
+pymilvus version: 0.2.5
 
 
 
@@ -66,7 +66,7 @@ For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ .
 
   > Note: In the query test of recall, we will test the following parameters with different values:
   >
-  > nq - grouped by: [1, 5, 10,  200, 400, 600, 800, 1000], 
+  > nq - grouped by: [10,  200, 400, 600, 800, 1000], 
   >
   > topk - grouped by: [1, 10, 100]
 
@@ -93,7 +93,7 @@ Milvus configuration
 - gpu_cache_capacity: 6
 - use_blas_threshold: 1100
 
-You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/.
+The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/.
 
 Test method
 

docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md

@@ -2,7 +2,7 @@
 
 ## 概述
 
-本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。
+本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。
 
 
 
@@ -28,13 +28,13 @@ GPU1: GeForce GTX 1080
 
 Docker版本: 18.09
 
-Nvidia Driver版本: 430.34
+NVIDIA Driver版本: 430.34
 
 Milvus版本: 0.5.3
 
 SDK接口: Python 3.6.8
 
-Pymilvus版本: 0.2.5
+pymilvus版本: 0.2.5
 
 
 
@@ -66,7 +66,7 @@ Pymilvus版本: 0.2.5
 
   > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果：
   >
-  > 被查询向量的数量nq将按照 [1, 5, 10,  200, 400, 600, 800, 1000]的数量分组，
+  > 被查询向量的数量nq将按照 [10,  200, 400, 600, 800, 1000]的数量分组，
   >
   > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。
 
@@ -93,7 +93,7 @@ Milvus设置：
 - gpu_cache_capacity: 6
 - use_blas_threshold: 1100
 
-你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。
+Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。
 
 测试方法