Merge branch '0.5.1' into 0.5.1-yzb

Former-commit-id: 1e1a78a79db59c10c3ede8fd3187aea41c582c59

CHANGELOG.md

@@ -5,11 +5,13 @@ Please mark all change in change log and use the ticket from JIRA.
 # Milvus 0.5.1 (TODO)
 
 ## Bug
+- \#134 - JFrog cache error
 
 ## Feature
 - \#90 - The server start error messages could be improved to enhance user experience
 - \#104 - test_scheduler core dump
 - \#115 - Using new structure for tasktable
+- \#139 - New config opion use_gpu_threshold
 
 ## Improvement
 - \#64 - Improvement dump function in scheduler

core/CMakeLists.txt

@@ -32,10 +32,14 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
 message(STATUS "Build time = ${BUILD_TIME}")
 
 MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
-    execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    if(GIT_BRANCH_NAME STREQUAL "")
+        execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    endif()
 ENDMACRO (GET_GIT_BRANCH_NAME)
 
 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
+message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
 if(NOT GIT_BRANCH_NAME STREQUAL "")
     string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
 endif()

core/conf/server_config.template

@@ -36,6 +36,7 @@ cache_config:
 engine_config:
   use_blas_threshold: 20            # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
                                     # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
+  use_gpu_threshold: 1000
 
 resource_config:
   search_resources:                 # define the GPUs used for search computation, must be in format: gpux

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -245,7 +245,8 @@ if(CUSTOMIZATION)
         # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
         # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
         # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
-         set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
+        # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
+         set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
     endif()
 else()
     set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz")

core/src/scheduler/optimizer/LargeSQ8HPass.cpp

@@ -21,11 +21,20 @@
 #include "scheduler/Utils.h"
 #include "scheduler/task/SearchTask.h"
 #include "scheduler/tasklabel/SpecResLabel.h"
+#include "server/Config.h"
 #include "utils/Log.h"
 
 namespace milvus {
 namespace scheduler {
 
+LargeSQ8HPass::LargeSQ8HPass() {
+    server::Config& config = server::Config::GetInstance();
+    Status s = config.GetEngineConfigUseGpuThreshold(threshold_);
+    if (!s.ok()) {
+        threshold_ = std::numeric_limits<int32_t>::max();
+    }
+}
+
 bool
 LargeSQ8HPass::Run(const TaskPtr& task) {
     if (task->Type() != TaskType::SearchTask) {
@@ -40,7 +49,8 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
     auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
 
     // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu
-    if (search_job->nq() < 100) {
+
+    if (search_job->nq() < threshold_) {
         return false;
     }
 

core/src/scheduler/optimizer/LargeSQ8HPass.h

@@ -18,6 +18,7 @@
 
 #include <condition_variable>
 #include <deque>
+#include <limits>
 #include <list>
 #include <memory>
 #include <mutex>
@@ -34,11 +35,14 @@ namespace scheduler {
 
 class LargeSQ8HPass : public Pass {
  public:
-    LargeSQ8HPass() = default;
+    LargeSQ8HPass();
 
  public:
     bool
     Run(const TaskPtr& task) override;
+
+ private:
+    int32_t threshold_ = std::numeric_limits<int32_t>::max();
 };
 
 using LargeSQ8HPassPtr = std::shared_ptr<LargeSQ8HPass>;

core/src/server/Config.cpp

@@ -193,6 +193,12 @@ Config::ValidateConfig() {
         return s;
     }
 
+    int32_t engine_use_gpu_threshold;
+    s = GetEngineConfigUseGpuThreshold(engine_use_gpu_threshold);
+    if (!s.ok()) {
+        return s;
+    }
+
     /* resource config */
     std::string resource_mode;
     s = GetResourceConfigMode(resource_mode);
@@ -324,6 +330,11 @@ Config::ResetDefaultConfig() {
         return s;
     }
 
+    s = SetEngineConfigUseGpuThreshold(CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT);
+    if (!s.ok()) {
+        return s;
+    }
+
     /* resource config */
     s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT);
     if (!s.ok()) {
@@ -656,6 +667,16 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
     return Status::OK();
 }
 
+Status
+Config::CheckEngineConfigUseGpuThreshold(const std::string& value) {
+    if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
+        std::string msg = "Invalid gpu threshold: " + value +
+                          ". Possible reason: engine_config.use_gpu_threshold is not a positive integer.";
+        return Status(SERVER_INVALID_ARGUMENT, msg);
+    }
+    return Status::OK();
+}
+
 Status
 Config::CheckResourceConfigMode(const std::string& value) {
     if (value != "simple") {
@@ -951,6 +972,19 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) {
     return Status::OK();
 }
 
+Status
+Config::GetEngineConfigUseGpuThreshold(int32_t& value) {
+    std::string str =
+        GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_GPU_THRESHOLD, CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT);
+    Status s = CheckEngineConfigUseGpuThreshold(str);
+    if (!s.ok()) {
+        return s;
+    }
+
+    value = std::stoi(str);
+    return Status::OK();
+}
+
 Status
 Config::GetResourceConfigMode(std::string& value) {
     value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT);
@@ -1203,6 +1237,17 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
     return Status::OK();
 }
 
+Status
+Config::SetEngineConfigUseGpuThreshold(const std::string& value) {
+    Status s = CheckEngineConfigUseGpuThreshold(value);
+    if (!s.ok()) {
+        return s;
+    }
+
+    SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_GPU_THRESHOLD, value);
+    return Status::OK();
+}
+
 /* resource config */
 Status
 Config::SetResourceConfigMode(const std::string& value) {

core/src/server/Config.h

@@ -84,6 +84,8 @@ static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold";
 static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20";
 static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num";
 static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
+static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD = "use_gpu_threshold";
+static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT = "1000";
 
 /* resource config */
 static const char* CONFIG_RESOURCE = "resource_config";
@@ -166,6 +168,8 @@ class Config {
     CheckEngineConfigUseBlasThreshold(const std::string& value);
     Status
     CheckEngineConfigOmpThreadNum(const std::string& value);
+    Status
+    CheckEngineConfigUseGpuThreshold(const std::string& value);
 
     /* resource config */
     Status
@@ -230,6 +234,8 @@ class Config {
     GetEngineConfigUseBlasThreshold(int32_t& value);
     Status
     GetEngineConfigOmpThreadNum(int32_t& value);
+    Status
+    GetEngineConfigUseGpuThreshold(int32_t& value);
 
     /* resource config */
     Status
@@ -289,6 +295,8 @@ class Config {
     SetEngineConfigUseBlasThreshold(const std::string& value);
     Status
     SetEngineConfigOmpThreadNum(const std::string& value);
+    Status
+    SetEngineConfigUseGpuThreshold(const std::string& value);
 
     /* resource config */
     Status