Merge branch '0.5.1' into '0.5.1'

0.5.1

See merge request megasearch/milvus!808

Former-commit-id: 04bacaed9d48e95253b25d858800b25751bafea9

CHANGELOG.md

@@ -6,12 +6,13 @@ Please mark all change in change log and use the ticket from JIRA.
 
 ## Bug
 - \#134 - JFrog cache error
+- \#161 - Search IVFSQHybrid crash on gpu
 
 ## Feature
 - \#90 - The server start error messages could be improved to enhance user experience
 - \#104 - test_scheduler core dump
 - \#115 - Using new structure for tasktable
-- \#139 - New config opion use_gpu_threshold
+- \#139 - New config option use_gpu_threshold
 - \#146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT
 
 ## Improvement

ci/jenkins/pod/milvus-build-env-pod.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   containers:
   - name: milvus-build-env
-    image: registry.zilliz.com/milvus/milvus-build-env:v0.5.0-ubuntu18.04
+    image: registry.zilliz.com/milvus/milvus-build-env:v0.5.1-ubuntu18.04
     env:
     - name: POD_IP
       valueFrom:

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -256,6 +256,7 @@ ExecutionEngineImpl::Load(bool to_cache) {
 
 Status
 ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
+#if 0
     if (hybrid) {
         const std::string key = location_ + ".quantizer";
         std::vector<uint64_t> gpus{device_id};
@@ -307,6 +308,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
         }
         return Status::OK();
     }
+#endif
 
     auto index = std::static_pointer_cast<VecIndex>(cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_));
     bool already_in_cache = (index != nullptr);

core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp

@@ -84,22 +84,14 @@ IVFSQHybrid::CopyGpuToCpu(const Config& config) {
 
 VectorIndexPtr
 IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
-    if (gpu_mode != 0) {
-        KNOWHERE_THROW_MSG("Not a GpuIndex Type");
-    }
-
     if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
         ResScope rs(res, device_id, false);
         faiss::gpu::GpuClonerOptions option;
         option.allInGpu = true;
 
-        faiss::IndexComposition index_composition;
-        index_composition.index = index_.get();
-        index_composition.quantizer = nullptr;
-        index_composition.mode = 0;  // copy all
-
-        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, &index_composition, &option);
-
+        auto idx = dynamic_cast<faiss::IndexIVF*>(index_.get());
+        idx->restore_quantizer();
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
         std::shared_ptr<faiss::Index> device_index = std::shared_ptr<faiss::Index>(gpu_index);
         auto new_idx = std::make_shared<IVFSQHybrid>(device_index, device_id, res);
         return new_idx;
@@ -119,9 +111,9 @@ IVFSQHybrid::LoadImpl(const BinarySet& index_binary) {
 void
 IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels,
                          const Config& cfg) {
-    //    std::lock_guard<std::mutex> lk(g_mutex);
-    //    static int64_t search_count;
-    //    ++search_count;
+    //        std::lock_guard<std::mutex> lk(g_mutex);
+    //        static int64_t search_count;
+    //        ++search_count;
 
     if (gpu_mode == 2) {
         GPUIVF::search_impl(n, data, k, distances, labels, cfg);

core/src/index/unittest/test_ivf.cpp

@@ -213,6 +213,10 @@ TEST_P(IVFTest, clone_test) {
         }
     }
 
+    if (index_type == "IVFSQHybrid") {
+        return;
+    }
+
     {
         // copy to gpu
         std::vector<std::string> support_idx_vec{"IVF", "GPUIVF", "IVFSQ", "GPUIVFSQ"};

docker/build_env/ubuntu16.04/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \
     apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
     sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
     apt-get update && apt-get install -y --no-install-recommends \
-    git flex bison gfortran \
+    git flex bison gfortran lsb-core \
     curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev \
     clang-format-6.0 clang-tidy-6.0 \
     lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 && \

docker/build_env/ubuntu18.04/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \
     apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
     sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
     apt-get update && apt-get install -y --no-install-recommends \
-    git flex bison gfortran \
+    git flex bison gfortran lsb-core \
     curl libtool automake libboost-all-dev libssl-dev pkg-config libcurl4-openssl-dev \
     clang-format-6.0 clang-tidy-6.0 \
     lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 && \