Merge remote-tracking branch 'upstream/branch-0.5.0' into branch-0.5.0-yk

Former-commit-id: 42e7261dba12f219ba620dfc35f6d7bbf554298b

cpp/CHANGELOG.md

@@ -51,6 +51,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-600 - Reconstruct unittest code
 - MS-602 - Remove zilliz namespace
 - MS-610 - Change error code base value from hex to decimal
+- MS-635 - Add compile option to support customized faiss
 
 # Milvus 0.4.0 (2019-09-12)
 

cpp/CMakeLists.txt

@@ -125,6 +125,10 @@ set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
 set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
 
+if (CUSTOMIZATION)
+    add_definitions(-DCUSTOMIZATION)
+endif (CUSTOMIZATION)
+
 include(ExternalProject)
 include(DefineOptions)
 include(BuildUtils)

cpp/build.sh

@@ -9,9 +9,12 @@ DB_PATH="/opt/milvus"
 PROFILING="OFF"
 USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
+CUSTOMIZATION="ON"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-while getopts "p:d:t:ulrcgjh" arg
+wget -q --method HEAD
+
+while getopts "p:d:t:ulrcgjhx" arg
 do
         case $arg in
              p)
@@ -45,6 +48,9 @@ do
              j)
                 USE_JFROG_CACHE="ON"
                 ;;
+             x)
+                CUSTOMIZATION="OFF"
+                ;;
              h) # help
                 echo "
 
@@ -87,6 +93,7 @@ CMAKE_CMD="cmake \
 -DMILVUS_DB_PATH=${DB_PATH} \
 -DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
+-DCUSTOMIZATION=${CUSTOMIZATION} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

cpp/src/core/cmake/ThirdPartyPackagesCore.cmake

@@ -228,17 +228,24 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
     set(${_LIB_NAME} "${_LIB_VERSION}")
 endforeach()
 
-if(DEFINED ENV{KNOWHERE_FAISS_URL})
-    set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}")
-else()
+if(CUSTOMIZATION)
     set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
-    message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
+    # set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
+    # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
+    # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
+    # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
+    set(FAISS_MD5 "94988b7bdac4eb82a9575c702a3f2df3") # commit-id 1407526b31cad26f98ceca8dddaface8f18c4c19 branch-0.2.1
+
+    execute_process(COMMAND wget -q --method HEAD ${FAISS_SOURCE_URL} RESULT_VARIABLE return_code)
+    message(STATUS "Check the remote cache file ${FAISS_SOURCE_URL}. return code = ${return_code}")
+    if (NOT return_code EQUAL 0)
+        set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")
+    endif()
+else()
+    set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")
+    set(FAISS_MD5 "0bc12737b23def156f6a1eb782050135")
 endif()
-# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
-# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
-# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
-# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
-set(FAISS_MD5 "5af237d77947ee632f169bcb36feee2b") # commit-id 2c8affd0da60354e4322fa4c0224519e7912b9c4 branch-0.2.1
+message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
 
 if(DEFINED ENV{KNOWHERE_ARROW_URL})
     set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")

cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp

@@ -64,17 +64,13 @@ FaissBaseIndex::LoadImpl(const BinarySet& index_binary) {
 
 void
 FaissBaseIndex::SealImpl() {
-    // TODO(linxj): enable
-    //#ifdef ZILLIZ_FAISS
+#ifdef CUSTOMIZATION
     faiss::Index* index = index_.get();
     auto idx = dynamic_cast<faiss::IndexIVF*>(index);
     if (idx != nullptr) {
         idx->to_readonly();
     }
-    // else {
-    //    KNOHWERE_ERROR_MSG("Seal failed");
-    //}
-    //#endif
+#endif
 }
 
 }  // namespace knowhere

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp

@@ -55,10 +55,15 @@ VectorIndexPtr
 IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
     if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
         ResScope rs(res, device_id, false);
+
+#ifdef CUSTOMIZATION
         faiss::gpu::GpuClonerOptions option;
         option.allInGpu = true;
 
         auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
+#else
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
+#endif
 
         std::shared_ptr<faiss::Index> device_index;
         device_index.reset(gpu_index);

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp

@@ -25,6 +25,7 @@
 
 namespace knowhere {
 
+#ifdef CUSTOMIZATION
 IndexModelPtr
 IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) {
     auto build_cfg = std::dynamic_pointer_cast<IVFSQCfg>(config);
@@ -221,4 +222,50 @@ FaissIVFQuantizer::~FaissIVFQuantizer() {
     // else do nothing
 }
 
+#else
+
+QuantizerPtr
+IVFSQHybrid::LoadQuantizer(const Config& conf) {
+    return knowhere::QuantizerPtr();
+}
+
+void
+IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) {
+}
+
+void
+IVFSQHybrid::UnsetQuantizer() {
+}
+
+void
+IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
+}
+
+IndexModelPtr
+IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) {
+    return GPUIVFSQ::Train(dataset, config);
+}
+
+VectorIndexPtr
+IVFSQHybrid::CopyGpuToCpu(const Config& config) {
+    return GPUIVFSQ::CopyGpuToCpu(config);
+}
+
+VectorIndexPtr
+IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
+    return IVF::CopyCpuToGpu(device_id, config);
+}
+
+void
+IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels,
+                         const Config& cfg) {
+    GPUIVF::search_impl(n, data, k, distances, labels, cfg);
+}
+
+void
+IVFSQHybrid::LoadImpl(const BinarySet& index_binary) {
+    GPUIVF::LoadImpl(index_binary);
+}
+
+#endif
 }  // namespace knowhere

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h

@@ -25,12 +25,14 @@
 
 namespace knowhere {
 
+#ifdef CUSTOMIZATION
 struct FaissIVFQuantizer : public Quantizer {
     faiss::gpu::GpuIndexFlat* quantizer = nullptr;
 
     ~FaissIVFQuantizer() override;
 };
 using FaissIVFQuantizerPtr = std::shared_ptr<FaissIVFQuantizer>;
+#endif
 
 class IVFSQHybrid : public GPUIVFSQ {
  public:

cpp/src/core/unittest/test_ivf.cpp

@@ -70,7 +70,6 @@ enum class ParameterType {
     ivf,
     ivfpq,
     ivfsq,
-    ivfsqhybrid,
     nsg,
 };
 
@@ -104,7 +103,7 @@ class ParamGenerator {
             tempconf->nbits = 8;
             tempconf->metric_type = knowhere::METRICTYPE::L2;
             return tempconf;
-        } else if (type == ParameterType::ivfsq || type == ParameterType::ivfsqhybrid) {
+        } else if (type == ParameterType::ivfsq) {
             auto tempconf = std::make_shared<knowhere::IVFSQCfg>();
             tempconf->d = DIM;
             tempconf->gpu_id = device_id;
@@ -158,8 +157,10 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
                                //                            std::make_tuple("IVFPQ", ParameterType::ivfpq),
                                //                            std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
                                std::make_tuple("IVFSQ", ParameterType::ivfsq),
-                               std::make_tuple("GPUIVFSQ", ParameterType::ivfsq),
-                               std::make_tuple("IVFSQHybrid", ParameterType::ivfsqhybrid)));
+#ifdef CUSTOMIZATION
+                               std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
+#endif
+                               std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)));
 
 void
 AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
@@ -558,6 +559,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) {
     }
 }
 
+#ifdef CUSTOMIZATION
 TEST_F(GPURESTEST, gpuivfsq) {
     {
         // knowhere gpu ivfsq
@@ -629,6 +631,7 @@ TEST_F(GPURESTEST, gpuivfsq) {
         delete search_idx;
     }
 }
+#endif
 
 TEST_F(GPURESTEST, copyandsearch) {
     // search and copy at the same time

cpp/src/scheduler/TaskTable.cpp

@@ -17,8 +17,8 @@
 
 #include "scheduler/TaskTable.h"
 #include "Utils.h"
-#include "utils/Log.h"
 #include "event/TaskTableUpdatedEvent.h"
+#include "utils/Log.h"
 
 #include <ctime>
 #include <sstream>
@@ -165,7 +165,8 @@ TaskTable::PickToLoad(uint64_t limit) {
         }
         if (table_[j]->state == TaskTableItemState::LOADED) {
             ++count;
-            if (count > 2) return std::vector<uint64_t >();
+            if (count > 2)
+                return std::vector<uint64_t>();
         }
     }
 

cpp/src/wrapper/VecImpl.h

@@ -17,12 +17,12 @@
 
 #pragma once
 
-#include "VecIndex.h"
-#include "knowhere/index/vector_index/VectorIndex.h"
-
 #include <memory>
 #include <utility>
 
+#include "VecIndex.h"
+#include "knowhere/index/vector_index/VectorIndex.h"
+
 namespace milvus {
 namespace engine {
 

cpp/src/wrapper/VecIndex.cpp

@@ -143,10 +143,12 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
             index = std::make_shared<knowhere::GPUIVFSQ>(gpu_device);
             break;
         }
+#ifdef CUSTOMIZATION
         case IndexType::FAISS_IVFSQ8_HYBRID: {
             index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
             return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
         }
+#endif
         case IndexType::NSG_MIX: {  // TODO(linxj): bug.
             index = std::make_shared<knowhere::NSG>(gpu_device);
             break;
@@ -159,6 +161,9 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
 VecIndexPtr
 LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binary) {
     auto index = GetVecIndexFactory(index_type);
+    if (index == nullptr)
+        return nullptr;
+    // else
     index->Load(index_binary);
     return index;
 }