Merge remote-tracking branch 'main/0.5.1' into 0.5.1

Former-commit-id: 980bc7ea1b399e9bcd494195c41ae7e775c45b8c

CHANGELOG.md

@@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA.
 # Milvus 0.5.1 (TODO)
 
 ## Bug
+- \#134 - JFrog cache error
 
 ## Feature
 - \#90 - The server start error messages could be improved to enhance user experience
@@ -21,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#118 - Using shared_ptr instead of weak_ptr to avoid performance loss
 - \#122 - Add unique id for Job
 - \#130 - Set task state MOVED after resource copy it completed
+- \#149 - Improve large query optimizer pass
 
 ## Task
 

core/CMakeLists.txt

@@ -32,10 +32,14 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
 message(STATUS "Build time = ${BUILD_TIME}")
 
 MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
-    execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    if(GIT_BRANCH_NAME STREQUAL "")
+        execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    endif()
 ENDMACRO (GET_GIT_BRANCH_NAME)
 
 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
+message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
 if(NOT GIT_BRANCH_NAME STREQUAL "")
     string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
 endif()

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -258,7 +258,7 @@ Status
 ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
     if (hybrid) {
         const std::string key = location_ + ".quantizer";
-        std::vector<uint64_t> gpus = scheduler::get_gpu_pool();
+        std::vector<uint64_t> gpus{device_id};
 
         const int64_t NOT_FOUND = -1;
         int64_t device_id = NOT_FOUND;

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -245,7 +245,8 @@ if(CUSTOMIZATION)
         # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
         # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
         # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
-         set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
+        # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
+         set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
     endif()
 else()
     set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz")

core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp

@@ -17,31 +17,40 @@
 
 #include <gtest/gtest.h>
 
+#include <hdf5.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
 #include <cassert>
 #include <cmath>
 #include <cstdio>
+#include <vector>
+
+#define USE_FAISS_V1_5_3 0
+
+#if USE_FAISS_V1_5_3
+#include <faiss/gpu/GpuAutoTune.h>
+#include <faiss/utils.h>
+#include <sys/stat.h>
 #include <cstdlib>
 #include <cstring>
 
+#else
+#include <faiss/gpu/GpuCloner.h>
+#include <faiss/index_factory.h>
+#include <faiss/utils/distances.h>
+#endif
+
 #include <faiss/AutoTune.h>
 #include <faiss/Index.h>
 #include <faiss/IndexIVF.h>
-#include <faiss/gpu/GpuAutoTune.h>
 #include <faiss/gpu/GpuIndexFlat.h>
-#ifdef CUSTOMIZATION
-#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
-#endif
 #include <faiss/gpu/StandardGpuResources.h>
 #include <faiss/index_io.h>
-#include <faiss/utils.h>
 
-#include <hdf5.h>
-
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <vector>
+#ifdef CUSTOMIZATION
+#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
+#endif
 
 /*****************************************************
  * To run this test, please download the HDF5 from
@@ -56,6 +65,8 @@ const char HDF5_DATASET_TEST[] = "test";
 const char HDF5_DATASET_NEIGHBORS[] = "neighbors";
 const char HDF5_DATASET_DISTANCES[] = "distances";
 
+const int32_t GPU_DEVICE_IDX = 0;
+
 enum QueryMode { MODE_CPU = 0, MODE_MIX, MODE_GPU };
 
 double
@@ -196,7 +207,7 @@ GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::i
     for (int i = 0; i < nq; i++) {
         // count the num of results exist in ground truth result set
         // each result replicates INDEX_ADD_LOOPS times
-        for (int j_c = 0; j_c < ground_k; j_c++) {
+        for (int j_c = 0; j_c < k; j_c++) {
             int r_c = index[i * k + j_c];
             for (int j_g = 0; j_g < ground_k / index_add_loops; j_g++) {
                 if (ground_index[i * ground_k + j_g] == r_c) {
@@ -239,7 +250,6 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
     double t0 = elapsed();
 
     const std::string ann_file_name = ann_test_name + HDF5_POSTFIX;
-    const int GPU_DEVICE_IDX = 0;
 
     faiss::Index *cpu_index = nullptr, *gpu_index = nullptr;
     faiss::distance_compute_blas_threshold = 800;
@@ -249,30 +259,6 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
     try {
         printf("[%.3f s] Reading index file: %s\n", elapsed() - t0, index_file_name.c_str());
         cpu_index = faiss::read_index(index_file_name.c_str());
-
-        if (mode != MODE_CPU) {
-            faiss::gpu::GpuClonerOptions option;
-            option.allInGpu = true;
-
-            faiss::IndexComposition index_composition;
-            index_composition.index = cpu_index;
-            index_composition.quantizer = nullptr;
-
-            switch (mode) {
-                case MODE_CPU:
-                    assert(false);
-                    break;
-                case MODE_MIX:
-                    index_composition.mode = 1;  // 0: all data, 1: copy quantizer, 2: copy data
-                    break;
-                case MODE_GPU:
-                    index_composition.mode = 0;  // 0: all data, 1: copy quantizer, 2: copy data
-                    break;
-            }
-
-            printf("[%.3f s] Cloning CPU index to GPU\n", elapsed() - t0);
-            gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option);
-        }
     } catch (...) {
         size_t nb, d;
         printf("[%.3f s] Loading HDF5 file: %s\n", elapsed() - t0, ann_file_name.c_str());
@@ -289,6 +275,7 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
 
         printf("[%.3f s] Cloning CPU index to GPU\n", elapsed() - t0);
         gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index);
+        delete cpu_index;
 
         printf("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb);
         gpu_index->train(nb, xb);
@@ -300,8 +287,9 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
         }
 
         printf("[%.3f s] Coping GPU index to CPU\n", elapsed() - t0);
-        delete cpu_index;
+
         cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index);
+        delete gpu_index;
 
         faiss::IndexIVF* cpu_ivf_index = dynamic_cast<faiss::IndexIVF*>(cpu_index);
         if (cpu_ivf_index != nullptr) {
@@ -314,21 +302,7 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
         delete[] xb;
     }
 
-    switch (mode) {
-        case MODE_CPU:
-        case MODE_MIX:
     index = cpu_index;
-            if (gpu_index) {
-                delete gpu_index;
-            }
-            break;
-        case MODE_GPU:
-            index = gpu_index;
-            if (cpu_index) {
-                delete cpu_index;
-            }
-            break;
-    }
 }
 
 void
@@ -379,10 +353,12 @@ load_ground_truth(faiss::Index::idx_t*& gt, size_t& k, const std::string& ann_te
 }
 
 void
-test_with_nprobes(const std::string& ann_test_name, const std::string& index_key, faiss::Index* index,
+test_with_nprobes(const std::string& ann_test_name, const std::string& index_key, faiss::Index* cpu_index,
                   faiss::gpu::StandardGpuResources& res, const QueryMode query_mode, const faiss::Index::distance_t* xq,
                   const faiss::Index::idx_t* gt, const std::vector<size_t> nprobes, const int32_t index_add_loops,
                   const int32_t search_loops) {
+    double t0 = elapsed();
+
     const size_t NQ = 1000, NQ_START = 10, NQ_STEP = 10;
     const size_t K = 1000, K_START = 100, K_STEP = 10;
     const size_t GK = 100;  // topk of ground truth
@@ -390,6 +366,59 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
     std::unordered_map<size_t, std::string> mode_str_map = {
         {MODE_CPU, "MODE_CPU"}, {MODE_MIX, "MODE_MIX"}, {MODE_GPU, "MODE_GPU"}};
 
+    faiss::Index *gpu_index, *index;
+    if (query_mode != MODE_CPU) {
+        faiss::gpu::GpuClonerOptions option;
+        option.allInGpu = true;
+
+        faiss::IndexComposition index_composition;
+        index_composition.index = cpu_index;
+        index_composition.quantizer = nullptr;
+
+        double copy_time;
+        switch (query_mode) {
+            case MODE_MIX: {
+                index_composition.mode = 1;  // 0: all data, 1: copy quantizer, 2: copy data
+
+                // warm up the transmission
+                gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option);
+                delete gpu_index;
+
+                copy_time = elapsed();
+                gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option);
+                delete gpu_index;
+                copy_time = elapsed() - copy_time;
+                printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time);
+
+                auto ivf_index = dynamic_cast<faiss::IndexIVF*>(cpu_index);
+                auto is_gpu_flat_index = dynamic_cast<faiss::gpu::GpuIndexFlat*>(ivf_index->quantizer);
+                if (is_gpu_flat_index == nullptr) {
+                    delete ivf_index->quantizer;
+                    ivf_index->quantizer = index_composition.quantizer;
+                }
+                index = cpu_index;
+                break;
+            }
+            case MODE_GPU:
+                index_composition.mode = 0;  // 0: all data, 1: copy quantizer, 2: copy data
+
+                // warm up the transmission
+                gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option);
+                delete gpu_index;
+
+                copy_time = elapsed();
+                gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option);
+                copy_time = elapsed() - copy_time;
+                printf("[%.3f s] Copy data completed, cost %f s\n", elapsed() - t0, copy_time);
+
+                delete cpu_index;
+                index = gpu_index;
+                break;
+        }
+    } else {
+        index = cpu_index;
+    }
+
     for (auto nprobe : nprobes) {
         switch (query_mode) {
             case MODE_CPU:
@@ -445,6 +474,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
         delete[] I;
         delete[] D;
     }
+
+    delete index;
 }
 
 void
@@ -488,7 +519,6 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c
 
     delete[] xq;
     delete[] gt;
-    delete index;
 }
 
 /************************************************************************************
@@ -520,8 +550,8 @@ TEST(FAISSTEST, BENCHMARK) {
 
 #ifdef CUSTOMIZATION
     test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
+    test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
     test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
-//    test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
 #endif
 
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -535,7 +565,7 @@ TEST(FAISSTEST, BENCHMARK) {
 
 #ifdef CUSTOMIZATION
     test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
+    test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
     test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
-//    test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
 #endif
 }

core/src/scheduler/optimizer/LargeSQ8HPass.cpp

@@ -55,16 +55,18 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
     }
 
     std::vector<uint64_t> gpus = scheduler::get_gpu_pool();
-    std::vector<int64_t> all_free_mem;
-    for (auto& gpu : gpus) {
-        auto cache = cache::GpuCacheMgr::GetInstance(gpu);
-        auto free_mem = cache->CacheCapacity() - cache->CacheUsage();
-        all_free_mem.push_back(free_mem);
-    }
-
-    auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());
-    auto best_index = std::distance(all_free_mem.begin(), max_e);
-    auto best_device_id = gpus[best_index];
+    //    std::vector<int64_t> all_free_mem;
+    //    for (auto& gpu : gpus) {
+    //        auto cache = cache::GpuCacheMgr::GetInstance(gpu);
+    //        auto free_mem = cache->CacheCapacity() - cache->CacheUsage();
+    //        all_free_mem.push_back(free_mem);
+    //    }
+    //
+    //    auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());
+    //    auto best_index = std::distance(all_free_mem.begin(), max_e);
+    //    auto best_device_id = gpus[best_index];
+    auto best_device_id = count_ % gpus.size();
+    count_++;
 
     ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
     if (not res_ptr) {

core/src/scheduler/optimizer/LargeSQ8HPass.h

@@ -43,6 +43,7 @@ class LargeSQ8HPass : public Pass {
 
  private:
     int32_t threshold_ = std::numeric_limits<int32_t>::max();
+    int64_t count_ = 0;
 };
 
 using LargeSQ8HPassPtr = std::shared_ptr<LargeSQ8HPass>;