solve conflicts

Former-commit-id: c17de2992aa5837228c833e85b749cdc1f92edd9

cpp/CHANGELOG.md

@@ -92,6 +92,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-487 - Define metric type in CreateTable
 - MS-488 - Improve code format in scheduler
 - MS-495 - cmake: integrated knowhere
+- MS-496 - Change the top_k limitation from 1024 to 2048
 - MS-502 - Update tasktable_test in scheduler
 - MS-504 - Update node_test in scheduler
 - MS-505 - Install core unit test and add to coverage

cpp/src/scheduler/task/SearchTask.cpp

@@ -106,6 +106,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
             index_engine_->CopyToCpu();
         } else {
             // TODO: exception
+            std::string msg = "Wrong load type";
+            ENGINE_LOG_ERROR << msg;
         }
     } catch (std::exception &ex) {
         //typical error: out of disk space or permition denied
@@ -150,17 +152,17 @@ XSearchTask::Execute() {
     server::CollectDurationMetrics metrics(index_type_);
 
     std::vector<long> output_ids;
-    std::vector<float> output_distence;
+    std::vector<float> output_distance;
     for (auto &context : search_contexts_) {
         //step 1: allocate memory
         auto inner_k = context->topk();
         auto nprobe = context->nprobe();
         output_ids.resize(inner_k * context->nq());
-        output_distence.resize(inner_k * context->nq());
+        output_distance.resize(inner_k * context->nq());
 
         try {
             //step 2: search
-            index_engine_->Search(context->nq(), context->vectors(), inner_k, nprobe, output_distence.data(),
+            index_engine_->Search(context->nq(), context->vectors(), inner_k, nprobe, output_distance.data(),
                                   output_ids.data());
 
             double span = rc.RecordSection("do search for context:" + context->Identity());
@@ -170,12 +172,12 @@ XSearchTask::Execute() {
             //step 3: cluster result
             SearchContext::ResultSet result_set;
             auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk();
-            XSearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set);
+            XSearchTask::ClusterResult(output_ids, output_distance, context->nq(), spec_k, result_set);
 
             span = rc.RecordSection("cluster result for context:" + context->Identity());
             context->AccumReduceCost(span);
 
-            //step 4: pick up topk result
+            // step 4: pick up topk result
             XSearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult());
 
             span = rc.RecordSection("reduce topk for context:" + context->Identity());
@@ -197,13 +199,13 @@ XSearchTask::Execute() {
 }
 
 Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
-                                  const std::vector<float> &output_distence,
+                                  const std::vector<float> &output_distance,
                                   uint64_t nq,
                                   uint64_t topk,
                                   SearchContext::ResultSet &result_set) {
-    if (output_ids.size() < nq * topk || output_distence.size() < nq * topk) {
+    if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) {
         std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) +
-            " distance array size: " + std::to_string(output_distence.size());
+            " distance array size: " + std::to_string(output_distance.size());
         ENGINE_LOG_ERROR << msg;
         return Status(DB_ERROR, msg);
     }
@@ -220,7 +222,7 @@ Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
                 if (output_ids[index] < 0) {
                     continue;
                 }
-                id_distance.push_back(std::make_pair(output_ids[index], output_distence[index]));
+                id_distance.push_back(std::make_pair(output_ids[index], output_distance[index]));
             }
             result_set[i] = id_distance;
         }

cpp/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -652,7 +652,7 @@ SearchTask::OnExecute() {
                    search_param_->query_record_array(i).vector_data().data(),
                    table_info.dimension_ * sizeof(float));
         }
-        rc.ElapseFromBegin("prepare vector data");
+        rc.RecordSection("prepare vector data");
 
         //step 6: search vectors
         engine::QueryResults results;
@@ -666,7 +666,7 @@ SearchTask::OnExecute() {
                                           record_count, nprobe, vec_f.data(), dates, results);
         }
 
-        rc.ElapseFromBegin("search vectors from engine");
+        rc.RecordSection("search vectors from engine");
         if (!stat.ok()) {
             return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
         }
@@ -681,8 +681,6 @@ SearchTask::OnExecute() {
             return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg);
         }
 
-        rc.ElapseFromBegin("do search");
-
         //step 7: construct result array
         for (auto &result : results) {
             ::milvus::grpc::TopKQueryResult *topk_query_result = topk_result_list->add_topk_query_result();
@@ -698,7 +696,7 @@ SearchTask::OnExecute() {
 #endif
 
         //step 8: print time cost percent
-        double span_result = rc.RecordSection("construct result");
+        rc.RecordSection("construct result and send");
         rc.ElapseFromBegin("totally cost");
 
 
@@ -972,4 +970,4 @@ DropIndexTask::OnExecute() {
 }
 }
 }
-}
\ No newline at end of file
+}

cpp/src/utils/ValidationUtil.cpp

@@ -94,7 +94,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
 
 ErrorCode
 ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
-    if (top_k <= 0 || top_k > 1024) {
+    if (top_k <= 0 || top_k > 2048) {
         return SERVER_INVALID_TOPK;
     }
 

cpp/src/wrapper/knowhere/vec_impl.cpp

@@ -103,6 +103,7 @@ ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, lon
         // TODO(linxj): avoid copy here.
         memcpy(ids, p_ids, sizeof(int64_t) * nq * k);
         memcpy(dist, p_dist, sizeof(float) * nq * k);
+
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return KNOWHERE_UNEXPECTED_ERROR;

cpp/src/wrapper/knowhere/vec_index.cpp

@@ -14,6 +14,8 @@
 #include "vec_impl.h"
 #include "wrapper_log.h"
 
+#include <cuda.h>
+
 
 namespace zilliz {
 namespace milvus {
@@ -246,11 +248,13 @@ void ParameterValidation(const IndexType &type, Config &cfg) {
         case IndexType::FAISS_IVFSQ8_GPU:
         case IndexType::FAISS_IVFFLAT_GPU:
         case IndexType::FAISS_IVFPQ_GPU: {
+            //search on GPU
             if (cfg.get_with_default("nprobe", 0) != 0) {
                 auto nprobe = cfg["nprobe"].as<int>();
                 if (nprobe > GPU_MAX_NRPOBE) {
-                    WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE << ", but you passed " << nprobe
-                                      << ". Search with " << GPU_MAX_NRPOBE << " instead";
+                    WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE
+                                        << ", but you passed " << nprobe
+                                        << ". Search with " << GPU_MAX_NRPOBE << " instead";
                     cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE);
                 }
             }

cpp/src/wrapper/knowhere/vec_index.h

@@ -14,8 +14,6 @@
 #include "knowhere/common/config.h"
 #include "knowhere/common/binary_set.h"
 
-#include "cuda.h"
-
 
 namespace zilliz {
 namespace milvus {
@@ -62,7 +60,7 @@ class VecIndex {
                              long *ids,
                              const Config &cfg = Config()) = 0;
 
-    virtual VecIndexPtr CopyToGpu(const int64_t& device_id,
+    virtual VecIndexPtr CopyToGpu(const int64_t &device_id,
                                   const Config &cfg = Config()) = 0;
 
     virtual VecIndexPtr CopyToCpu(const Config &cfg = Config()) = 0;
@@ -86,16 +84,16 @@ extern ErrorCode write_index(VecIndexPtr index, const std::string &location);
 
 extern VecIndexPtr read_index(const std::string &location);
 
-extern VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config& cfg = Config());
+extern VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg = Config());
 
 extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary);
 
-extern void AutoGenParams(const IndexType& type, const long& size, Config& cfg);
+extern void AutoGenParams(const IndexType &type, const long &size, Config &cfg);
 
-extern void ParameterValidation(const IndexType& type, Config& cfg);
+extern void ParameterValidation(const IndexType &type, Config &cfg);
 
-extern IndexType ConvertToCpuIndexType(const IndexType& type);
-extern IndexType ConvertToGpuIndexType(const IndexType& type);
+extern IndexType ConvertToCpuIndexType(const IndexType &type);
+extern IndexType ConvertToGpuIndexType(const IndexType &type);
 
 }
 }