MS-496 Change the top_k limitation from 1024 to 2048

Former-commit-id: d63d6d69432888324424ed1daf0b1236a5243f4e

cpp/CHANGELOG.md

@@ -92,6 +92,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-487 - Define metric type in CreateTable
 - MS-488 - Improve code format in scheduler
 - MS-495 - cmake: integrated knowhere
+- MS-496 - Change the top_k limitation from 1024 to 2048
 
 ## New Feature
 - MS-343 - Implement ResourceMgr

cpp/src/utils/ValidationUtil.cpp

@@ -94,7 +94,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
 
 ServerError
 ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
-    if (top_k <= 0 || top_k > 1024) {
+    if (top_k <= 0) {
         return SERVER_INVALID_TOPK;
     }
 

cpp/src/wrapper/knowhere/vec_index.cpp

@@ -14,6 +14,8 @@
 #include "vec_impl.h"
 #include "wrapper_log.h"
 
+#include <cuda.h>
+
 
 namespace zilliz {
 namespace milvus {
@@ -241,19 +243,67 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co
 #define GPU_MAX_NRPOBE 1024
 #endif
 
+#define GPU_MAX_TOP_K GPU_MAX_NRPOBE
+// TODO(yzb): may be changed latter
+#define CPU_MAX_TOP_K GPU_MAX_TOP_K
+#define DEFAULT_MAX_TOP_K GPU_MAX_TOP_K
+
 void ParameterValidation(const IndexType &type, Config &cfg) {
     switch (type) {
+        case IndexType::FAISS_IVFFLAT_CPU:
+        case IndexType::FAISS_IVFPQ_CPU:
+        case IndexType::FAISS_IVFSQ8_CPU: {
+            //search on CPU
+            if (cfg.get_with_default("k", 0) != 0) {
+                auto k = cfg["k"].as<int>();
+                if (k > CPU_MAX_TOP_K) {
+                    WRAPPER_LOG_WARNING << "When search with CPU, top_k shoud be no more than " << CPU_MAX_TOP_K
+                                        << ", but you passed " << k
+                                        << ". Search with " << CPU_MAX_TOP_K << " instead";
+                    cfg.insert_or_assign("k", CPU_MAX_TOP_K);
+                }
+            }
+            break;
+        }
         case IndexType::FAISS_IVFSQ8_GPU:
         case IndexType::FAISS_IVFFLAT_GPU:
         case IndexType::FAISS_IVFPQ_GPU: {
+            //search on GPU
             if (cfg.get_with_default("nprobe", 0) != 0) {
                 auto nprobe = cfg["nprobe"].as<int>();
                 if (nprobe > GPU_MAX_NRPOBE) {
-                    WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE << ", but you passed " << nprobe
-                                      << ". Search with " << GPU_MAX_NRPOBE << " instead";
+                    WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE
+                                        << ", but you passed " << nprobe
+                                        << ". Search with " << GPU_MAX_NRPOBE << " instead";
                     cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE);
                 }
             }
+            if (cfg.get_with_default("k", 0) != 0) {
+                auto k = cfg["k"].as<int>();
+                if (k > GPU_MAX_TOP_K) {
+                    WRAPPER_LOG_WARNING << "When search with GPU, top_k shoud be no more than " << GPU_MAX_TOP_K
+                                        << ", but you passed " << k
+                                        << ". Search with " << GPU_MAX_TOP_K << " instead";
+                    cfg.insert_or_assign("k", GPU_MAX_TOP_K);
+                }
+            }
+            break;
+        }
+        case IndexType::FAISS_IDMAP:
+        case IndexType::FAISS_IVFFLAT_MIX:
+        case IndexType::SPTAG_KDT_RNT_CPU:
+        case IndexType::FAISS_IVFSQ8_MIX:
+        case IndexType::NSG_MIX: {
+            // TODO(yzb): need to figure out where it search
+            if (cfg.get_with_default("k", 0) != 0) {
+                auto k = cfg["k"].as<int>();
+                if (k > DEFAULT_MAX_TOP_K) {
+                    WRAPPER_LOG_WARNING << "top_k shoud be no more than " << DEFAULT_MAX_TOP_K << ", but you passed "
+                                        << k
+                                        << ". Search with " << DEFAULT_MAX_TOP_K << " instead";
+                    cfg.insert_or_assign("k", DEFAULT_MAX_TOP_K);
+                }
+            }
             break;
         }
         default:break;

cpp/src/wrapper/knowhere/vec_index.h

@@ -14,8 +14,6 @@
 #include "knowhere/common/config.h"
 #include "knowhere/common/binary_set.h"
 
-#include "cuda.h"
-
 
 namespace zilliz {
 namespace milvus {
@@ -62,7 +60,7 @@ class VecIndex {
                                          long *ids,
                                          const Config &cfg = Config()) = 0;
 
-    virtual VecIndexPtr CopyToGpu(const int64_t& device_id,
+    virtual VecIndexPtr CopyToGpu(const int64_t &device_id,
                                   const Config &cfg = Config()) = 0;
 
     virtual VecIndexPtr CopyToCpu(const Config &cfg = Config()) = 0;
@@ -86,16 +84,16 @@ extern server::KnowhereError write_index(VecIndexPtr index, const std::string &l
 
 extern VecIndexPtr read_index(const std::string &location);
 
-extern VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config& cfg = Config());
+extern VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg = Config());
 
 extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary);
 
-extern void AutoGenParams(const IndexType& type, const long& size, Config& cfg);
+extern void AutoGenParams(const IndexType &type, const long &size, Config &cfg);
 
-extern void ParameterValidation(const IndexType& type, Config& cfg);
+extern void ParameterValidation(const IndexType &type, Config &cfg);
 
-extern IndexType ConvertToCpuIndexType(const IndexType& type);
-extern IndexType ConvertToGpuIndexType(const IndexType& type);
+extern IndexType ConvertToCpuIndexType(const IndexType &type);
+extern IndexType ConvertToGpuIndexType(const IndexType &type);
 
 }
 }