Http update endpoints (#2187)

* replace table with collection in http module
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

* add search_by_ids in http module
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

* add API
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

* add new API and unittest
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

* Update endpoint in http module(#2186); change vector id to string format(2185)
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

* allow filter when show partition
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

* move constant variables defination to .cpp
Signed-off-by: NYhz <yinghao.zou@zilliz.com>

CHANGELOG.md

@@ -39,6 +39,8 @@ Please mark all change in change log and use the issue from GitHub
 -   \#2167 Merge log_config.conf with server_config.yaml
 -   \#2173 Check storage permission
 -   \#2178 Using elkan K-Means to improve IVF
+-   \#2185 Change id to string format in http module
+-   \#2186 Update endpoints in http module
 -   \#2190 Fix memory usage is twice of index size when using GPU searching
 
 ## Task

core/src/server/web_impl/Constants.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "server/web_impl/Constants.h"
+
+namespace milvus {
+namespace server {
+namespace web {
+
+const char* NAME_ENGINE_TYPE_FLAT = "FLAT";
+const char* NAME_ENGINE_TYPE_IVFFLAT = "IVFFLAT";
+const char* NAME_ENGINE_TYPE_IVFSQ8 = "IVFSQ8";
+const char* NAME_ENGINE_TYPE_IVFSQ8H = "IVFSQ8H";
+const char* NAME_ENGINE_TYPE_RNSG = "RNSG";
+const char* NAME_ENGINE_TYPE_IVFPQ = "IVFPQ";
+const char* NAME_ENGINE_TYPE_HNSW = "HNSW";
+const char* NAME_ENGINE_TYPE_ANNOY = "ANNOY";
+
+const char* NAME_METRIC_TYPE_L2 = "L2";
+const char* NAME_METRIC_TYPE_IP = "IP";
+const char* NAME_METRIC_TYPE_HAMMING = "HAMMING";
+const char* NAME_METRIC_TYPE_JACCARD = "JACCARD";
+const char* NAME_METRIC_TYPE_TANIMOTO = "TANIMOTO";
+const char* NAME_METRIC_TYPE_SUBSTRUCTURE = "SUBSTRUCTURE";
+const char* NAME_METRIC_TYPE_SUPERSTRUCTURE = "SUPERSTRUCTURE";
+
+////////////////////////////////////////////////////
+const int64_t VALUE_COLLECTION_INDEX_FILE_SIZE_DEFAULT = 1024;
+const char* VALUE_COLLECTION_METRIC_TYPE_DEFAULT = "L2";
+
+const char* VALUE_PARTITION_TAG_DEFAULT = "";
+
+const char* VALUE_INDEX_INDEX_TYPE_DEFAULT = NAME_ENGINE_TYPE_FLAT;
+const int64_t VALUE_INDEX_NLIST_DEFAULT = 16384;
+
+const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT = 4;
+const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT = false;
+
+/////////////////////////////////////////////////////
+const std::unordered_map<engine::EngineType, std::string> IndexMap = {
+    {engine::EngineType::FAISS_IDMAP, NAME_ENGINE_TYPE_FLAT},
+    {engine::EngineType::FAISS_IVFFLAT, NAME_ENGINE_TYPE_IVFFLAT},
+    {engine::EngineType::FAISS_IVFSQ8, NAME_ENGINE_TYPE_IVFSQ8},
+    {engine::EngineType::FAISS_IVFSQ8H, NAME_ENGINE_TYPE_IVFSQ8H},
+    {engine::EngineType::NSG_MIX, NAME_ENGINE_TYPE_RNSG},
+    {engine::EngineType::FAISS_PQ, NAME_ENGINE_TYPE_IVFPQ},
+    {engine::EngineType::HNSW, NAME_ENGINE_TYPE_HNSW},
+    {engine::EngineType::ANNOY, NAME_ENGINE_TYPE_ANNOY},
+};
+
+const std::unordered_map<std::string, engine::EngineType> IndexNameMap = {
+    {NAME_ENGINE_TYPE_FLAT, engine::EngineType::FAISS_IDMAP},
+    {NAME_ENGINE_TYPE_IVFFLAT, engine::EngineType::FAISS_IVFFLAT},
+    {NAME_ENGINE_TYPE_IVFSQ8, engine::EngineType::FAISS_IVFSQ8},
+    {NAME_ENGINE_TYPE_IVFSQ8H, engine::EngineType::FAISS_IVFSQ8H},
+    {NAME_ENGINE_TYPE_RNSG, engine::EngineType::NSG_MIX},
+    {NAME_ENGINE_TYPE_IVFPQ, engine::EngineType::FAISS_PQ},
+    {NAME_ENGINE_TYPE_HNSW, engine::EngineType::HNSW},
+    {NAME_ENGINE_TYPE_ANNOY, engine::EngineType::ANNOY},
+};
+
+const std::unordered_map<engine::MetricType, std::string> MetricMap = {
+    {engine::MetricType::L2, NAME_METRIC_TYPE_L2},
+    {engine::MetricType::IP, NAME_METRIC_TYPE_IP},
+    {engine::MetricType::HAMMING, NAME_METRIC_TYPE_HAMMING},
+    {engine::MetricType::JACCARD, NAME_METRIC_TYPE_JACCARD},
+    {engine::MetricType::TANIMOTO, NAME_METRIC_TYPE_TANIMOTO},
+    {engine::MetricType::SUBSTRUCTURE, NAME_METRIC_TYPE_SUBSTRUCTURE},
+    {engine::MetricType::SUPERSTRUCTURE, NAME_METRIC_TYPE_SUPERSTRUCTURE},
+};
+
+const std::unordered_map<std::string, engine::MetricType> MetricNameMap = {
+    {NAME_METRIC_TYPE_L2, engine::MetricType::L2},
+    {NAME_METRIC_TYPE_IP, engine::MetricType::IP},
+    {NAME_METRIC_TYPE_HAMMING, engine::MetricType::HAMMING},
+    {NAME_METRIC_TYPE_JACCARD, engine::MetricType::JACCARD},
+    {NAME_METRIC_TYPE_TANIMOTO, engine::MetricType::TANIMOTO},
+    {NAME_METRIC_TYPE_SUBSTRUCTURE, engine::MetricType::SUBSTRUCTURE},
+    {NAME_METRIC_TYPE_SUPERSTRUCTURE, engine::MetricType::SUPERSTRUCTURE},
+};
+}  // namespace web
+}  // namespace server
+}  // namespace milvus

core/src/server/web_impl/Constants.h

@@ -11,71 +11,46 @@
 
 #pragma once
 
+#include "db/engine/ExecutionEngine.h"
+
+#include <string>
+#include <unordered_map>
+
 namespace milvus {
 namespace server {
 namespace web {
 
-////////////////////////////////////////////////////
-static const char* WEB_LOG_PREFIX = "[Web] ";
-
-////////////////////////////////////////////////////
-
-static const char* CORS_KEY_METHODS = "Access-Control-Allow-Methods";
-static const char* CORS_KEY_ORIGIN = "Access-Control-Allow-Origin";
-static const char* CORS_KEY_HEADERS = "Access-Control-Allow-Headers";
-static const char* CORS_KEY_AGE = "Access-Control-Max-Age";
-
-static const char* CORS_VALUE_METHODS = "GET, POST, PUT, OPTIONS, DELETE";
-static const char* CORS_VALUE_ORIGIN = "*";
-static const char* CORS_VALUE_HEADERS =
-    "DNT, User-Agent, X-Requested-With, If-Modified-Since, Cache-Control, Content-Type, Range, Authorization";
-static const char* CORS_VALUE_AGE = "1728000";
-
-////////////////////////////////////////////////////
-
-static const char* NAME_ENGINE_TYPE_FLAT = "FLAT";
-static const char* NAME_ENGINE_TYPE_IVFFLAT = "IVFFLAT";
-static const char* NAME_ENGINE_TYPE_IVFSQ8 = "IVFSQ8";
-static const char* NAME_ENGINE_TYPE_IVFSQ8H = "IVFSQ8H";
-static const char* NAME_ENGINE_TYPE_RNSG = "RNSG";
-static const char* NAME_ENGINE_TYPE_IVFPQ = "IVFPQ";
-static const char* NAME_ENGINE_TYPE_HNSW = "HNSW";
-static const char* NAME_ENGINE_TYPE_ANNOY = "ANNOY";
-
-static const char* NAME_METRIC_TYPE_L2 = "L2";
-static const char* NAME_METRIC_TYPE_IP = "IP";
-static const char* NAME_METRIC_TYPE_HAMMING = "HAMMING";
-static const char* NAME_METRIC_TYPE_JACCARD = "JACCARD";
-static const char* NAME_METRIC_TYPE_TANIMOTO = "TANIMOTO";
-static const char* NAME_METRIC_TYPE_SUBSTRUCTURE = "SUBSTRUCTURE";
-static const char* NAME_METRIC_TYPE_SUPERSTRUCTURE = "SUPERSTRUCTURE";
-
-////////////////////////////////////////////////////
-
-static const char* KEY_TABLE_TABLE_NAME = "table_name";
-static const char* KEY_TABLE_DIMENSION = "dimension";
-static const char* KEY_TABLE_INDEX_FILE_SIZE = "index_file_size";
-static const char* KEY_TABLE_INDEX_METRIC_TYPE = "metric_type";
-static const char* KEY_TABLE_COUNT = "count";
-
-static const char* KEY_INDEX_INDEX_TYPE = "index_type";
-static const char* KEY_INDEX_NLIST = "nlist";
-
-static const char* KEY_PARTITION_NAME = "partition_name";
-static const char* KEY_PARTITION_TAG = "partition_tag";
+extern const char* NAME_ENGINE_TYPE_FLAT;
+extern const char* NAME_ENGINE_TYPE_IVFFLAT;
+extern const char* NAME_ENGINE_TYPE_IVFSQ8;
+extern const char* NAME_ENGINE_TYPE_IVFSQ8H;
+extern const char* NAME_ENGINE_TYPE_RNSG;
+extern const char* NAME_ENGINE_TYPE_IVFPQ;
+extern const char* NAME_ENGINE_TYPE_HNSW;
+extern const char* NAME_ENGINE_TYPE_ANNOY;
+
+extern const char* NAME_METRIC_TYPE_L2;
+extern const char* NAME_METRIC_TYPE_IP;
+extern const char* NAME_METRIC_TYPE_HAMMING;
+extern const char* NAME_METRIC_TYPE_JACCARD;
+extern const char* NAME_METRIC_TYPE_TANIMOTO;
+extern const char* NAME_METRIC_TYPE_SUBSTRUCTURE;
+extern const char* NAME_METRIC_TYPE_SUPERSTRUCTURE;
 
 ////////////////////////////////////////////////////
-
-static const int64_t VALUE_TABLE_INDEX_FILE_SIZE_DEFAULT = 1024;
-static const char* VALUE_TABLE_METRIC_TYPE_DEFAULT = "L2";
-
-static const char* VALUE_PARTITION_TAG_DEFAULT = "";
-
-static const char* VALUE_INDEX_INDEX_TYPE_DEFAULT = NAME_ENGINE_TYPE_FLAT;
-static const int64_t VALUE_INDEX_NLIST_DEFAULT = 16384;
-
-static const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT = 4;
-static const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT = false;
+extern const int64_t VALUE_COLLECTION_INDEX_FILE_SIZE_DEFAULT;
+extern const char* VALUE_COLLECTION_METRIC_TYPE_DEFAULT;
+extern const char* VALUE_PARTITION_TAG_DEFAULT;
+extern const char* VALUE_INDEX_INDEX_TYPE_DEFAULT;
+extern const int64_t VALUE_INDEX_NLIST_DEFAULT;
+extern const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT;
+extern const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT;
+
+/////////////////////////////////////////////////////
+extern const std::unordered_map<engine::EngineType, std::string> IndexMap;
+extern const std::unordered_map<std::string, engine::EngineType> IndexNameMap;
+extern const std::unordered_map<engine::MetricType, std::string> MetricMap;
+extern const std::unordered_map<std::string, engine::MetricType> MetricNameMap;
 
 }  // namespace web
 }  // namespace server

core/src/server/web_impl/Types.h

@@ -17,7 +17,6 @@
 #include <oatpp/core/data/mapping/type/Object.hpp>
 #include <oatpp/web/protocol/http/Http.hpp>
 
-#include "db/engine/ExecutionEngine.h"
 #include "server/web_impl/Constants.h"
 
 namespace milvus {
@@ -73,48 +72,6 @@ enum StatusCode : int {
     MAX = ILLEGAL_QUERY_PARAM
 };
 
-static const std::unordered_map<engine::EngineType, std::string> IndexMap = {
-    {engine::EngineType::FAISS_IDMAP, NAME_ENGINE_TYPE_FLAT},
-    {engine::EngineType::FAISS_IVFFLAT, NAME_ENGINE_TYPE_IVFFLAT},
-    {engine::EngineType::FAISS_IVFSQ8, NAME_ENGINE_TYPE_IVFSQ8},
-    {engine::EngineType::FAISS_IVFSQ8H, NAME_ENGINE_TYPE_IVFSQ8H},
-    {engine::EngineType::NSG_MIX, NAME_ENGINE_TYPE_RNSG},
-    {engine::EngineType::FAISS_PQ, NAME_ENGINE_TYPE_IVFPQ},
-    {engine::EngineType::HNSW, NAME_ENGINE_TYPE_HNSW},
-    {engine::EngineType::ANNOY, NAME_ENGINE_TYPE_ANNOY},
-};
-
-static const std::unordered_map<std::string, engine::EngineType> IndexNameMap = {
-    {NAME_ENGINE_TYPE_FLAT, engine::EngineType::FAISS_IDMAP},
-    {NAME_ENGINE_TYPE_IVFFLAT, engine::EngineType::FAISS_IVFFLAT},
-    {NAME_ENGINE_TYPE_IVFSQ8, engine::EngineType::FAISS_IVFSQ8},
-    {NAME_ENGINE_TYPE_IVFSQ8H, engine::EngineType::FAISS_IVFSQ8H},
-    {NAME_ENGINE_TYPE_RNSG, engine::EngineType::NSG_MIX},
-    {NAME_ENGINE_TYPE_IVFPQ, engine::EngineType::FAISS_PQ},
-    {NAME_ENGINE_TYPE_HNSW, engine::EngineType::HNSW},
-    {NAME_ENGINE_TYPE_ANNOY, engine::EngineType::ANNOY},
-};
-
-static const std::unordered_map<engine::MetricType, std::string> MetricMap = {
-    {engine::MetricType::L2, NAME_METRIC_TYPE_L2},
-    {engine::MetricType::IP, NAME_METRIC_TYPE_IP},
-    {engine::MetricType::HAMMING, NAME_METRIC_TYPE_HAMMING},
-    {engine::MetricType::JACCARD, NAME_METRIC_TYPE_JACCARD},
-    {engine::MetricType::TANIMOTO, NAME_METRIC_TYPE_TANIMOTO},
-    {engine::MetricType::SUBSTRUCTURE, NAME_METRIC_TYPE_SUBSTRUCTURE},
-    {engine::MetricType::SUPERSTRUCTURE, NAME_METRIC_TYPE_SUPERSTRUCTURE},
-};
-
-static const std::unordered_map<std::string, engine::MetricType> MetricNameMap = {
-    {NAME_METRIC_TYPE_L2, engine::MetricType::L2},
-    {NAME_METRIC_TYPE_IP, engine::MetricType::IP},
-    {NAME_METRIC_TYPE_HAMMING, engine::MetricType::HAMMING},
-    {NAME_METRIC_TYPE_JACCARD, engine::MetricType::JACCARD},
-    {NAME_METRIC_TYPE_TANIMOTO, engine::MetricType::TANIMOTO},
-    {NAME_METRIC_TYPE_SUBSTRUCTURE, engine::MetricType::SUBSTRUCTURE},
-    {NAME_METRIC_TYPE_SUPERSTRUCTURE, engine::MetricType::SUPERSTRUCTURE},
-};
-
 }  // namespace web
 }  // namespace server
 }  // namespace milvus

core/src/server/web_impl/controller/WebController.hpp

@@ -19,16 +19,17 @@
 #include <oatpp/parser/json/mapping/ObjectMapper.hpp>
 #include <oatpp/web/server/api/ApiController.hpp>
 
-#include "utils/Log.h"
-#include "utils/TimeRecorder.h"
-
 #include "server/web_impl/Constants.h"
+#include "server/web_impl/dto/CollectionDto.hpp"
 #include "server/web_impl/dto/ConfigDto.hpp"
 #include "server/web_impl/dto/IndexDto.hpp"
 #include "server/web_impl/dto/PartitionDto.hpp"
-#include "server/web_impl/dto/TableDto.hpp"
 #include "server/web_impl/dto/VectorDto.hpp"
 #include "server/web_impl/handler/WebRequestHandler.h"
+#include "utils/Log.h"
+#include "utils/TimeRecorder.h"
+
+#define WEB_LOG_PREFIX "[Web] "
 
 namespace milvus {
 namespace server {
@@ -206,22 +207,22 @@ class WebController : public oatpp::web::server::api::ApiController {
 
 #endif
 
-    ADD_CORS(TablesOptions)
+    ADD_CORS(CollectionsOptions)
 
-    ENDPOINT("OPTIONS", "/collections", TablesOptions) {
+    ENDPOINT("OPTIONS", "/collections", CollectionsOptions) {
         return createResponse(Status::CODE_204, "No Content");
     }
 
-    ADD_CORS(CreateTable)
+    ADD_CORS(CreateCollection)
 
-    ENDPOINT("POST", "/collections", CreateTable, BODY_DTO(TableRequestDto::ObjectWrapper, body)) {
+    ENDPOINT("POST", "/collections", CreateCollection, BODY_DTO(CollectionRequestDto::ObjectWrapper, body)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections\'");
         tr.RecordSection("Received request.");
 
         WebRequestHandler handler = WebRequestHandler();
 
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.CreateTable(body);
+        auto status_dto = handler.CreateCollection(body);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
                 response = createDtoResponse(Status::CODE_201, status_dto);
@@ -236,16 +237,16 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
-    ADD_CORS(ShowTables)
+    ADD_CORS(ShowCollections)
 
-    ENDPOINT("GET", "/collections", ShowTables, QUERIES(const QueryParams&, query_params)) {
+    ENDPOINT("GET", "/collections", ShowCollections, QUERIES(const QueryParams&, query_params)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections\'");
         tr.RecordSection("Received request.");
 
         WebRequestHandler handler = WebRequestHandler();
 
         String result;
-        auto status_dto = handler.ShowTables(query_params, result);
+        auto status_dto = handler.ShowCollections(query_params, result);
         std::shared_ptr<OutgoingResponse> response;
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
@@ -262,15 +263,15 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
-    ADD_CORS(TableOptions)
+    ADD_CORS(CollectionOptions)
 
-    ENDPOINT("OPTIONS", "/collections/{collection_name}", TableOptions) {
+    ENDPOINT("OPTIONS", "/collections/{collection_name}", CollectionOptions) {
         return createResponse(Status::CODE_204, "No Content");
     }
 
-    ADD_CORS(GetTable)
+    ADD_CORS(GetCollection)
 
-    ENDPOINT("GET", "/collections/{collection_name}", GetTable, PATH(String, collection_name),
+    ENDPOINT("GET", "/collections/{collection_name}", GetCollection, PATH(String, collection_name),
              QUERIES(const QueryParams&, query_params)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "\'");
         tr.RecordSection("Received request.");
@@ -278,7 +279,7 @@ class WebController : public oatpp::web::server::api::ApiController {
         WebRequestHandler handler = WebRequestHandler();
 
         String response_str;
-        auto status_dto = handler.GetTable(collection_name, query_params, response_str);
+        auto status_dto = handler.GetCollection(collection_name, query_params, response_str);
 
         std::shared_ptr<OutgoingResponse> response;
         switch (status_dto->code->getValue()) {
@@ -299,16 +300,16 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
-    ADD_CORS(DropTable)
+    ADD_CORS(DropCollection)
 
-    ENDPOINT("DELETE", "/collections/{collection_name}", DropTable, PATH(String, collection_name)) {
+    ENDPOINT("DELETE", "/collections/{collection_name}", DropCollection, PATH(String, collection_name)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() + "\'");
         tr.RecordSection("Received request.");
 
         WebRequestHandler handler = WebRequestHandler();
 
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.DropTable(collection_name);
+        auto status_dto = handler.DropCollection(collection_name);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
                 response = createDtoResponse(Status::CODE_204, status_dto);
@@ -460,7 +461,7 @@ class WebController : public oatpp::web::server::api::ApiController {
     ADD_CORS(ShowPartitions)
 
     ENDPOINT("GET", "/collections/{collection_name}/partitions", ShowPartitions, PATH(String, collection_name),
-             QUERIES(const QueryParams&, query_params)) {
+             QUERIES(const QueryParams&, query_params), BODY_STRING(String, body)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() +
                         "/partitions\'");
         tr.RecordSection("Received request.");
@@ -472,7 +473,7 @@ class WebController : public oatpp::web::server::api::ApiController {
         auto handler = WebRequestHandler();
 
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.ShowPartitions(collection_name, query_params, partition_list_dto);
+        auto status_dto = handler.ShowPartitions(collection_name, query_params, body, partition_list_dto);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
                 response = createDtoResponse(Status::CODE_200, partition_list_dto);
@@ -577,10 +578,10 @@ class WebController : public oatpp::web::server::api::ApiController {
      * GetVectorByID ?id=
      */
     ENDPOINT("GET", "/collections/{collection_name}/vectors", GetVectors, PATH(String, collection_name),
-             QUERIES(const QueryParams&, query_params)) {
+             BODY_STRING(String, body), QUERIES(const QueryParams&, query_params)) {
         auto handler = WebRequestHandler();
         String response;
-        auto status_dto = handler.GetVector(collection_name, query_params, response);
+        auto status_dto = handler.GetVector(collection_name, body, query_params, response);
 
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:

core/src/server/web_impl/dto/TableDto.hpp → core/src/server/web_impl/dto/CollectionDto.hpp

@@ -21,18 +21,17 @@ namespace web {
 
 #include OATPP_CODEGEN_BEGIN(DTO)
 
-
-class TableRequestDto : public oatpp::data::mapping::type::Object {
- DTO_INIT(TableRequestDto, Object)
+class CollectionRequestDto : public oatpp::data::mapping::type::Object {
+ DTO_INIT(CollectionRequestDto, Object)
 
     DTO_FIELD(String, collection_name, "collection_name");
     DTO_FIELD(Int64, dimension, "dimension");
-    DTO_FIELD(Int64, index_file_size, "index_file_size") = VALUE_TABLE_INDEX_FILE_SIZE_DEFAULT;
-    DTO_FIELD(String, metric_type, "metric_type") = VALUE_TABLE_METRIC_TYPE_DEFAULT;
+    DTO_FIELD(Int64, index_file_size, "index_file_size") = VALUE_COLLECTION_INDEX_FILE_SIZE_DEFAULT;
+    DTO_FIELD(String, metric_type, "metric_type") = VALUE_COLLECTION_METRIC_TYPE_DEFAULT;
 };
 
-class TableFieldsDto : public oatpp::data::mapping::type::Object {
- DTO_INIT(TableFieldsDto, Object)
+class CollectionFieldsDto : public oatpp::data::mapping::type::Object {
+ DTO_INIT(CollectionFieldsDto, Object)
 
     DTO_FIELD(String, collection_name);
     DTO_FIELD(Int64, dimension);
@@ -43,16 +42,16 @@ class TableFieldsDto : public oatpp::data::mapping::type::Object {
     DTO_FIELD(String, index_params);
 };
 
-class TableListDto : public OObject {
- DTO_INIT(TableListDto, Object)
+class CollectionListDto : public OObject {
+ DTO_INIT(CollectionListDto, Object)
 
     DTO_FIELD(List<String>::ObjectWrapper, collection_names);
 };
 
-class TableListFieldsDto : public OObject {
-    DTO_INIT(TableListFieldsDto, Object)
+class CollectionListFieldsDto : public OObject {
+    DTO_INIT(CollectionListFieldsDto, Object)
 
-    DTO_FIELD(List<TableFieldsDto::ObjectWrapper>::ObjectWrapper, collections);
+    DTO_FIELD(List<CollectionFieldsDto::ObjectWrapper>::ObjectWrapper, collections);
     DTO_FIELD(Int64, count) = 0L;
 };
 

core/src/server/web_impl/handler/WebRequestHandler.cpp

@@ -85,7 +85,7 @@ WebRequestHandler::AddStatusToJson(nlohmann::json& json, int64_t code, const std
 }
 
 Status
-WebRequestHandler::IsBinaryTable(const std::string& collection_name, bool& bin) {
+WebRequestHandler::IsBinaryCollection(const std::string& collection_name, bool& bin) {
     CollectionSchema schema;
     auto status = request_handler_.DescribeCollection(context_ptr_, collection_name, schema);
     if (status.ok()) {
@@ -131,7 +131,7 @@ WebRequestHandler::CopyRecordsFromJson(const nlohmann::json& json, engine::Vecto
 
 ///////////////////////// WebRequestHandler methods ///////////////////////////////////////
 Status
-WebRequestHandler::GetTableMetaInfo(const std::string& collection_name, nlohmann::json& json_out) {
+WebRequestHandler::GetCollectionMetaInfo(const std::string& collection_name, nlohmann::json& json_out) {
     CollectionSchema schema;
     auto status = request_handler_.DescribeCollection(context_ptr_, collection_name, schema);
     if (!status.ok()) {
@@ -162,7 +162,7 @@ WebRequestHandler::GetTableMetaInfo(const std::string& collection_name, nlohmann
 }
 
 Status
-WebRequestHandler::GetTableStat(const std::string& collection_name, nlohmann::json& json_out) {
+WebRequestHandler::GetCollectionStat(const std::string& collection_name, nlohmann::json& json_out) {
     std::string collection_info;
     auto status = request_handler_.ShowCollectionInfo(context_ptr_, collection_name, collection_info);
 
@@ -170,6 +170,8 @@ WebRequestHandler::GetTableStat(const std::string& collection_name, nlohmann::js
         try {
             json_out = nlohmann::json::parse(collection_info);
         } catch (std::exception& e) {
+            return Status(SERVER_UNEXPECTED_ERROR,
+                          "Error occurred when parsing collection stat information: " + std::string(e.what()));
         }
     }
 
@@ -248,7 +250,7 @@ WebRequestHandler::Cmd(const std::string& cmd, std::string& result_str) {
 }
 
 Status
-WebRequestHandler::PreLoadTable(const nlohmann::json& json, std::string& result_str) {
+WebRequestHandler::PreLoadCollection(const nlohmann::json& json, std::string& result_str) {
     if (!json.contains("collection_name")) {
         return Status(BODY_FIELD_LOSS, "Field \"load\" must contains collection_name");
     }
@@ -408,6 +410,10 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
     }
     int64_t topk = json["topk"];
 
+    if (!json.contains("params")) {
+        return Status(BODY_FIELD_LOSS, "Field \'params\' is required");
+    }
+
     std::vector<std::string> partition_tags;
     if (json.contains("partition_tags")) {
         auto tags = json["partition_tags"];
@@ -420,6 +426,22 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
         }
     }
 
+    TopKQueryResult result;
+    Status status;
+    if (json.contains("ids")) {
+        auto vec_ids = json["ids"];
+        if (!vec_ids.is_array()) {
+            return Status(BODY_PARSE_FAIL, "Field \"ids\" must be ad array");
+        }
+
+        std::vector<int64_t> id_array;
+        for (auto& id_str : vec_ids) {
+            id_array.emplace_back(std::stol(id_str.get<std::string>()));
+        }
+        //        std::vector<int64_t> id_array(vec_ids.begin(), vec_ids.end());
+        status = request_handler_.SearchByID(context_ptr_, collection_name, id_array, topk, json["params"],
+                                             partition_tags, result);
+    } else {
         std::vector<std::string> file_id_vec;
         if (json.contains("file_ids")) {
             auto ids = json["file_ids"];
@@ -431,12 +453,8 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
             }
         }
 
-    if (!json.contains("params")) {
-        return Status(BODY_FIELD_LOSS, "Field \'params\' is required");
-    }
-
         bool bin_flag = false;
-    auto status = IsBinaryTable(collection_name, bin_flag);
+        status = IsBinaryCollection(collection_name, bin_flag);
         if (!status.ok()) {
             return status;
         }
@@ -451,10 +469,9 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
             return status;
         }
 
-    TopKQueryResult result;
-    status = request_handler_.Search(context_ptr_, collection_name, vectors_data, topk, json["params"], partition_tags,
-                                     file_id_vec, result);
-
+        status = request_handler_.Search(context_ptr_, collection_name, vectors_data, topk, json["params"],
+                                         partition_tags, file_id_vec, result);
+    }
     if (!status.ok()) {
         return status;
     }
@@ -769,7 +786,7 @@ WebRequestHandler::GetVectorsByIDs(const std::string& collection_name, const std
     }
 
     bool bin;
-    status = IsBinaryTable(collection_name, bin);
+    status = IsBinaryCollection(collection_name, bin);
     if (!status.ok()) {
         return status;
     }
@@ -1059,7 +1076,7 @@ WebRequestHandler::SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dt
  * Collection {
  */
 StatusDto::ObjectWrapper
-WebRequestHandler::CreateTable(const TableRequestDto::ObjectWrapper& collection_schema) {
+WebRequestHandler::CreateCollection(const CollectionRequestDto::ObjectWrapper& collection_schema) {
     if (nullptr == collection_schema->collection_name.get()) {
         RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'collection_name\' is missing")
     }
@@ -1133,7 +1150,7 @@ WebRequestHandler::CreateHybridCollection(const milvus::server::web::OString& bo
 }
 
 StatusDto::ObjectWrapper
-WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result) {
+WebRequestHandler::ShowCollections(const OQueryParams& query_params, OString& result) {
     int64_t offset = 0;
     auto status = ParseQueryInteger(query_params, "offset", offset);
     if (!status.ok()) {
@@ -1173,7 +1190,7 @@ WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result)
     nlohmann::json collections_json;
     for (int64_t i = offset; i < page_size + offset; i++) {
         nlohmann::json collection_json;
-        status = GetTableMetaInfo(collections.at(i), collection_json);
+        status = GetCollectionMetaInfo(collections.at(i), collection_json);
         if (!status.ok()) {
             ASSIGN_RETURN_STATUS_DTO(status)
         }
@@ -1194,7 +1211,7 @@ WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result)
 }
 
 StatusDto::ObjectWrapper
-WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams& query_params, OString& result) {
+WebRequestHandler::GetCollection(const OString& collection_name, const OQueryParams& query_params, OString& result) {
     if (nullptr == collection_name.get()) {
         RETURN_STATUS_DTO(PATH_PARAM_LOSS, "Path param \'collection_name\' is required!");
     }
@@ -1207,11 +1224,11 @@ WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams&
 
     if (!stat.empty() && stat == "stat") {
         nlohmann::json json;
-        status = GetTableStat(collection_name->std_str(), json);
+        status = GetCollectionStat(collection_name->std_str(), json);
         result = status.ok() ? json.dump().c_str() : "NULL";
     } else {
         nlohmann::json json;
-        status = GetTableMetaInfo(collection_name->std_str(), json);
+        status = GetCollectionMetaInfo(collection_name->std_str(), json);
         result = status.ok() ? json.dump().c_str() : "NULL";
     }
 
@@ -1219,7 +1236,7 @@ WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams&
 }
 
 StatusDto::ObjectWrapper
-WebRequestHandler::DropTable(const OString& collection_name) {
+WebRequestHandler::DropCollection(const OString& collection_name) {
     auto status = request_handler_.DropCollection(context_ptr_, collection_name->std_str());
 
     ASSIGN_RETURN_STATUS_DTO(status)
@@ -1294,7 +1311,7 @@ WebRequestHandler::CreatePartition(const OString& collection_name, const Partiti
 }
 
 StatusDto::ObjectWrapper
-WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryParams& query_params,
+WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryParams& query_params, const OString& body,
                                   PartitionListDto::ObjectWrapper& partition_list_dto) {
     int64_t offset = 0;
     auto status = ParseQueryInteger(query_params, "offset", offset);
@@ -1313,6 +1330,35 @@ WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryPa
             Status(SERVER_UNEXPECTED_ERROR, "Query param 'offset' or 'page_size' should equal or bigger than 0"));
     }
 
+    if (nullptr != body.get() && body->getSize() > 0) {
+        auto body_json = nlohmann::json::parse(body->c_str());
+        if (!body_json.contains("filter")) {
+            RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'filter\' is required.")
+        }
+        auto filter_json = body_json["filter"];
+        if (filter_json.contains("partition_tag")) {
+            std::string tag = filter_json["partition_tag"];
+            bool exists = false;
+            status = request_handler_.HasPartition(context_ptr_, collection_name->std_str(), tag, exists);
+            if (!status.ok()) {
+                ASSIGN_RETURN_STATUS_DTO(status)
+            }
+            auto partition_dto = PartitionFieldsDto::createShared();
+            if (exists) {
+                partition_list_dto->count = 1;
+                partition_dto->partition_tag = tag.c_str();
+            } else {
+                partition_list_dto->count = 0;
+            }
+            partition_list_dto->partitions = partition_list_dto->partitions->createShared();
+            partition_list_dto->partitions->pushBack(partition_dto);
+
+            ASSIGN_RETURN_STATUS_DTO(status)
+        } else {
+            RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Unknown field.")
+        }
+    }
+
     bool all_required = false;
     auto required = query_params.get("all_required");
     if (nullptr != required.get()) {
@@ -1465,7 +1511,7 @@ WebRequestHandler::Insert(const OString& collection_name, const OString& body, V
 
     // step 1: copy vectors
     bool bin_flag;
-    auto status = IsBinaryTable(collection_name->std_str(), bin_flag);
+    auto status = IsBinaryCollection(collection_name->std_str(), bin_flag);
     if (!status.ok()) {
         ASSIGN_RETURN_STATUS_DTO(status)
     }
@@ -1488,8 +1534,9 @@ WebRequestHandler::Insert(const OString& collection_name, const OString& body, V
         }
         auto& id_array = vectors.id_array_;
         id_array.clear();
-        for (auto& id : ids_json) {
-            id_array.emplace_back(id.get<int64_t>());
+        for (auto& id_str : ids_json) {
+            int64_t id = std::stol(id_str.get<std::string>());
+            id_array.emplace_back(id);
         }
     }
 
@@ -1578,7 +1625,7 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
             }
             case engine::meta::hybrid::DataType::VECTOR: {
                 bool bin_flag;
-                status = IsBinaryTable(collection_name->c_str(), bin_flag);
+                status = IsBinaryCollection(collection_name->c_str(), bin_flag);
                 if (!status.ok()) {
                     ASSIGN_RETURN_STATUS_DTO(status)
                 }
@@ -1612,17 +1659,26 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
 }
 
 StatusDto::ObjectWrapper
-WebRequestHandler::GetVector(const OString& collection_name, const OQueryParams& query_params, OString& response) {
-    int64_t id = 0;
-    auto status = ParseQueryInteger(query_params, "id", id, false);
-    if (!status.ok()) {
-        RETURN_STATUS_DTO(status.code(), status.message().c_str());
+WebRequestHandler::GetVector(const OString& collection_name, const OString& body, const OQueryParams& query_params,
+                             OString& response) {
+    auto status = Status::OK();
+    try {
+        auto body_json = nlohmann::json::parse(body->c_str());
+        if (!body_json.contains("ids")) {
+            RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'ids\' is required.")
+        }
+        auto ids = body_json["ids"];
+        if (!ids.is_array()) {
+            RETURN_STATUS_DTO(BODY_PARSE_FAIL, "Field \'ids\' must be a array.")
         }
 
-    std::vector<int64_t> ids = {id};
+        std::vector<int64_t> vector_ids;
+        for (auto& id : ids) {
+            vector_ids.push_back(std::stol(id.get<std::string>()));
+        }
         engine::VectorsData vectors;
         nlohmann::json vectors_json;
-    status = GetVectorsByIDs(collection_name->std_str(), ids, vectors_json);
+        status = GetVectorsByIDs(collection_name->std_str(), vector_ids, vectors_json);
         if (!status.ok()) {
             response = "NULL";
             ASSIGN_RETURN_STATUS_DTO(status)
@@ -1635,10 +1691,12 @@ WebRequestHandler::GetVector(const OString& collection_name, const OQueryParams&
         } else {
             json["vectors"] = vectors_json;
         }
-
         response = json.dump().c_str();
+    } catch (std::exception& e) {
+        RETURN_STATUS_DTO(SERVER_UNEXPECTED_ERROR, e.what());
+    }
 
-    ASSIGN_RETURN_STATUS_DTO(status)
+    ASSIGN_RETURN_STATUS_DTO(status);
 }
 
 StatusDto::ObjectWrapper
@@ -1718,7 +1776,7 @@ WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString&
         nlohmann::json j = nlohmann::json::parse(body_str->c_str());
         if (op->equals("task")) {
             if (j.contains("load")) {
-                status = PreLoadTable(j["load"], result_str);
+                status = PreLoadCollection(j["load"], result_str);
             } else if (j.contains("flush")) {
                 status = Flush(j["flush"], result_str);
             }

core/src/server/web_impl/handler/WebRequestHandler.h

@@ -27,11 +27,11 @@
 #include "server/context/Context.h"
 #include "server/delivery/RequestHandler.h"
 #include "server/web_impl/Types.h"
+#include "server/web_impl/dto/CollectionDto.hpp"
 #include "server/web_impl/dto/ConfigDto.hpp"
 #include "server/web_impl/dto/DevicesDto.hpp"
 #include "server/web_impl/dto/IndexDto.hpp"
 #include "server/web_impl/dto/PartitionDto.hpp"
-#include "server/web_impl/dto/TableDto.hpp"
 #include "server/web_impl/dto/VectorDto.hpp"
 #include "thirdparty/nlohmann/json.hpp"
 #include "utils/Status.h"
@@ -82,17 +82,17 @@ class WebRequestHandler {
     AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg);
 
     Status
-    IsBinaryTable(const std::string& collection_name, bool& bin);
+    IsBinaryCollection(const std::string& collection_name, bool& bin);
 
     Status
     CopyRecordsFromJson(const nlohmann::json& json, engine::VectorsData& vectors, bool bin);
 
  protected:
     Status
-    GetTableMetaInfo(const std::string& collection_name, nlohmann::json& json_out);
+    GetCollectionMetaInfo(const std::string& collection_name, nlohmann::json& json_out);
 
     Status
-    GetTableStat(const std::string& collection_name, nlohmann::json& json_out);
+    GetCollectionStat(const std::string& collection_name, nlohmann::json& json_out);
 
     Status
     GetSegmentVectors(const std::string& collection_name, const std::string& segment_name, int64_t page_size,
@@ -109,7 +109,7 @@ class WebRequestHandler {
     Cmd(const std::string& cmd, std::string& result_str);
 
     Status
-    PreLoadTable(const nlohmann::json& json, std::string& result_str);
+    PreLoadCollection(const nlohmann::json& json, std::string& result_str);
 
     Status
     Flush(const nlohmann::json& json, std::string& result_str);
@@ -166,18 +166,18 @@ class WebRequestHandler {
 #endif
 
     StatusDto::ObjectWrapper
-    CreateTable(const TableRequestDto::ObjectWrapper& table_schema);
+    CreateCollection(const CollectionRequestDto::ObjectWrapper& table_schema);
     StatusDto::ObjectWrapper
-    ShowTables(const OQueryParams& query_params, OString& result);
+    ShowCollections(const OQueryParams& query_params, OString& result);
 
     StatusDto::ObjectWrapper
     CreateHybridCollection(const OString& body);
 
     StatusDto::ObjectWrapper
-    GetTable(const OString& collection_name, const OQueryParams& query_params, OString& result);
+    GetCollection(const OString& collection_name, const OQueryParams& query_params, OString& result);
 
     StatusDto::ObjectWrapper
-    DropTable(const OString& collection_name);
+    DropCollection(const OString& collection_name);
 
     StatusDto::ObjectWrapper
     CreateIndex(const OString& collection_name, const OString& body);
@@ -192,7 +192,7 @@ class WebRequestHandler {
     CreatePartition(const OString& collection_name, const PartitionRequestDto::ObjectWrapper& param);
 
     StatusDto::ObjectWrapper
-    ShowPartitions(const OString& collection_name, const OQueryParams& query_params,
+    ShowPartitions(const OString& collection_name, const OQueryParams& query_params, const OString& body,
                    PartitionListDto::ObjectWrapper& partition_list_dto);
 
     StatusDto::ObjectWrapper
@@ -220,7 +220,7 @@ class WebRequestHandler {
     InsertEntity(const OString& collection_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto);
 
     StatusDto::ObjectWrapper
-    GetVector(const OString& collection_name, const OQueryParams& query_params, OString& response);
+    GetVector(const OString& collection_name, const OString& body, const OQueryParams& query_params, OString& response);
 
     StatusDto::ObjectWrapper
     VectorsOp(const OString& collection_name, const OString& payload, OString& response);