add id mapper class

Former-commit-id: b12be12ea33f4b3f99a4f65b1156d7a86007dcc1

cpp/src/server/VecIdMapper.cpp

@@ -6,16 +6,83 @@
 
 #include "VecIdMapper.h"
 
+#include "utils/Log.h"
+
 namespace zilliz {
 namespace vecwise {
 namespace server {
 
-VecIdMapper::VecIdMapper() {
+IVecIdMapper* IVecIdMapper::GetInstance() {
+    static SimpleIdMapper s_mapper;
+    return &s_mapper;
+}
+
+SimpleIdMapper::SimpleIdMapper() {
+
+}
+
+SimpleIdMapper::~SimpleIdMapper() {
+
+}
+
+ServerError SimpleIdMapper::Put(INTEGER_ID nid, const std::string& sid) {
+    ids_[nid] = sid;
+    return SERVER_SUCCESS;
+}
+
+ServerError SimpleIdMapper::Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) {
+    return Put(nid.data(), nid.size(), sid);
+}
+
+ServerError SimpleIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) {
+    if(count != sid.size()) {
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    for(int64_t i = 0; i < count; i++) {
+        ids_[nid[i]] = sid[i];
+    }
+
+    return SERVER_SUCCESS;
+}
+
+ServerError SimpleIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
+    auto iter = ids_.find(nid);
+    if(iter == ids_.end()) {
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    sid = iter->second;
 
+    return SERVER_SUCCESS;
 }
 
-VecIdMapper::~VecIdMapper() {
+ServerError SimpleIdMapper::Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const {
+    return Get(nid.data(), nid.size(), sid);
+}
+
+ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const {
+    sid.clear();
+
+    ServerError err = SERVER_SUCCESS;
+    for(uint64_t i = 0; i < count; i++) {
+        auto iter = ids_.find(nid[i]);
+        if(iter == ids_.end()) {
+            sid.push_back("");
+            SERVER_LOG_ERROR << "Failed to find id: " << nid[i];
+            err = SERVER_INVALID_ARGUMENT;
+            continue;
+        }
+
+        sid.push_back(iter->second);
+    }
+
+    return err;
+}
 
+ServerError SimpleIdMapper::Delete(INTEGER_ID nid) {
+    ids_.erase(nid);
+    return SERVER_SUCCESS;
 }
 
 }

cpp/src/server/VecIdMapper.h

@@ -5,18 +5,71 @@
  ******************************************************************************/
 #pragma once
 
+#include "utils/Error.h"
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+
 namespace zilliz {
 namespace vecwise {
 namespace server {
 
-class VecIdMapper {
+using INTEGER_ID = int64_t;
+
+class IVecIdMapper {
 public:
-    VecIdMapper();
-    ~VecIdMapper();
+    static IVecIdMapper* GetInstance();
+
+    virtual ~IVecIdMapper(){}
 
+    virtual ServerError Put(INTEGER_ID nid, const std::string& sid) = 0;
+    virtual ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) = 0;
+    virtual ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) = 0;
 
+    virtual ServerError Get(INTEGER_ID nid, std::string& sid) const = 0;
+    virtual ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const = 0;
+    virtual ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const = 0;
+
+    virtual ServerError Delete(INTEGER_ID nid) = 0;
 };
 
+class SimpleIdMapper : public IVecIdMapper{
+public:
+    SimpleIdMapper();
+    ~SimpleIdMapper();
+
+    ServerError Put(INTEGER_ID nid, const std::string& sid) override;
+    ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) override;
+    ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) override;
+
+    ServerError Get(INTEGER_ID nid, std::string& sid) const override;
+    ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const override;
+    ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const override;
+
+    ServerError Delete(INTEGER_ID nid) override;
+
+private:
+    std::unordered_map<INTEGER_ID, std::string> ids_;
+};
+
+//class RocksIdMapper : public IVecIdMapper{
+//public:
+//    RocksIdMapper();
+//    ~RocksIdMapper();
+//
+//    ServerError Put(INTEGER_ID nid, const std::string& sid) override;
+//    ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) override;
+//    ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) override;
+//
+//    ServerError Get(INTEGER_ID nid, std::string& sid) const override;
+//    ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const override;
+//    ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const override;
+//
+//    ServerError Delete(INTEGER_ID nid) override;
+//
+//};
+
 }
 }
 }

cpp/src/server/VecServiceHandler.cpp

@@ -5,6 +5,7 @@
  ******************************************************************************/
 #include "VecServiceHandler.h"
 #include "ServerConfig.h"
+#include "VecIdMapper.h"
 #include "utils/Log.h"
 #include "utils/CommonUtil.h"
 
@@ -93,6 +94,11 @@ VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tens
         if(!stat.ok()) {
             SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
         } else {
+            if(vector_ids.size() != 1) {
+                SERVER_LOG_ERROR << "Vector ID not returned";
+            } else {
+                IVecIdMapper::GetInstance()->Put(vector_ids[0], tensor.uid);
+            }
         }
 
         SERVER_LOG_INFO << "add_vector() finished";
@@ -119,6 +125,13 @@ VecServiceHandler::add_vector_batch(const std::string &group_id,
         if(!stat.ok()) {
             SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
         } else {
+            if(vector_ids.size() != tensor_list.tensor_list.size()) {
+                SERVER_LOG_ERROR << "Vector ID not returned";
+            } else {
+                for(size_t i = 0; i < vector_ids.size(); i++) {
+                    IVecIdMapper::GetInstance()->Put(vector_ids[i], tensor_list.tensor_list[i].uid);
+                }
+            }
         }
 
         SERVER_LOG_INFO << "add_vector_batch() finished";
@@ -148,7 +161,9 @@ VecServiceHandler::search_vector(VecSearchResult &_return,
         } else {
             if(!results.empty()) {
                 for(auto id : results[0]) {
-                    _return.id_list.push_back(std::to_string(id));
+                    std::string sid;
+                    IVecIdMapper::GetInstance()->Get(id, sid);
+                    _return.id_list.push_back(sid);
                 }
             }
         }
@@ -182,11 +197,11 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return,
             SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
         } else {
             for(engine::QueryResult& res : results){
-                VecSearchResult v_res;
-                for(auto id : results[0]) {
-                    v_res.id_list.push_back(std::to_string(id));
+                for(auto nid : results) {
+                    VecSearchResult v_res;
+                    IVecIdMapper::GetInstance()->Get(nid, v_res.id_list);
+                    _return.result_list.push_back(v_res);
                 }
-                _return.result_list.push_back(v_res);
             }
         }
 

cpp/unittest/CMakeLists.txt

@@ -16,7 +16,6 @@ set(unittest_libs
         gmock_main
         pthread)
 
-add_subdirectory(cache)
-add_subdirectory(log)
+add_subdirectory(server)
 add_subdirectory(db)
 add_subdirectory(faiss_wrapper)
\ No newline at end of file

cpp/unittest/log/CMakeLists.txt

-#-------------------------------------------------------------------------------
-# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-# Unauthorized copying of this file, via any medium is strictly prohibited.
-# Proprietary and confidential.
-#-------------------------------------------------------------------------------
-set(log_test_src ${unittest_srcs} log_test.cpp)
-
-add_executable(log_test ${log_test_src})
-
-target_link_libraries(log_test ${unittest_libs})

cpp/unittest/cache/CMakeLists.txt → cpp/unittest/server/CMakeLists.txt

@@ -6,12 +6,13 @@
 include_directories(../../src)
 
 aux_source_directory(../../src/cache cache_srcs)
+aux_source_directory(./ test_srcs)
 
-set(cache_test_src
+add_executable(server_test
         ${unittest_srcs}
         ${cache_srcs}
-        cache_test.cpp)
+        ${test_srcs}
+        ../../src/server/VecIdMapper.cpp
+        )
 
-add_executable(cache_test ${cache_test_src})
-
-target_link_libraries(cache_test ${unittest_libs})
+target_link_libraries(server_test ${unittest_libs})

cpp/unittest/server/idmapper_test.cpp

+////////////////////////////////////////////////////////////////////////////////
+// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// Unauthorized copying of this file, via any medium is strictly prohibited.
+// Proprietary and confidential.
+////////////////////////////////////////////////////////////////////////////////
+#include <gtest/gtest.h>
+
+#include "server/VecIdMapper.h"
+
+using namespace zilliz::vecwise;
+
+
+TEST(IdMapperTest, IDMAPPER_TEST) {
+    server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance();
+
+    std::vector<int64_t> nid = {1,50, 900, 10000};
+    std::vector<std::string> sid = {"one", "fifty", "nine zero zero", "many"};
+    server::ServerError err = mapper->Put(nid, sid);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
+    sid.clear();
+    err = mapper->Put(nid, sid);
+    ASSERT_NE(err, server::SERVER_SUCCESS);
+
+    std::vector<std::string> res;
+    err = mapper->Get(nid, res);
+    ASSERT_EQ(res.size(), nid.size());
+
+    std::string str_id;
+    err = mapper->Get(50, str_id);
+    ASSERT_EQ(str_id, "fifty");
+
+    err = mapper->Delete(900);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
+    err = mapper->Get(900, str_id);
+    ASSERT_NE(err, server::SERVER_SUCCESS);
+}
+