diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index 18550f18393bd81aef6347f8ba2d3b5f972d333b..77633c74872a4e045812960621172a7f46f022ec 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -6,16 +6,83 @@ #include "VecIdMapper.h" +#include "utils/Log.h" + namespace zilliz { namespace vecwise { namespace server { -VecIdMapper::VecIdMapper() { +IVecIdMapper* IVecIdMapper::GetInstance() { + static SimpleIdMapper s_mapper; + return &s_mapper; +} + +SimpleIdMapper::SimpleIdMapper() { + +} + +SimpleIdMapper::~SimpleIdMapper() { + +} + +ServerError SimpleIdMapper::Put(INTEGER_ID nid, const std::string& sid) { + ids_[nid] = sid; + return SERVER_SUCCESS; +} + +ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid) { + return Put(nid.data(), nid.size(), sid); +} + +ServerError SimpleIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) { + if(count != sid.size()) { + return SERVER_INVALID_ARGUMENT; + } + + for(int64_t i = 0; i < count; i++) { + ids_[nid[i]] = sid[i]; + } + + return SERVER_SUCCESS; +} + +ServerError SimpleIdMapper::Get(INTEGER_ID nid, std::string& sid) const { + auto iter = ids_.find(nid); + if(iter == ids_.end()) { + return SERVER_INVALID_ARGUMENT; + } + + sid = iter->second; + return SERVER_SUCCESS; } -VecIdMapper::~VecIdMapper() { +ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid) const { + return Get(nid.data(), nid.size(), sid); +} + +ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const { + sid.clear(); + + ServerError err = SERVER_SUCCESS; + for(uint64_t i = 0; i < count; i++) { + auto iter = ids_.find(nid[i]); + if(iter == ids_.end()) { + sid.push_back(""); + SERVER_LOG_ERROR << "Failed to find id: " << nid[i]; + err = SERVER_INVALID_ARGUMENT; + continue; + } + + sid.push_back(iter->second); + } + + return err; +} +ServerError SimpleIdMapper::Delete(INTEGER_ID nid) { + ids_.erase(nid); + return SERVER_SUCCESS; } } diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index 6dec386e50aae94476c789a2c216a8a28edd1b0f..1ebece2df06127d47667bf67d262935a24af6b1a 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -5,18 +5,71 @@ ******************************************************************************/ #pragma once +#include "utils/Error.h" + +#include +#include +#include + namespace zilliz { namespace vecwise { namespace server { -class VecIdMapper { +using INTEGER_ID = int64_t; + +class IVecIdMapper { public: - VecIdMapper(); - ~VecIdMapper(); + static IVecIdMapper* GetInstance(); + + virtual ~IVecIdMapper(){} + virtual ServerError Put(INTEGER_ID nid, const std::string& sid) = 0; + virtual ServerError Put(const std::vector& nid, const std::vector& sid) = 0; + virtual ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) = 0; + virtual ServerError Get(INTEGER_ID nid, std::string& sid) const = 0; + virtual ServerError Get(const std::vector& nid, std::vector& sid) const = 0; + virtual ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const = 0; + + virtual ServerError Delete(INTEGER_ID nid) = 0; }; +class SimpleIdMapper : public IVecIdMapper{ +public: + SimpleIdMapper(); + ~SimpleIdMapper(); + + ServerError Put(INTEGER_ID nid, const std::string& sid) override; + ServerError Put(const std::vector& nid, const std::vector& sid) override; + ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) override; + + ServerError Get(INTEGER_ID nid, std::string& sid) const override; + ServerError Get(const std::vector& nid, std::vector& sid) const override; + ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const override; + + ServerError Delete(INTEGER_ID nid) override; + +private: + std::unordered_map ids_; +}; + +//class RocksIdMapper : public IVecIdMapper{ +//public: +// RocksIdMapper(); +// ~RocksIdMapper(); +// +// ServerError Put(INTEGER_ID nid, const std::string& sid) override; +// ServerError Put(const std::vector& nid, const std::vector& sid) override; +// ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector& sid) override; +// +// ServerError Get(INTEGER_ID nid, std::string& sid) const override; +// ServerError Get(const std::vector& nid, std::vector& sid) const override; +// ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector& sid) const override; +// +// ServerError Delete(INTEGER_ID nid) override; +// +//}; + } } } diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index 9dcdc888d132e6afb7e3f0bec535106fb05d7a7b..b67fb349ef1a83e6a1074962946d9a1cf294a02b 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "VecServiceHandler.h" #include "ServerConfig.h" +#include "VecIdMapper.h" #include "utils/Log.h" #include "utils/CommonUtil.h" @@ -93,6 +94,11 @@ VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tens if(!stat.ok()) { SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); } else { + if(vector_ids.size() != 1) { + SERVER_LOG_ERROR << "Vector ID not returned"; + } else { + IVecIdMapper::GetInstance()->Put(vector_ids[0], tensor.uid); + } } SERVER_LOG_INFO << "add_vector() finished"; @@ -119,6 +125,13 @@ VecServiceHandler::add_vector_batch(const std::string &group_id, if(!stat.ok()) { SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); } else { + if(vector_ids.size() != tensor_list.tensor_list.size()) { + SERVER_LOG_ERROR << "Vector ID not returned"; + } else { + for(size_t i = 0; i < vector_ids.size(); i++) { + IVecIdMapper::GetInstance()->Put(vector_ids[i], tensor_list.tensor_list[i].uid); + } + } } SERVER_LOG_INFO << "add_vector_batch() finished"; @@ -148,7 +161,9 @@ VecServiceHandler::search_vector(VecSearchResult &_return, } else { if(!results.empty()) { for(auto id : results[0]) { - _return.id_list.push_back(std::to_string(id)); + std::string sid; + IVecIdMapper::GetInstance()->Get(id, sid); + _return.id_list.push_back(sid); } } } @@ -182,11 +197,11 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return, SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); } else { for(engine::QueryResult& res : results){ - VecSearchResult v_res; - for(auto id : results[0]) { - v_res.id_list.push_back(std::to_string(id)); + for(auto nid : results) { + VecSearchResult v_res; + IVecIdMapper::GetInstance()->Get(nid, v_res.id_list); + _return.result_list.push_back(v_res); } - _return.result_list.push_back(v_res); } } diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index dfd16c15fc03a5b89ad78c3d4066bada8317f433..b970cf23940d9f7280ad770d8e9420fbf33669d8 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -16,7 +16,6 @@ set(unittest_libs gmock_main pthread) -add_subdirectory(cache) -add_subdirectory(log) +add_subdirectory(server) add_subdirectory(db) add_subdirectory(faiss_wrapper) \ No newline at end of file diff --git a/cpp/unittest/log/CMakeLists.txt b/cpp/unittest/log/CMakeLists.txt deleted file mode 100644 index d6964b9c52ea4702e9b0db47be7219dcb36ee839..0000000000000000000000000000000000000000 --- a/cpp/unittest/log/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. -#------------------------------------------------------------------------------- -set(log_test_src ${unittest_srcs} log_test.cpp) - -add_executable(log_test ${log_test_src}) - -target_link_libraries(log_test ${unittest_libs}) diff --git a/cpp/unittest/cache/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt similarity index 72% rename from cpp/unittest/cache/CMakeLists.txt rename to cpp/unittest/server/CMakeLists.txt index 787f3cb11b2d0ff517709efa04889db8daba55c4..b2d87478d349e612bd18eb14e178e9755f1403ab 100644 --- a/cpp/unittest/cache/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -6,12 +6,13 @@ include_directories(../../src) aux_source_directory(../../src/cache cache_srcs) +aux_source_directory(./ test_srcs) -set(cache_test_src +add_executable(server_test ${unittest_srcs} ${cache_srcs} - cache_test.cpp) + ${test_srcs} + ../../src/server/VecIdMapper.cpp + ) -add_executable(cache_test ${cache_test_src}) - -target_link_libraries(cache_test ${unittest_libs}) +target_link_libraries(server_test ${unittest_libs}) diff --git a/cpp/unittest/cache/cache_test.cpp b/cpp/unittest/server/cache_test.cpp similarity index 100% rename from cpp/unittest/cache/cache_test.cpp rename to cpp/unittest/server/cache_test.cpp diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f2195d83620e9b451e0e264d420ac2b78aab390c --- /dev/null +++ b/cpp/unittest/server/idmapper_test.cpp @@ -0,0 +1,39 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "server/VecIdMapper.h" + +using namespace zilliz::vecwise; + + +TEST(IdMapperTest, IDMAPPER_TEST) { + server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance(); + + std::vector nid = {1,50, 900, 10000}; + std::vector sid = {"one", "fifty", "nine zero zero", "many"}; + server::ServerError err = mapper->Put(nid, sid); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + sid.clear(); + err = mapper->Put(nid, sid); + ASSERT_NE(err, server::SERVER_SUCCESS); + + std::vector res; + err = mapper->Get(nid, res); + ASSERT_EQ(res.size(), nid.size()); + + std::string str_id; + err = mapper->Get(50, str_id); + ASSERT_EQ(str_id, "fifty"); + + err = mapper->Delete(900); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + err = mapper->Get(900, str_id); + ASSERT_NE(err, server::SERVER_SUCCESS); +} + diff --git a/cpp/unittest/log/log_test.cpp b/cpp/unittest/server/log_test.cpp similarity index 100% rename from cpp/unittest/log/log_test.cpp rename to cpp/unittest/server/log_test.cpp