add id mapper

Former-commit-id: 56cfee9e3beeb474e2a56c3e3a3fdda111cfabb2

cpp/README.md

 ### Compilation
 #### Step 1: install necessery tools
 
-    centos7 : yum install gfortran
-    ubuntu16.04 : sudo apt-install install gfortran libsqlite3-dev
+    centos7 : 
+        yum install gfortran libsqlite3-dev libsnappy-dev zlib bzip2
+        
+    ubuntu16.04 : 
+        sudo apt-install install gfortran libsqlite3-dev libsnappy-dev zlib bzip2
 
 #### Step 2: build third-parties
 Note: If you want to debug into third-parties, you can build debug with CXXFLAGS='-g -O0' with option 

cpp/src/CMakeLists.txt

@@ -24,6 +24,7 @@ set(vecwise_engine_src
         ${wrapper_files}
         )
 
+include_directories(/usr/include)
 include_directories(/usr/local/cuda/include)
 find_library(cuda_library cudart cublas HINTS /usr/local/cuda/lib64)
 
@@ -37,6 +38,8 @@ set(engine_libs
         sqlite3
         )
 
+link_libraries(/usr/lib/x86_64-linux-gnu)
+
 target_link_libraries(vecwise_engine ${engine_libs} ${cuda_library})
 
 add_executable(vecwise_server
@@ -48,12 +51,17 @@ add_executable(vecwise_server
         )
 
 set(server_libs
+        vecwise_engine
+        rocksdb
+        thrift
+        pthread
         yaml-cpp
         boost_system
         boost_filesystem
-        thrift
         pthread
-        vecwise_engine
+        snappy
+        bz2
+        z
         )
 
 target_link_libraries(vecwise_server ${server_libs})

cpp/src/server/VecIdMapper.cpp

@@ -7,19 +7,25 @@
 #include "VecIdMapper.h"
 #include "ServerConfig.h"
 #include "utils/Log.h"
+#include "utils/CommonUtil.h"
+
+#include "rocksdb/db.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/options.h"
 
 namespace zilliz {
 namespace vecwise {
 namespace server {
 
 IVecIdMapper* IVecIdMapper::GetInstance() {
-#if 1
+#if 0
     static SimpleIdMapper s_mapper;
     return &s_mapper;
 #else
     ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER);
     std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH);
     db_path += "/id_mapping";
+    CommonUtil::CreateDirectory(db_path);
     static RocksIdMapper s_mapper(db_path);
     return &s_mapper;
 #endif
@@ -78,7 +84,7 @@ ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vect
         auto iter = ids_.find(nid[i]);
         if(iter == ids_.end()) {
             sid.push_back("");
-            SERVER_LOG_ERROR << "Failed to find id: " << nid[i];
+            SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i];
             err = SERVER_INVALID_ARGUMENT;
             continue;
         }
@@ -97,37 +103,117 @@ ServerError SimpleIdMapper::Delete(INTEGER_ID nid) {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 RocksIdMapper::RocksIdMapper(const std::string& store_path) {
-
+    rocksdb::Options options;
+    // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
+    options.IncreaseParallelism();
+    options.OptimizeLevelStyleCompaction();
+    // create the DB if it's not already present
+    options.create_if_missing = true;
+
+    // open DB
+    rocksdb::Status s = rocksdb::DB::Open(options, store_path, &db_);
+    if(!s.ok()) {
+        SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
+        db_ = nullptr;
+    }
 }
 RocksIdMapper::~RocksIdMapper() {
-
+    if(db_) {
+        db_->Close();
+        delete db_;
+    }
 }
 
 ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) {
+    if(db_ == nullptr) {
+        return SERVER_NULL_POINTER;
+    }
+
+    std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
+    rocksdb::Slice key(str_id);
+    rocksdb::Slice value(sid);
+    rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value);
+    if(!s.ok()) {
+        SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString();
+        return SERVER_UNEXPECTED_ERROR;
+    }
+
     return SERVER_SUCCESS;
 }
 
 ServerError RocksIdMapper::Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) {
-    return SERVER_SUCCESS;
+    return Put(nid.data(), nid.size(), sid);
 }
 
 ServerError RocksIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) {
+    if(count != sid.size()) {
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    ServerError err = SERVER_SUCCESS;
+    for(int64_t i = 0; i < count; i++) {
+        err = Put(nid[i], sid[i]);
+        if(err != SERVER_SUCCESS) {
+            return err;
+        }
+    }
+
     return SERVER_SUCCESS;
 }
 
 ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
+    if(db_ == nullptr) {
+        return SERVER_NULL_POINTER;
+    }
+
+    std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
+    rocksdb::Slice key(str_id);
+    rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid);
+    if(!s.ok()) {
+        SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString();
+        return SERVER_UNEXPECTED_ERROR;
+    }
+
     return SERVER_SUCCESS;
 }
 
 ServerError RocksIdMapper::Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const {
-    return SERVER_SUCCESS;
+    return Get(nid.data(), nid.size(), sid);
 }
 
 ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const {
-    return SERVER_SUCCESS;
+    sid.clear();
+
+    ServerError err = SERVER_SUCCESS;
+    for(uint64_t i = 0; i < count; i++) {
+        std::string str_id;
+        ServerError temp_err = Get(nid[i], str_id);
+        if(temp_err != SERVER_SUCCESS) {
+            sid.push_back("");
+            SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i];
+            err = temp_err;
+            continue;
+        }
+
+        sid.push_back(str_id);
+    }
+
+    return err;
 }
 
 ServerError RocksIdMapper::Delete(INTEGER_ID nid) {
+    if(db_ == nullptr) {
+        return SERVER_NULL_POINTER;
+    }
+
+    std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
+    rocksdb::Slice key(str_id);
+    rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key);
+    if(!s.ok()) {
+        SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString();
+        return SERVER_UNEXPECTED_ERROR;
+    }
+
     return SERVER_SUCCESS;
 }
 

cpp/src/server/VecIdMapper.h

@@ -11,6 +11,10 @@
 #include <vector>
 #include <unordered_map>
 
+namespace rocksdb {
+    class DB;
+}
+
 namespace zilliz {
 namespace vecwise {
 namespace server {
@@ -68,6 +72,8 @@ public:
 
     ServerError Delete(INTEGER_ID nid) override;
 
+private:
+    rocksdb::DB* db_;
 };
 
 }

cpp/src/server/VecServiceHandler.cpp

@@ -23,6 +23,8 @@ VecServiceHandler::VecServiceHandler() {
     std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH);
     opt.meta.path = db_path + "/db";
 
+    CommonUtil::CreateDirectory(opt.meta.path);
+
     zilliz::vecwise::engine::DB::Open(opt, &db_);
 }
 

cpp/unittest/db/db_tests.cpp

@@ -34,7 +34,7 @@ TEST(DBTest, DB_TEST) {
     group_info.dimension = group_dim;
     group_info.group_id = group_name;
     engine::Status stat = db->add_group(group_info);
-    ASSERT_STATS(stat);
+    //ASSERT_STATS(stat);
 
     engine::meta::GroupSchema group_info_get;
     group_info_get.group_id = group_name;

cpp/unittest/server/CMakeLists.txt

@@ -4,6 +4,7 @@
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
 include_directories(../../src)
+include_directories(/usr/include)
 
 aux_source_directory(../../src/config config_files)
 aux_source_directory(../../src/cache cache_srcs)
@@ -12,6 +13,7 @@ aux_source_directory(./ test_srcs)
 set(require_files
         ../../src/server/VecIdMapper.cpp
         ../../src/server/ServerConfig.cpp
+        ../../src/utils/CommonUtil.cpp
         )
 
 add_executable(server_test
@@ -26,7 +28,14 @@ set(require_libs
         yaml-cpp
         boost_system
         boost_filesystem
-        pthread)
+        pthread
+        snappy
+        bz2
+        z
+        rocksdb
+        )
+
+link_libraries(/usr/lib/x86_64-linux-gnu)
 
 target_link_libraries(server_test
         ${unittest_libs}

cpp/unittest/server/idmapper_test.cpp

@@ -21,13 +21,15 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
     server::ServerError err = mapper->Put(nid, sid);
     ASSERT_EQ(err, server::SERVER_SUCCESS);
 
-    sid.clear();
-    err = mapper->Put(nid, sid);
+    err = mapper->Put(nid, std::vector<std::string>());
     ASSERT_NE(err, server::SERVER_SUCCESS);
 
     std::vector<std::string> res;
     err = mapper->Get(nid, res);
     ASSERT_EQ(res.size(), nid.size());
+    for(size_t i = 0; i < res.size(); i++) {
+        ASSERT_EQ(res[i], sid[i]);
+    }
 
     std::string str_id;
     err = mapper->Get(50, str_id);