feat(cpp/db): update for memory and meta operations

Former-commit-id: 69a8c7b6b9656532d4908013f8158d848481a8e6

cpp/src/db/db.h

@@ -25,6 +25,9 @@ public:
                                    const int date_delta_,
                                    GroupFilesSchema& group_files_info_) = 0;
 
+    virtual Status add_vectors(const std::string& group_id_,
+            size_t n, const float* vectors) = 0;
+
     DB() = default;
     DB(const DB&) = delete;
     DB& operator=(const DB&) = delete;

cpp/src/db/db_impl.cpp

@@ -9,7 +9,8 @@ DBImpl::DBImpl(const Options& options_, const std::string& name_)
       _options(options_),
       _bg_work_finish_signal(_mutex),
       _bg_compaction_scheduled(false),
-      _pMeta(new DBMetaImpl(*(_options.pMetaOptions))) {
+      _pMeta(new DBMetaImpl(*(_options.pMetaOptions))),
+      _pMemMgr(new MemManager(_pMeta)) {
 }
 
 Status DBImpl::add_group(const GroupOptions& options_,
@@ -36,6 +37,11 @@ Status DBImpl::get_group_files(const std::string& group_id_,
 
 }
 
+Status DBImpl::add_vectors(const std::string& group_id_,
+        size_t n, const float* vectors, IDNumbers& vector_ids_) {
+    return _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_);
+}
+
 void DBImpl::try_schedule_compaction() {
     if (_bg_compaction_scheduled) return;
     if (!_bg_error.ok()) return;

cpp/src/db/db_impl.h

@@ -6,6 +6,7 @@
 #include <memory>
 #include "db.h"
 #include "memvectors.h"
+#include "types.h"
 
 namespace zilliz {
 namespace vecwise {
@@ -27,11 +28,15 @@ public:
                                    const int date_delta_,
                                    GroupFilesSchema& group_files_info_) override;
 
-    void try_schedule_compaction();
+    virtual Status add_vectors(const std::string& group_id_,
+            size_t n, const float* vectors, IDNumbers& vector_ids_) override;
 
     virtual ~DBImpl();
+
 private:
 
+    void try_schedule_compaction();
+
     static void BGWork(void* db);
     void background_call();
     void background_compaction();
@@ -45,8 +50,8 @@ private:
     bool _bg_compaction_scheduled;
     Status _bg_error;
 
-    MemManager _memMgr;
     std::shared_ptr<Meta> _pMeta;
+    std::shared_ptr<MemManager> _pMemMgr;
 
 }; // DBImpl
 

cpp/src/db/db_meta.h

@@ -11,6 +11,7 @@ struct GroupSchema {
     size_t files_cnt = 0;
     uint16_t dimension;
     std::string location = "";
+    std::string next_file_location = "";
 }; // GroupSchema
 
 

cpp/src/db/id_generators.h

@@ -2,15 +2,12 @@
 #define UTILS_ID_GENERATORS_H_
 
 #include <vector>
+#include "types.h"
 
 namespace zilliz {
 namespace vecwise {
 namespace engine {
 
-#define uint64_t IDNumber;
-#define IDNumber* IDNumberPtr;
-#define std::vector<IDNumber> IDNumbers;
-
 class IDGenerator {
 public:
     virtual IDNumber getNextIDNumber() = 0;

cpp/src/db/memvectors.cpp

@@ -3,6 +3,7 @@
 #include <index_io.h>
 
 #include "memvectors.h"
+#include "db_meta.h"
 
 
 namespace vecengine {
@@ -15,10 +16,9 @@ MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) :
     _pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) {
 }
 
-IDNumbers&& MemVectors::add(size_t n, const float* vectors) {
-    IDNumbers&& ids = _pIdGenerator->getNextIDNumbers(n);
-    _pIdMapIndex->add_with_ids(n, vectors, pIds, &ids[0]);
-    return ids;
+void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
+    vector_ids_ = _pIdGenerator->getNextIDNumbers(n_);
+    _pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]);
 }
 
 size_t MemVectors::total() const {
@@ -57,27 +57,33 @@ MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) {
     if memIt != _memMap.end() {
         return &(memIt->second);
     }
-    // PXU TODO:
-    // 1. Read Group meta info
-    // 2. Initalize MemVectors base meta info
-    return nullptr;
-    /* GroupMetaInfo info; */
-    /* bool succ = env->getGroupMeta(group_id, &info); */
-    /* if (!succ) { */
-    /*     return nullptr; */
-    /* } */
-    /* _memMap[group_id] = MemVectors(info.dimension, info.next_file_location); */
-    /* return &(_memMap[group_id]); */
+
+    GroupSchema group_info;
+    Status status = _pMeta->get_group(group_id_, group_info);
+    if (!status.ok()) {
+        return nullptr;
+    }
+    _memMap[group_id] = MemVectors(group_info.dimension, group_info.next_file_location);
+    return &(_memMap[group_id]);
+}
+
+Status MemManager::add_vectors(const std::string& group_id_,
+        size_t n_,
+        const float* vectors_,
+        IDNumbers& vector_ids_) {
+    // PXU TODO
+    return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_);
 }
 
-IDNumbers&& MemManager::add_vectors_no_lock(const std::string& group_id_,
+Status MemManager::add_vectors_no_lock(const std::string& group_id_,
         size_t n,
-        const float* vectors) {
-    auto mem = get_group_mem(group_id_);
+        const float* vectors,
+        IDNumbers& vector_ids_) {
+    auto mem = get_mem_by_group(group_id_);
     if (mem == nullptr) {
-        return IDNumbers();
+        return Status::NotFound("Group " + group_id_ " not found!");
     }
-    return mem->add(n, vectors);
+    return mem->add(n, vectors, vector_ids_);
 }
 
 

cpp/src/db/memvectors.h

@@ -4,6 +4,7 @@
 #include <map>
 #include <string>
 #include "id_generators.h"
+#include "status.h"
 
 class faiss::IndexIDMap;
 class faiss::Index;
@@ -37,19 +38,24 @@ private:
 }; // MemVectors
 
 
+class Meta;
+
 class MemManager {
 public:
-    MemManager() = default;
+    MemManager(const std::shared_ptr<Meta>& meta_) : _pMeta(meta_) {}
 
     MemVectors* get_mem_by_group(const std::string& group_id_);
 
+    Status add_vectors(const std::string& group_id_,
+            size_t n_, const float* vectors_, IDNumbers& vector_ids_);
+
 private:
-    IDNumbers&& add_vectors_no_lock(const std::string& group_id_,
-            size_t n,
-            const float* vectors);
+    Status add_vectors_no_lock(const std::string& group_id_,
+            size_t n_, const float* vectors_, IDNumbers& vector_ids_);
 
     typedef std::map<std::string, MemVectors> MemMap;
     MemMap _memMap;
+    std::shared_ptr<Meta> _pMeta;
 }; // MemManager
 
 

cpp/src/db/options.h

@@ -9,6 +9,7 @@ namespace vecwise {
 namespace engine {
 
 class MetaOptions;
+class Env;
 
 struct Options {
     uint16_t  memory_sync_interval = 10;

cpp/src/db/status.h

@@ -17,14 +17,20 @@ public:
     Status& operator=(const Status& rhs_) noexcept;
 
     static Status OK() { return Status(); }
+    static Status NotFound(const std::string& msg_, const std::string& msg2_="") {
+        return Status(kNotFound, msg_, msg2_);
+    }
 
     bool ok() const { return _state == nullptr; }
 
+    bool IsNotFound() const { return code() == kNotFound; }
+
 private:
     const char* _state;
 
     enum Code {
         kOK = 0,
+        kNotFound,
     };
 
     Code code() const {

cpp/src/db/types.h

+#ifndef VECENGINE_TYPES_H_
+#define VECENGINE_TYPES_H_
+
+#include <vector>
+
+namespace zilliz {
+namespace vecwise {
+namespace engine {
+
+#define uint64_t IDNumber;
+#define IDNumber* IDNumberPtr;
+#define std::vector<IDNumber> IDNumbers;
+
+
+} // namespace engine
+} // namespace vecwise
+} // namespace zilliz
+
+#endif // VECENGINE_TYPES_H_