diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index 87c83e7027c79d6ff0b10cdf197ba0095b2bb316..4c00d07e0a372e60a70a5fc45a9086498c16b8e8 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include "db_impl.h" #include "db_meta_impl.h" #include "env.h" @@ -100,7 +101,11 @@ void DBImpl::background_call() { } void DBImpl::background_compaction() { - _pMemMgr->serialize(); + std::vector group_ids; + _pMemMgr->serialize(group_ids); + for (auto group_id : group_ids) { + std::cout << __func__ << " group_id=" << group_id << std::endl; + } } DBImpl::~DBImpl() { diff --git a/cpp/src/db/db_meta_impl.cpp b/cpp/src/db/db_meta_impl.cpp index 7ab9f650009c42d34e396f2a636e71981a56c5eb..5ae038e77a105ba9d599378a7b71abbcf436df2b 100644 --- a/cpp/src/db/db_meta_impl.cpp +++ b/cpp/src/db/db_meta_impl.cpp @@ -30,6 +30,7 @@ Status DBMetaImpl::get_group(const std::string& group_id_, GroupSchema& group_in SimpleIDGenerator g; ss.str(""); ss << "/tmp/test/" << g.getNextIDNumber() << ".log"; + group_info_.group_id = "1"; group_info_.dimension = 64; group_info_.next_file_location = ss.str(); return Status::OK(); diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index 2bafe3a9de4b2d8716237081708383bbd0a3b3d9..dac9e8b478d0747ebf5ae15e85d858e928ae09eb 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -13,10 +13,12 @@ namespace zilliz { namespace vecwise { namespace engine { -MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : - _file_location(file_location_), +MemVectors::MemVectors(const std::string& group_id, + size_t dimension, const std::string& file_location) : + group_id_(group_id), + _file_location(file_location), _pIdGenerator(new SimpleIDGenerator()), - _dimension(dimension_), + _dimension(dimension), _pInnerIndex(new faiss::IndexFlat(_dimension)), _pIdMapIndex(new faiss::IndexIDMap(_pInnerIndex)) { } @@ -37,13 +39,15 @@ size_t MemVectors::approximate_size() const { return total() * _dimension; } -void MemVectors::serialize() { +Status MemVectors::serialize(std::string& group_id) { /* std::stringstream ss; */ /* ss << "/tmp/test/" << _pIdGenerator->getNextIDNumber(); */ /* faiss::write_index(_pIdMapIndex, ss.str().c_str()); */ /* std::cout << _pIdMapIndex->ntotal << std::endl; */ /* std::cout << _file_location << std::endl; */ faiss::write_index(_pIdMapIndex, _file_location.c_str()); + group_id = group_id_; + return Status::OK(); } MemVectors::~MemVectors() { @@ -77,7 +81,8 @@ VectorsPtr MemManager::get_mem_by_group(const std::string& group_id) { return nullptr; } - _memMap[group_id] = std::shared_ptr(new MemVectors(group_info.dimension, + _memMap[group_id] = std::shared_ptr(new MemVectors(group_info.group_id, + group_info.dimension, group_info.next_file_location)); return _memMap[group_id]; } @@ -126,10 +131,13 @@ Status MemManager::mark_memory_as_immutable() { /* return false; */ /* } */ -Status MemManager::serialize() { +Status MemManager::serialize(std::vector& group_ids) { mark_memory_as_immutable(); + std::string group_id; + group_ids.clear(); for (auto& mem : _immMems) { - mem->serialize(); + mem->serialize(group_id); + group_ids.push_back(group_id); } _immMems.clear(); return Status::OK(); diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index f9a0bba1f223252a2fe41607300f47d20444379d..6b70df267c75ba7a99f436c9cde99f37f012410f 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -21,7 +21,9 @@ namespace engine { class MemVectors { public: - explicit MemVectors(size_t dimension_, const std::string& file_location_); + explicit MemVectors(const std::string& group_id, + size_t dimension, + const std::string& file_location); void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); @@ -29,7 +31,7 @@ public: size_t approximate_size() const; - void serialize(); + Status serialize(std::string& group_id); ~MemVectors(); @@ -40,6 +42,7 @@ private: MemVectors(const MemVectors&) = delete; MemVectors& operator=(const MemVectors&) = delete; + std::string group_id_; const std::string _file_location; IDGenerator* _pIdGenerator; size_t _dimension; @@ -62,7 +65,7 @@ public: Status add_vectors(const std::string& group_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); - Status serialize(); + Status serialize(std::vector& group_ids); private: Status add_vectors_no_lock(const std::string& group_id_,