diff --git a/cpp/src/db/db.h b/cpp/src/db/db.h index 525a661e1eb801cac085121fa6b146286686d385..d55530408a7e3d29157d7f90aec33a0009accbef 100644 --- a/cpp/src/db/db.h +++ b/cpp/src/db/db.h @@ -25,6 +25,9 @@ public: const int date_delta_, GroupFilesSchema& group_files_info_) = 0; + virtual Status add_vectors(const std::string& group_id_, + size_t n, const float* vectors) = 0; + DB() = default; DB(const DB&) = delete; DB& operator=(const DB&) = delete; diff --git a/cpp/src/db/db_impl.cpp b/cpp/src/db/db_impl.cpp index 332942073cbc1f1b9d05a0df7c3b0a36ed83bbe3..b5c85b1c2011756ff992f7719d575da7390ad883 100644 --- a/cpp/src/db/db_impl.cpp +++ b/cpp/src/db/db_impl.cpp @@ -9,7 +9,8 @@ DBImpl::DBImpl(const Options& options_, const std::string& name_) _options(options_), _bg_work_finish_signal(_mutex), _bg_compaction_scheduled(false), - _pMeta(new DBMetaImpl(*(_options.pMetaOptions))) { + _pMeta(new DBMetaImpl(*(_options.pMetaOptions))), + _pMemMgr(new MemManager(_pMeta)) { } Status DBImpl::add_group(const GroupOptions& options_, @@ -36,6 +37,11 @@ Status DBImpl::get_group_files(const std::string& group_id_, } +Status DBImpl::add_vectors(const std::string& group_id_, + size_t n, const float* vectors, IDNumbers& vector_ids_) { + return _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_); +} + void DBImpl::try_schedule_compaction() { if (_bg_compaction_scheduled) return; if (!_bg_error.ok()) return; diff --git a/cpp/src/db/db_impl.h b/cpp/src/db/db_impl.h index cc52a05dfff3d400458a5acc4f27f313c886ec06..03fc3d3c0b0446ef2ccd7e99a02dcc8cc1029494 100644 --- a/cpp/src/db/db_impl.h +++ b/cpp/src/db/db_impl.h @@ -6,6 +6,7 @@ #include #include "db.h" #include "memvectors.h" +#include "types.h" namespace zilliz { namespace vecwise { @@ -27,11 +28,15 @@ public: const int date_delta_, GroupFilesSchema& group_files_info_) override; - void try_schedule_compaction(); + virtual Status add_vectors(const std::string& group_id_, + size_t n, const float* vectors, IDNumbers& vector_ids_) override; virtual ~DBImpl(); + private: + void try_schedule_compaction(); + static void BGWork(void* db); void background_call(); void background_compaction(); @@ -45,8 +50,8 @@ private: bool _bg_compaction_scheduled; Status _bg_error; - MemManager _memMgr; std::shared_ptr _pMeta; + std::shared_ptr _pMemMgr; }; // DBImpl diff --git a/cpp/src/db/db_meta.h b/cpp/src/db/db_meta.h index 0570c44eba56aa676a43e36ae6d28c896ca67d05..da2b40edf0d1be6484e6a516d5c6d72d40bccb75 100644 --- a/cpp/src/db/db_meta.h +++ b/cpp/src/db/db_meta.h @@ -11,6 +11,7 @@ struct GroupSchema { size_t files_cnt = 0; uint16_t dimension; std::string location = ""; + std::string next_file_location = ""; }; // GroupSchema diff --git a/cpp/src/db/id_generators.h b/cpp/src/db/id_generators.h index 767c3ce3224352078db904881779f40d3efb56a7..02f3915bcc7129cb01346494c61802ef39932d9c 100644 --- a/cpp/src/db/id_generators.h +++ b/cpp/src/db/id_generators.h @@ -2,15 +2,12 @@ #define UTILS_ID_GENERATORS_H_ #include +#include "types.h" namespace zilliz { namespace vecwise { namespace engine { -#define uint64_t IDNumber; -#define IDNumber* IDNumberPtr; -#define std::vector IDNumbers; - class IDGenerator { public: virtual IDNumber getNextIDNumber() = 0; diff --git a/cpp/src/db/memvectors.cpp b/cpp/src/db/memvectors.cpp index 3a30dfda9efda3622c773b620578f93fa49ec679..f546fcf02b61bbf790285b10773eca559dd06888 100644 --- a/cpp/src/db/memvectors.cpp +++ b/cpp/src/db/memvectors.cpp @@ -3,6 +3,7 @@ #include #include "memvectors.h" +#include "db_meta.h" namespace vecengine { @@ -15,10 +16,9 @@ MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : _pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) { } -IDNumbers&& MemVectors::add(size_t n, const float* vectors) { - IDNumbers&& ids = _pIdGenerator->getNextIDNumbers(n); - _pIdMapIndex->add_with_ids(n, vectors, pIds, &ids[0]); - return ids; +void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { + vector_ids_ = _pIdGenerator->getNextIDNumbers(n_); + _pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]); } size_t MemVectors::total() const { @@ -57,27 +57,33 @@ MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) { if memIt != _memMap.end() { return &(memIt->second); } - // PXU TODO: - // 1. Read Group meta info - // 2. Initalize MemVectors base meta info - return nullptr; - /* GroupMetaInfo info; */ - /* bool succ = env->getGroupMeta(group_id, &info); */ - /* if (!succ) { */ - /* return nullptr; */ - /* } */ - /* _memMap[group_id] = MemVectors(info.dimension, info.next_file_location); */ - /* return &(_memMap[group_id]); */ + + GroupSchema group_info; + Status status = _pMeta->get_group(group_id_, group_info); + if (!status.ok()) { + return nullptr; + } + _memMap[group_id] = MemVectors(group_info.dimension, group_info.next_file_location); + return &(_memMap[group_id]); +} + +Status MemManager::add_vectors(const std::string& group_id_, + size_t n_, + const float* vectors_, + IDNumbers& vector_ids_) { + // PXU TODO + return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); } -IDNumbers&& MemManager::add_vectors_no_lock(const std::string& group_id_, +Status MemManager::add_vectors_no_lock(const std::string& group_id_, size_t n, - const float* vectors) { - auto mem = get_group_mem(group_id_); + const float* vectors, + IDNumbers& vector_ids_) { + auto mem = get_mem_by_group(group_id_); if (mem == nullptr) { - return IDNumbers(); + return Status::NotFound("Group " + group_id_ " not found!"); } - return mem->add(n, vectors); + return mem->add(n, vectors, vector_ids_); } diff --git a/cpp/src/db/memvectors.h b/cpp/src/db/memvectors.h index 159ee441d48f8697d6e9971b424e643fd27f0bc8..a6741971a1c54ce56a3fdc8d33b062ea4773da57 100644 --- a/cpp/src/db/memvectors.h +++ b/cpp/src/db/memvectors.h @@ -4,6 +4,7 @@ #include #include #include "id_generators.h" +#include "status.h" class faiss::IndexIDMap; class faiss::Index; @@ -37,19 +38,24 @@ private: }; // MemVectors +class Meta; + class MemManager { public: - MemManager() = default; + MemManager(const std::shared_ptr& meta_) : _pMeta(meta_) {} MemVectors* get_mem_by_group(const std::string& group_id_); + Status add_vectors(const std::string& group_id_, + size_t n_, const float* vectors_, IDNumbers& vector_ids_); + private: - IDNumbers&& add_vectors_no_lock(const std::string& group_id_, - size_t n, - const float* vectors); + Status add_vectors_no_lock(const std::string& group_id_, + size_t n_, const float* vectors_, IDNumbers& vector_ids_); typedef std::map MemMap; MemMap _memMap; + std::shared_ptr _pMeta; }; // MemManager diff --git a/cpp/src/db/options.h b/cpp/src/db/options.h index 7da3e3ce8be3ee870434ce3e4d2d949c9178ebbe..cd0ceb89a795c83e1ec8518327c73abb3b7da398 100644 --- a/cpp/src/db/options.h +++ b/cpp/src/db/options.h @@ -9,6 +9,7 @@ namespace vecwise { namespace engine { class MetaOptions; +class Env; struct Options { uint16_t memory_sync_interval = 10; diff --git a/cpp/src/db/status.h b/cpp/src/db/status.h index 3133c649242f9d540c9ec6837425f64adc788ca1..85ac57c1c85473288bd2a7dc1aadc4d79dc41e05 100644 --- a/cpp/src/db/status.h +++ b/cpp/src/db/status.h @@ -17,14 +17,20 @@ public: Status& operator=(const Status& rhs_) noexcept; static Status OK() { return Status(); } + static Status NotFound(const std::string& msg_, const std::string& msg2_="") { + return Status(kNotFound, msg_, msg2_); + } bool ok() const { return _state == nullptr; } + bool IsNotFound() const { return code() == kNotFound; } + private: const char* _state; enum Code { kOK = 0, + kNotFound, }; Code code() const { diff --git a/cpp/src/db/types.h b/cpp/src/db/types.h new file mode 100644 index 0000000000000000000000000000000000000000..185fb73641b138d7cd67000986921b01a3090db1 --- /dev/null +++ b/cpp/src/db/types.h @@ -0,0 +1,19 @@ +#ifndef VECENGINE_TYPES_H_ +#define VECENGINE_TYPES_H_ + +#include + +namespace zilliz { +namespace vecwise { +namespace engine { + +#define uint64_t IDNumber; +#define IDNumber* IDNumberPtr; +#define std::vector IDNumbers; + + +} // namespace engine +} // namespace vecwise +} // namespace zilliz + +#endif // VECENGINE_TYPES_H_