diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index e3c9407d861038dcc690078da36b57a2d96b583c..88f1df71223f16668ef4ae8fb73f98e3143e5caf 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -1,14 +1,12 @@ -#include #include #include #include -#include -#include #include #include "MemManager.h" #include "Meta.h" +#include "FaissSerializer.h" namespace zilliz { @@ -21,43 +19,36 @@ MemVectors::MemVectors(const std::shared_ptr& meta_ptr, options_(options), schema_(schema), _pIdGenerator(new SimpleIDGenerator()), - pIndex_(faiss::index_factory(schema_.dimension, "IDMap,Flat")) { + pSerializer_(new FaissSerializer(schema_.dimension, schema_.location)) { } void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { _pIdGenerator->getNextIDNumbers(n_, vector_ids_); - pIndex_->add_with_ids(n_, vectors_, &vector_ids_[0]); + pSerializer_->AddWithIds(n_, vectors_, vector_ids_.data()); for(auto i=0 ; intotal; + return pSerializer_->Count(); } size_t MemVectors::approximate_size() const { - return total() * schema_.dimension; + return pSerializer_->Size(); } Status MemVectors::serialize(std::string& group_id) { - /* std::stringstream ss; */ - /* ss << "/tmp/test/" << _pIdGenerator->getNextIDNumber(); */ - /* faiss::write_index(pIndex_, ss.str().c_str()); */ - /* std::cout << pIndex_->ntotal << std::endl; */ - /* std::cout << _file_location << std::endl; */ - /* faiss::write_index(pIndex_, _file_location.c_str()); */ group_id = schema_.group_id; auto rows = approximate_size(); - write_index(pIndex_.get(), schema_.location.c_str()); + pSerializer_->Serialize(); schema_.rows = rows; schema_.file_type = (rows >= options_.index_trigger_size) ? meta::GroupFileSchema::TO_INDEX : meta::GroupFileSchema::RAW; auto status = pMeta_->update_group_file(schema_); - zilliz::vecwise::cache::CpuCacheMgr::GetInstance( - )->InsertItem(schema_.location, std::make_shared(pIndex_)); + pSerializer_->Cache(); return status; } diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 077e045286627e9200d725cc35b5773e4049684f..e29b09817c30ac1a8c702c254c7d84968e52da93 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -10,10 +10,6 @@ #include "Status.h" #include "Meta.h" -namespace faiss { - class Index; -} - namespace zilliz { namespace vecwise { @@ -23,6 +19,8 @@ namespace meta { class Meta; } +class Serializer; + class MemVectors { public: explicit MemVectors(const std::shared_ptr&, @@ -49,7 +47,7 @@ private: Options options_; meta::GroupFileSchema schema_; IDGenerator* _pIdGenerator; - std::shared_ptr pIndex_; + std::shared_ptr pSerializer_; }; // MemVectors