提交 79ce07c1 编写于 作者: X Xu Peng

feat(cpp/db): update for memory and meta operations


Former-commit-id: 69a8c7b6b9656532d4908013f8158d848481a8e6
上级 41f85d34
......@@ -25,6 +25,9 @@ public:
const int date_delta_,
GroupFilesSchema& group_files_info_) = 0;
virtual Status add_vectors(const std::string& group_id_,
size_t n, const float* vectors) = 0;
DB() = default;
DB(const DB&) = delete;
DB& operator=(const DB&) = delete;
......
......@@ -9,7 +9,8 @@ DBImpl::DBImpl(const Options& options_, const std::string& name_)
_options(options_),
_bg_work_finish_signal(_mutex),
_bg_compaction_scheduled(false),
_pMeta(new DBMetaImpl(*(_options.pMetaOptions))) {
_pMeta(new DBMetaImpl(*(_options.pMetaOptions))),
_pMemMgr(new MemManager(_pMeta)) {
}
Status DBImpl::add_group(const GroupOptions& options_,
......@@ -36,6 +37,11 @@ Status DBImpl::get_group_files(const std::string& group_id_,
}
Status DBImpl::add_vectors(const std::string& group_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) {
return _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_);
}
void DBImpl::try_schedule_compaction() {
if (_bg_compaction_scheduled) return;
if (!_bg_error.ok()) return;
......
......@@ -6,6 +6,7 @@
#include <memory>
#include "db.h"
#include "memvectors.h"
#include "types.h"
namespace zilliz {
namespace vecwise {
......@@ -27,11 +28,15 @@ public:
const int date_delta_,
GroupFilesSchema& group_files_info_) override;
void try_schedule_compaction();
virtual Status add_vectors(const std::string& group_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) override;
virtual ~DBImpl();
private:
void try_schedule_compaction();
static void BGWork(void* db);
void background_call();
void background_compaction();
......@@ -45,8 +50,8 @@ private:
bool _bg_compaction_scheduled;
Status _bg_error;
MemManager _memMgr;
std::shared_ptr<Meta> _pMeta;
std::shared_ptr<MemManager> _pMemMgr;
}; // DBImpl
......
......@@ -11,6 +11,7 @@ struct GroupSchema {
size_t files_cnt = 0;
uint16_t dimension;
std::string location = "";
std::string next_file_location = "";
}; // GroupSchema
......
......@@ -2,15 +2,12 @@
#define UTILS_ID_GENERATORS_H_
#include <vector>
#include "types.h"
namespace zilliz {
namespace vecwise {
namespace engine {
#define uint64_t IDNumber;
#define IDNumber* IDNumberPtr;
#define std::vector<IDNumber> IDNumbers;
class IDGenerator {
public:
virtual IDNumber getNextIDNumber() = 0;
......
......@@ -3,6 +3,7 @@
#include <index_io.h>
#include "memvectors.h"
#include "db_meta.h"
namespace vecengine {
......@@ -15,10 +16,9 @@ MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) :
_pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) {
}
IDNumbers&& MemVectors::add(size_t n, const float* vectors) {
IDNumbers&& ids = _pIdGenerator->getNextIDNumbers(n);
_pIdMapIndex->add_with_ids(n, vectors, pIds, &ids[0]);
return ids;
void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
vector_ids_ = _pIdGenerator->getNextIDNumbers(n_);
_pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]);
}
size_t MemVectors::total() const {
......@@ -57,27 +57,33 @@ MemVectors* MemManager::get_mem_by_group(const std::string& group_id_) {
if memIt != _memMap.end() {
return &(memIt->second);
}
// PXU TODO:
// 1. Read Group meta info
// 2. Initalize MemVectors base meta info
return nullptr;
/* GroupMetaInfo info; */
/* bool succ = env->getGroupMeta(group_id, &info); */
/* if (!succ) { */
/* return nullptr; */
/* } */
/* _memMap[group_id] = MemVectors(info.dimension, info.next_file_location); */
/* return &(_memMap[group_id]); */
GroupSchema group_info;
Status status = _pMeta->get_group(group_id_, group_info);
if (!status.ok()) {
return nullptr;
}
_memMap[group_id] = MemVectors(group_info.dimension, group_info.next_file_location);
return &(_memMap[group_id]);
}
Status MemManager::add_vectors(const std::string& group_id_,
size_t n_,
const float* vectors_,
IDNumbers& vector_ids_) {
// PXU TODO
return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_);
}
IDNumbers&& MemManager::add_vectors_no_lock(const std::string& group_id_,
Status MemManager::add_vectors_no_lock(const std::string& group_id_,
size_t n,
const float* vectors) {
auto mem = get_group_mem(group_id_);
const float* vectors,
IDNumbers& vector_ids_) {
auto mem = get_mem_by_group(group_id_);
if (mem == nullptr) {
return IDNumbers();
return Status::NotFound("Group " + group_id_ " not found!");
}
return mem->add(n, vectors);
return mem->add(n, vectors, vector_ids_);
}
......
......@@ -4,6 +4,7 @@
#include <map>
#include <string>
#include "id_generators.h"
#include "status.h"
class faiss::IndexIDMap;
class faiss::Index;
......@@ -37,19 +38,24 @@ private:
}; // MemVectors
class Meta;
class MemManager {
public:
MemManager() = default;
MemManager(const std::shared_ptr<Meta>& meta_) : _pMeta(meta_) {}
MemVectors* get_mem_by_group(const std::string& group_id_);
Status add_vectors(const std::string& group_id_,
size_t n_, const float* vectors_, IDNumbers& vector_ids_);
private:
IDNumbers&& add_vectors_no_lock(const std::string& group_id_,
size_t n,
const float* vectors);
Status add_vectors_no_lock(const std::string& group_id_,
size_t n_, const float* vectors_, IDNumbers& vector_ids_);
typedef std::map<std::string, MemVectors> MemMap;
MemMap _memMap;
std::shared_ptr<Meta> _pMeta;
}; // MemManager
......
......@@ -9,6 +9,7 @@ namespace vecwise {
namespace engine {
class MetaOptions;
class Env;
struct Options {
uint16_t memory_sync_interval = 10;
......
......@@ -17,14 +17,20 @@ public:
Status& operator=(const Status& rhs_) noexcept;
static Status OK() { return Status(); }
static Status NotFound(const std::string& msg_, const std::string& msg2_="") {
return Status(kNotFound, msg_, msg2_);
}
bool ok() const { return _state == nullptr; }
bool IsNotFound() const { return code() == kNotFound; }
private:
const char* _state;
enum Code {
kOK = 0,
kNotFound,
};
Code code() const {
......
#ifndef VECENGINE_TYPES_H_
#define VECENGINE_TYPES_H_
#include <vector>
namespace zilliz {
namespace vecwise {
namespace engine {
#define uint64_t IDNumber;
#define IDNumber* IDNumberPtr;
#define std::vector<IDNumber> IDNumbers;
} // namespace engine
} // namespace vecwise
} // namespace zilliz
#endif // VECENGINE_TYPES_H_
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册