提交 e94499cd 编写于 作者: X Xu Peng

refactor(db): change groupschema and groupfileschema to table*


Former-commit-id: fbc69151149faa8ed32547bcdd1a9935dac34070
上级 aa60c628
...@@ -21,8 +21,8 @@ class DB { ...@@ -21,8 +21,8 @@ class DB {
public: public:
static void Open(const Options& options, DB** dbptr); static void Open(const Options& options, DB** dbptr);
virtual Status add_group(meta::GroupSchema& group_info_) = 0; virtual Status add_group(meta::TableSchema& group_info_) = 0;
virtual Status get_group(meta::GroupSchema& group_info_) = 0; virtual Status get_group(meta::TableSchema& group_info_) = 0;
virtual Status delete_vectors(const std::string& group_id, virtual Status delete_vectors(const std::string& group_id,
const meta::DatesT& dates) = 0; const meta::DatesT& dates) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
......
...@@ -35,12 +35,12 @@ DBImpl<EngineT>::DBImpl(const Options& options) ...@@ -35,12 +35,12 @@ DBImpl<EngineT>::DBImpl(const Options& options)
} }
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::add_group(meta::GroupSchema& group_info) { Status DBImpl<EngineT>::add_group(meta::TableSchema& group_info) {
return _pMeta->add_group(group_info); return _pMeta->add_group(group_info);
} }
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::get_group(meta::GroupSchema& group_info) { Status DBImpl<EngineT>::get_group(meta::TableSchema& group_info) {
return _pMeta->get_group(group_info); return _pMeta->get_group(group_info);
} }
...@@ -93,7 +93,7 @@ Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq, ...@@ -93,7 +93,7 @@ Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq,
meta::GroupFilesSchema raw_files; meta::GroupFilesSchema raw_files;
for (auto &day_files : files) { for (auto &day_files : files) {
for (auto &file : day_files.second) { for (auto &file : day_files.second) {
file.file_type == meta::GroupFileSchema::INDEX ? file.file_type == meta::TableFileSchema::INDEX ?
index_files.push_back(file) : raw_files.push_back(file); index_files.push_back(file) : raw_files.push_back(file);
} }
} }
...@@ -259,7 +259,7 @@ void DBImpl<EngineT>::background_call() { ...@@ -259,7 +259,7 @@ void DBImpl<EngineT>::background_call() {
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::DateT& date, Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::DateT& date,
const meta::GroupFilesSchema& files) { const meta::GroupFilesSchema& files) {
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group_id; group_file.group_id = group_id;
group_file.date = date; group_file.date = date;
Status status = _pMeta->add_group_file(group_file); Status status = _pMeta->add_group_file(group_file);
...@@ -277,7 +277,7 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat ...@@ -277,7 +277,7 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
for (auto& file : files) { for (auto& file : files) {
index.Merge(file.location); index.Merge(file.location);
auto file_schema = file; auto file_schema = file;
file_schema.file_type = meta::GroupFileSchema::TO_DELETE; file_schema.file_type = meta::TableFileSchema::TO_DELETE;
updated.push_back(file_schema); updated.push_back(file_schema);
LOG(DEBUG) << "Merging file " << file_schema.file_id; LOG(DEBUG) << "Merging file " << file_schema.file_id;
index_size = index.Size(); index_size = index.Size();
...@@ -288,9 +288,9 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat ...@@ -288,9 +288,9 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
index.Serialize(); index.Serialize();
if (index_size >= _options.index_trigger_size) { if (index_size >= _options.index_trigger_size) {
group_file.file_type = meta::GroupFileSchema::TO_INDEX; group_file.file_type = meta::TableFileSchema::TO_INDEX;
} else { } else {
group_file.file_type = meta::GroupFileSchema::RAW; group_file.file_type = meta::TableFileSchema::RAW;
} }
group_file.size = index_size; group_file.size = index_size;
updated.push_back(group_file); updated.push_back(group_file);
...@@ -336,8 +336,8 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) { ...@@ -336,8 +336,8 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
} }
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) { Status DBImpl<EngineT>::build_index(const meta::TableFileSchema& file) {
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = file.group_id; group_file.group_id = file.group_id;
group_file.date = file.date; group_file.date = file.date;
Status status = _pMeta->add_group_file(group_file); Status status = _pMeta->add_group_file(group_file);
...@@ -350,11 +350,11 @@ Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) { ...@@ -350,11 +350,11 @@ Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) {
to_index.Load(); to_index.Load();
auto index = to_index.BuildIndex(group_file.location); auto index = to_index.BuildIndex(group_file.location);
group_file.file_type = meta::GroupFileSchema::INDEX; group_file.file_type = meta::TableFileSchema::INDEX;
group_file.size = index->Size(); group_file.size = index->Size();
auto to_remove = file; auto to_remove = file;
to_remove.file_type = meta::GroupFileSchema::TO_DELETE; to_remove.file_type = meta::TableFileSchema::TO_DELETE;
meta::GroupFilesSchema update_files = {to_remove, group_file}; meta::GroupFilesSchema update_files = {to_remove, group_file};
_pMeta->update_files(update_files); _pMeta->update_files(update_files);
......
...@@ -33,8 +33,8 @@ public: ...@@ -33,8 +33,8 @@ public:
DBImpl(const Options& options); DBImpl(const Options& options);
virtual Status add_group(meta::GroupSchema& group_info) override; virtual Status add_group(meta::TableSchema& group_info) override;
virtual Status get_group(meta::GroupSchema& group_info) override; virtual Status get_group(meta::TableSchema& group_info) override;
virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override; virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
...@@ -62,7 +62,7 @@ public: ...@@ -62,7 +62,7 @@ public:
private: private:
void background_build_index(); void background_build_index();
Status build_index(const meta::GroupFileSchema&); Status build_index(const meta::TableFileSchema&);
Status try_build_index(); Status try_build_index();
Status merge_files(const std::string& group_id, Status merge_files(const std::string& group_id,
const meta::DateT& date, const meta::DateT& date,
......
此差异已折叠。
...@@ -19,11 +19,11 @@ class DBMetaImpl : public Meta { ...@@ -19,11 +19,11 @@ class DBMetaImpl : public Meta {
public: public:
DBMetaImpl(const DBMetaOptions& options_); DBMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info) override; virtual Status add_group(TableSchema& group_info) override;
virtual Status get_group(GroupSchema& group_info_) override; virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override; virtual Status add_group_file(TableFileSchema& group_file_info) override;
virtual Status delete_group_partitions(const std::string& group_id, virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) override; const meta::DatesT& dates) override;
...@@ -32,8 +32,8 @@ public: ...@@ -32,8 +32,8 @@ public:
bool& has_or_not_) override; bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_, virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) override; TableFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override; virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_, virtual Status get_group_files(const std::string& group_id_,
const int date_delta_, const int date_delta_,
...@@ -68,10 +68,10 @@ private: ...@@ -68,10 +68,10 @@ private:
Status NextFileId(std::string& file_id); Status NextFileId(std::string& file_id);
Status NextGroupId(std::string& group_id); Status NextGroupId(std::string& group_id);
Status discard_files_of_size(long to_discard_size); Status discard_files_of_size(long to_discard_size);
Status get_group_no_lock(GroupSchema& group_info); Status get_group_no_lock(TableSchema& group_info);
std::string GetGroupPath(const std::string& group_id); std::string GetGroupPath(const std::string& group_id);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
void GetGroupFilePath(GroupFileSchema& group_file); void GetGroupFilePath(TableFileSchema& group_file);
Status initialize(); Status initialize();
const DBMetaOptions _options; const DBMetaOptions _options;
......
...@@ -38,8 +38,8 @@ std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id ...@@ -38,8 +38,8 @@ std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id
} }
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type) { TableFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == GroupFileSchema::RAW) ? ".raw" : ".index"; std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index";
SimpleIDGenerator g; SimpleIDGenerator g;
std::stringstream ss; std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
...@@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) { ...@@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(group_id)); return GetGroupMetaPathByGroupPath(GetGroupPath(group_id));
} }
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info) { Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) {
boost::property_tree::ptree ptree; boost::property_tree::ptree ptree;
boost::property_tree::read_json(path, ptree); boost::property_tree::read_json(path, ptree);
auto files_cnt = ptree.get_child("files_cnt").data(); auto files_cnt = ptree.get_child("files_cnt").data();
...@@ -71,7 +71,7 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchem ...@@ -71,7 +71,7 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchem
} }
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info) { Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) {
group_info.group_id = group_id; group_info.group_id = group_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info); return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info);
} }
...@@ -90,7 +90,7 @@ Status LocalMetaImpl::initialize() { ...@@ -90,7 +90,7 @@ Status LocalMetaImpl::initialize() {
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::add_group(GroupSchema& group_info) { Status LocalMetaImpl::add_group(TableSchema& group_info) {
std::string real_gid; std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber(); size_t id = SimpleIDGenerator().getNextIDNumber();
if (group_info.group_id == "") { if (group_info.group_id == "") {
...@@ -123,7 +123,7 @@ Status LocalMetaImpl::add_group(GroupSchema& group_info) { ...@@ -123,7 +123,7 @@ Status LocalMetaImpl::add_group(GroupSchema& group_info) {
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::get_group(GroupSchema& group_info) { Status LocalMetaImpl::get_group(TableSchema& group_info) {
bool group_exist; bool group_exist;
has_group(group_info.group_id, group_exist); has_group(group_info.group_id, group_exist);
if (!group_exist) { if (!group_exist) {
...@@ -138,8 +138,8 @@ Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { ...@@ -138,8 +138,8 @@ Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::add_group_file(GroupFileSchema& group_file_info) { Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
GroupSchema group_info; TableSchema group_info;
/* auto status = get_group(group_info); */ /* auto status = get_group(group_info); */
/* if (!status.ok()) { */ /* if (!status.ok()) { */
/* return status; */ /* return status; */
...@@ -159,7 +159,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) { ...@@ -159,7 +159,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
boost::filesystem::directory_iterator end_itr; boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
auto group_path = itr->path().string(); auto group_path = itr->path().string();
GroupSchema group_info; TableSchema group_info;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info);
for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
auto partition_path = innerItr->path().string(); auto partition_path = innerItr->path().string();
...@@ -169,7 +169,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) { ...@@ -169,7 +169,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
if (suffix == "index") continue; if (suffix == "index") continue;
if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue; if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue;
std::cout << "[About to index] " << location << std::endl; std::cout << "[About to index] " << location << std::endl;
GroupFileSchema f; TableFileSchema f;
f.location = location; f.location = location;
/* f.group_id = group_id; */ /* f.group_id = group_id; */
f.dimension = group_info.dimension; f.dimension = group_info.dimension;
...@@ -188,7 +188,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, ...@@ -188,7 +188,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* boost::filesystem::directory_iterator end_itr; */ /* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */ /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */ /* auto group_path = itr->path().string(); */
/* GroupSchema group_info; */ /* TableSchema group_info; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */ /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */ /* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */ /* auto partition_path = innerItr->path().string(); */
...@@ -198,7 +198,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, ...@@ -198,7 +198,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* if (suffix == "index") continue; */ /* if (suffix == "index") continue; */
/* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */ /* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
/* std::cout << "[About to index] " << location << std::endl; */ /* std::cout << "[About to index] " << location << std::endl; */
/* GroupFileSchema f; */ /* TableFileSchema f; */
/* f.location = location; */ /* f.location = location; */
/* f.group_id = group_id; */ /* f.group_id = group_id; */
/* f.dimension = group_info.dimension; */ /* f.dimension = group_info.dimension; */
...@@ -219,7 +219,7 @@ Status LocalMetaImpl::has_group_file(const std::string& group_id_, ...@@ -219,7 +219,7 @@ Status LocalMetaImpl::has_group_file(const std::string& group_id_,
Status LocalMetaImpl::get_group_file(const std::string& group_id_, Status LocalMetaImpl::get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) { TableFileSchema& group_file_info_) {
//PXU TODO //PXU TODO
return Status::OK(); return Status::OK();
} }
...@@ -231,7 +231,7 @@ Status LocalMetaImpl::get_group_files(const std::string& group_id_, ...@@ -231,7 +231,7 @@ Status LocalMetaImpl::get_group_files(const std::string& group_id_,
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::update_group_file(GroupFileSchema& group_file_) { Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) {
//PXU TODO //PXU TODO
return Status::OK(); return Status::OK();
} }
......
...@@ -18,11 +18,11 @@ public: ...@@ -18,11 +18,11 @@ public:
const size_t INDEX_TRIGGER_SIZE = 1024*1024*500; const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
LocalMetaImpl(const DBMetaOptions& options_); LocalMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info_) override; virtual Status add_group(TableSchema& group_info_) override;
virtual Status get_group(GroupSchema& group_info_) override; virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override; virtual Status add_group_file(TableFileSchema& group_file_info) override;
/* virtual Status delete_group_partitions(const std::string& group_id, */ /* virtual Status delete_group_partitions(const std::string& group_id, */
/* const meta::DatesT& dates) override; */ /* const meta::DatesT& dates) override; */
...@@ -31,8 +31,8 @@ public: ...@@ -31,8 +31,8 @@ public:
bool& has_or_not_) override; bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_, virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) override; TableFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override; virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_, virtual Status get_group_files(const std::string& group_id_,
const int date_delta_, const int date_delta_,
...@@ -59,16 +59,16 @@ public: ...@@ -59,16 +59,16 @@ public:
private: private:
Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info); Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path); std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info); Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info);
std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type); TableFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
std::string GetGroupPath(const std::string& group_id); std::string GetGroupPath(const std::string& group_id);
std::string GetGroupMetaPath(const std::string& group_id); std::string GetGroupMetaPath(const std::string& group_id);
Status CreateGroupMeta(const GroupSchema& group_schema); Status CreateGroupMeta(const TableSchema& group_schema);
long GetFileSize(const std::string& filename); long GetFileSize(const std::string& filename);
Status initialize(); Status initialize();
......
...@@ -22,7 +22,7 @@ namespace engine { ...@@ -22,7 +22,7 @@ namespace engine {
template<typename EngineT> template<typename EngineT>
MemVectors<EngineT>::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr, MemVectors<EngineT>::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
const meta::GroupFileSchema& schema, const Options& options) const meta::TableFileSchema& schema, const Options& options)
: pMeta_(meta_ptr), : pMeta_(meta_ptr),
options_(options), options_(options),
schema_(schema), schema_(schema),
...@@ -53,11 +53,11 @@ Status MemVectors<EngineT>::serialize(std::string& group_id) { ...@@ -53,11 +53,11 @@ Status MemVectors<EngineT>::serialize(std::string& group_id) {
pEE_->Serialize(); pEE_->Serialize();
schema_.size = size; schema_.size = size;
schema_.file_type = (size >= options_.index_trigger_size) ? schema_.file_type = (size >= options_.index_trigger_size) ?
meta::GroupFileSchema::TO_INDEX : meta::GroupFileSchema::RAW; meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
auto status = pMeta_->update_group_file(schema_); auto status = pMeta_->update_group_file(schema_);
LOG(DEBUG) << "New " << ((schema_.file_type == meta::GroupFileSchema::RAW) ? "raw" : "to_index") LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M"; << " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M";
pEE_->Cache(); pEE_->Cache();
...@@ -85,7 +85,7 @@ typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::get_mem_by_grou ...@@ -85,7 +85,7 @@ typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::get_mem_by_grou
return memIt->second; return memIt->second;
} }
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group_id; group_file.group_id = group_id;
auto status = _pMeta->add_group_file(group_file); auto status = _pMeta->add_group_file(group_file);
if (!status.ok()) { if (!status.ok()) {
......
...@@ -31,7 +31,7 @@ public: ...@@ -31,7 +31,7 @@ public:
typedef std::shared_ptr<MemVectors<EngineT>> Ptr; typedef std::shared_ptr<MemVectors<EngineT>> Ptr;
explicit MemVectors(const std::shared_ptr<meta::Meta>&, explicit MemVectors(const std::shared_ptr<meta::Meta>&,
const meta::GroupFileSchema&, const Options&); const meta::TableFileSchema&, const Options&);
void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_);
...@@ -52,7 +52,7 @@ private: ...@@ -52,7 +52,7 @@ private:
MetaPtr pMeta_; MetaPtr pMeta_;
Options options_; Options options_;
meta::GroupFileSchema schema_; meta::TableFileSchema schema_;
IDGenerator* _pIdGenerator; IDGenerator* _pIdGenerator;
EnginePtr pEE_; EnginePtr pEE_;
......
...@@ -20,13 +20,13 @@ namespace meta { ...@@ -20,13 +20,13 @@ namespace meta {
class Meta { class Meta {
public: public:
typedef std::shared_ptr<Meta> Ptr; using Ptr = std::shared_ptr<Meta>;
virtual Status add_group(GroupSchema& group_info) = 0; virtual Status add_group(TableSchema& group_info) = 0;
virtual Status get_group(GroupSchema& group_info) = 0; virtual Status get_group(TableSchema& group_info) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status add_group_file(GroupFileSchema& group_file_info) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0;
virtual Status delete_group_partitions(const std::string& group_id, virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) = 0; const meta::DatesT& dates) = 0;
...@@ -35,8 +35,8 @@ public: ...@@ -35,8 +35,8 @@ public:
bool& has_or_not_) = 0; bool& has_or_not_) = 0;
virtual Status get_group_file(const std::string& group_id_, virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) = 0; TableFileSchema& group_file_info_) = 0;
virtual Status update_group_file(GroupFileSchema& group_file_) = 0; virtual Status update_group_file(TableFileSchema& group_file_) = 0;
virtual Status get_group_files(const std::string& group_id_, virtual Status get_group_files(const std::string& group_id_,
const int date_delta_, const int date_delta_,
......
...@@ -18,16 +18,16 @@ typedef int DateT; ...@@ -18,16 +18,16 @@ typedef int DateT;
const DateT EmptyDate = -1; const DateT EmptyDate = -1;
typedef std::vector<DateT> DatesT; typedef std::vector<DateT> DatesT;
struct GroupSchema { struct TableSchema {
size_t id; size_t id;
std::string group_id; std::string group_id;
size_t files_cnt = 0; size_t files_cnt = 0;
uint16_t dimension; uint16_t dimension;
std::string location = ""; std::string location;
long created_on; long created_on;
}; // GroupSchema }; // TableSchema
struct GroupFileSchema { struct TableFileSchema {
typedef enum { typedef enum {
NEW, NEW,
RAW, RAW,
...@@ -43,12 +43,12 @@ struct GroupFileSchema { ...@@ -43,12 +43,12 @@ struct GroupFileSchema {
size_t size; size_t size;
DateT date = EmptyDate; DateT date = EmptyDate;
uint16_t dimension; uint16_t dimension;
std::string location = ""; std::string location;
long updated_time; long updated_time;
long created_on; long created_on;
}; // GroupFileSchema }; // TableFileSchema
typedef std::vector<GroupFileSchema> GroupFilesSchema; typedef std::vector<TableFileSchema> GroupFilesSchema;
typedef std::map<DateT, GroupFilesSchema> DatePartionedGroupFilesSchema; typedef std::map<DateT, GroupFilesSchema> DatePartionedGroupFilesSchema;
} // namespace meta } // namespace meta
......
...@@ -66,12 +66,12 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { ...@@ -66,12 +66,12 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
static const int group_dim = 256; static const int group_dim = 256;
long size; long size;
engine::meta::GroupSchema group_info; engine::meta::TableSchema group_info;
group_info.dimension = group_dim; group_info.dimension = group_dim;
group_info.group_id = group_name; group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info); engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get; engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name; group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get); stat = db_->get_group(group_info_get);
ASSERT_STATS(stat); ASSERT_STATS(stat);
...@@ -111,12 +111,12 @@ TEST_F(DBTest, DB_TEST) { ...@@ -111,12 +111,12 @@ TEST_F(DBTest, DB_TEST) {
static const std::string group_name = "test_group"; static const std::string group_name = "test_group";
static const int group_dim = 256; static const int group_dim = 256;
engine::meta::GroupSchema group_info; engine::meta::TableSchema group_info;
group_info.dimension = group_dim; group_info.dimension = group_dim;
group_info.group_id = group_name; group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info); engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get; engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name; group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get); stat = db_->get_group(group_info_get);
ASSERT_STATS(stat); ASSERT_STATS(stat);
...@@ -197,12 +197,12 @@ TEST_F(DBTest, SEARCH_TEST) { ...@@ -197,12 +197,12 @@ TEST_F(DBTest, SEARCH_TEST) {
static const std::string group_name = "test_group"; static const std::string group_name = "test_group";
static const int group_dim = 256; static const int group_dim = 256;
engine::meta::GroupSchema group_info; engine::meta::TableSchema group_info;
group_info.dimension = group_dim; group_info.dimension = group_dim;
group_info.group_id = group_name; group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info); engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get; engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name; group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get); stat = db_->get_group(group_info_get);
ASSERT_STATS(stat); ASSERT_STATS(stat);
......
...@@ -20,7 +20,7 @@ using namespace zilliz::vecwise::engine; ...@@ -20,7 +20,7 @@ using namespace zilliz::vecwise::engine;
TEST_F(MetaTest, GROUP_TEST) { TEST_F(MetaTest, GROUP_TEST) {
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl_->add_group(group); auto status = impl_->add_group(group);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
...@@ -44,19 +44,19 @@ TEST_F(MetaTest, GROUP_TEST) { ...@@ -44,19 +44,19 @@ TEST_F(MetaTest, GROUP_TEST) {
TEST_F(MetaTest, GROUP_FILE_TEST) { TEST_F(MetaTest, GROUP_FILE_TEST) {
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl_->add_group(group); auto status = impl_->add_group(group);
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.file_type, meta::GroupFileSchema::NEW); ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW);
auto file_id = group_file.file_id; auto file_id = group_file.file_id;
auto new_file_type = meta::GroupFileSchema::INDEX; auto new_file_type = meta::TableFileSchema::INDEX;
group_file.file_type = new_file_type; group_file.file_type = new_file_type;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
...@@ -79,7 +79,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { ...@@ -79,7 +79,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2)); ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(group_file.file_type == meta::GroupFileSchema::TO_DELETE); ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
dates.clear(); dates.clear();
dates.push_back(group_file.date); dates.push_back(group_file.date);
...@@ -87,7 +87,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { ...@@ -87,7 +87,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_TRUE(group_file.file_type == meta::GroupFileSchema::TO_DELETE); ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
} }
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
...@@ -102,12 +102,12 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { ...@@ -102,12 +102,12 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
auto impl = meta::DBMetaImpl(options); auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl.add_group(group); auto status = impl.add_group(group);
meta::GroupFilesSchema files; meta::GroupFilesSchema files;
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
auto cnt = 100; auto cnt = 100;
...@@ -115,7 +115,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { ...@@ -115,7 +115,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
std::vector<int> days; std::vector<int> days;
for (auto i=0; i<cnt; ++i) { for (auto i=0; i<cnt; ++i) {
status = impl.add_group_file(group_file); status = impl.add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW; group_file.file_type = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2); int day = rand() % (days_num*2);
group_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000; group_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000;
status = impl.update_group_file(group_file); status = impl.update_group_file(group_file);
...@@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { ...@@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.get_group_file(file.group_id, file.file_id, file); status = impl.get_group_file(file.group_id, file.file_id, file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
if (days[i] < days_num) { if (days[i] < days_num) {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::NEW); ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
} else { } else {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::TO_DELETE); ASSERT_EQ(file.file_type, meta::TableFileSchema::TO_DELETE);
} }
i++; i++;
} }
...@@ -148,19 +148,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { ...@@ -148,19 +148,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
auto impl = meta::DBMetaImpl(options); auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl.add_group(group); auto status = impl.add_group(group);
meta::GroupFilesSchema files; meta::GroupFilesSchema files;
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
auto cnt = 10; auto cnt = 10;
auto each_size = 2UL; auto each_size = 2UL;
for (auto i=0; i<cnt; ++i) { for (auto i=0; i<cnt; ++i) {
status = impl.add_group_file(group_file); status = impl.add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW; group_file.file_type = meta::TableFileSchema::NEW;
group_file.size = each_size * meta::G; group_file.size = each_size * meta::G;
status = impl.update_group_file(group_file); status = impl.update_group_file(group_file);
files.push_back(group_file); files.push_back(group_file);
...@@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { ...@@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.get_group_file(file.group_id, file.file_id, file); status = impl.get_group_file(file.group_id, file.file_id, file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
if (i < 5) { if (i < 5) {
ASSERT_TRUE(file.file_type == meta::GroupFileSchema::TO_DELETE); ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
} else { } else {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::NEW); ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
} }
++i; ++i;
} }
...@@ -186,7 +186,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { ...@@ -186,7 +186,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MetaTest, GROUP_FILES_TEST) { TEST_F(MetaTest, GROUP_FILES_TEST) {
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl_->add_group(group); auto status = impl_->add_group(group);
...@@ -195,30 +195,30 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { ...@@ -195,30 +195,30 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
int to_index_files_cnt = 6; int to_index_files_cnt = 6;
int index_files_cnt = 7; int index_files_cnt = 7;
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
for (auto i=0; i<new_files_cnt; ++i) { for (auto i=0; i<new_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW; group_file.file_type = meta::TableFileSchema::NEW;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
for (auto i=0; i<raw_files_cnt; ++i) { for (auto i=0; i<raw_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::RAW; group_file.file_type = meta::TableFileSchema::RAW;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
for (auto i=0; i<to_index_files_cnt; ++i) { for (auto i=0; i<to_index_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::TO_INDEX; group_file.file_type = meta::TableFileSchema::TO_INDEX;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
for (auto i=0; i<index_files_cnt; ++i) { for (auto i=0; i<index_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::INDEX; group_file.file_type = meta::TableFileSchema::INDEX;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册