Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

从无法访问的项目Fork

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 f2bb20c9 编写于 作者: X Xu Peng
浏览文件
操作

refactor(db): change group_id to table_id 


Former-commit-id: bf0cec2552fcad95bf1b77cdfbdd423984e6f18f
上级 209d0e48
隐藏空白更改
内联 并排
Showing with 208 addition and 303 deletion
cpp/src/db/DB.h
浏览文件 @ f2bb20c9
......@@ -23,27 +23,27 @@ public:
virtual Status add_group(meta::TableSchema& group_info_) = 0;
virtual Status get_group(meta::TableSchema& group_info_) = 0;
virtual Status delete_vectors(const std::string& group_id,
virtual Status delete_vectors(const std::string& table_id,
const meta::DatesT& dates) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status get_group_files(const std::string& group_id_,
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0;
virtual Status get_group_files(const std::string& table_id_,
const int date_delta_,
meta::TableFilesSchema& group_files_info_) = 0;
virtual Status add_vectors(const std::string& group_id_,
virtual Status add_vectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status search(const std::string& table_id, size_t k, size_t nq,
const float* vectors, QueryResults& results) = 0;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status search(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status size(long& result) = 0;
virtual Status drop_all() = 0;
virtual Status count(const std::string& group_id, long& result) = 0;
virtual Status count(const std::string& table_id, long& result) = 0;
DB() = default;
DB(const DB&) = delete;
......
cpp/src/db/DBImpl.cpp
浏览文件 @ f2bb20c9
......@@ -45,46 +45,46 @@ Status DBImpl<EngineT>::get_group(meta::TableSchema& group_info) {
}
template<typename EngineT>
Status DBImpl<EngineT>::delete_vectors(const std::string& group_id,
Status DBImpl<EngineT>::delete_vectors(const std::string& table_id,
const meta::DatesT& dates) {
return _pMeta->delete_group_partitions(group_id, dates);
return _pMeta->delete_group_partitions(table_id, dates);
}
template<typename EngineT>
Status DBImpl<EngineT>::has_group(const std::string& group_id_, bool& has_or_not_) {
return _pMeta->has_group(group_id_, has_or_not_);
Status DBImpl<EngineT>::has_group(const std::string& table_id_, bool& has_or_not_) {
return _pMeta->has_group(table_id_, has_or_not_);
}
template<typename EngineT>
Status DBImpl<EngineT>::get_group_files(const std::string& group_id,
Status DBImpl<EngineT>::get_group_files(const std::string& table_id,
const int date_delta,
meta::TableFilesSchema& group_files_info) {
return _pMeta->get_group_files(group_id, date_delta, group_files_info);
return _pMeta->get_group_files(table_id, date_delta, group_files_info);
}
template<typename EngineT>
Status DBImpl<EngineT>::add_vectors(const std::string& group_id_,
Status DBImpl<EngineT>::add_vectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) {
Status status = _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_);
Status status = _pMemMgr->add_vectors(table_id_, n, vectors, vector_ids_);
if (!status.ok()) {
return status;
}
}
template<typename EngineT>
Status DBImpl<EngineT>::search(const std::string &group_id, size_t k, size_t nq,
Status DBImpl<EngineT>::search(const std::string &table_id, size_t k, size_t nq,
const float *vectors, QueryResults &results) {
meta::DatesT dates = {meta::Meta::GetDate()};
return search(group_id, k, nq, vectors, dates, results);
return search(table_id, k, nq, vectors, dates, results);
}
template<typename EngineT>
Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq,
Status DBImpl<EngineT>::search(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
meta::DatePartionedTableFilesSchema files;
auto status = _pMeta->files_to_search(group_id, dates, files);
auto status = _pMeta->files_to_search(table_id, dates, files);
if (!status.ok()) { return status; }
LOG(DEBUG) << "Search DateT Size=" << files.size();
......@@ -204,7 +204,7 @@ Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq,
}
if (results.empty()) {
return Status::NotFound("Group " + group_id + ", search result not found!");
return Status::NotFound("Group " + table_id + ", search result not found!");
}
return Status::OK();
}
......@@ -257,10 +257,10 @@ void DBImpl<EngineT>::background_call() {
template<typename EngineT>
Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::DateT& date,
Status DBImpl<EngineT>::merge_files(const std::string& table_id, const meta::DateT& date,
const meta::TableFilesSchema& files) {
meta::TableFileSchema group_file;
group_file.group_id = group_id;
group_file.table_id = table_id;
group_file.date = date;
Status status = _pMeta->add_group_file(group_file);
......@@ -304,9 +304,9 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
}
template<typename EngineT>
Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
Status DBImpl<EngineT>::background_merge_files(const std::string& table_id) {
meta::DatePartionedTableFilesSchema raw_files;
auto status = _pMeta->files_to_merge(group_id, raw_files);
auto status = _pMeta->files_to_merge(table_id, raw_files);
if (!status.ok()) {
return status;
}
......@@ -323,7 +323,7 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
continue;
}
has_merge = true;
merge_files(group_id, kv.first, kv.second);
merge_files(table_id, kv.first, kv.second);
}
_pMeta->archive_files();
......@@ -338,7 +338,7 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
template<typename EngineT>
Status DBImpl<EngineT>::build_index(const meta::TableFileSchema& file) {
meta::TableFileSchema group_file;
group_file.group_id = file.group_id;
group_file.table_id = file.table_id;
group_file.date = file.date;
Status status = _pMeta->add_group_file(group_file);
if (!status.ok()) {
......@@ -402,12 +402,12 @@ Status DBImpl<EngineT>::try_build_index() {
template<typename EngineT>
void DBImpl<EngineT>::background_compaction() {
std::vector<std::string> group_ids;
_pMemMgr->serialize(group_ids);
std::vector<std::string> table_ids;
_pMemMgr->serialize(table_ids);
Status status;
for (auto group_id : group_ids) {
status = background_merge_files(group_id);
for (auto table_id : table_ids) {
status = background_merge_files(table_id);
if (!status.ok()) {
_bg_error = status;
return;
......@@ -421,8 +421,8 @@ Status DBImpl<EngineT>::drop_all() {
}
template<typename EngineT>
Status DBImpl<EngineT>::count(const std::string& group_id, long& result) {
return _pMeta->count(group_id, result);
Status DBImpl<EngineT>::count(const std::string& table_id, long& result) {
return _pMeta->count(table_id, result);
}
template<typename EngineT>
......
cpp/src/db/DBImpl.h
浏览文件 @ f2bb20c9
......@@ -35,25 +35,25 @@ public:
virtual Status add_group(meta::TableSchema& group_info) override;
virtual Status get_group(meta::TableSchema& group_info) override;
virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
virtual Status get_group_files(const std::string& group_id_,
virtual Status get_group_files(const std::string& table_id_,
const int date_delta_,
meta::TableFilesSchema& group_files_info_) override;
virtual Status add_vectors(const std::string& group_id_,
virtual Status add_vectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) override;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status search(const std::string& table_id, size_t k, size_t nq,
const float* vectors, QueryResults& results) override;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status search(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) override;
virtual Status drop_all() override;
virtual Status count(const std::string& group_id, long& result) override;
virtual Status count(const std::string& table_id, long& result) override;
virtual Status size(long& result) override;
......@@ -64,10 +64,10 @@ private:
void background_build_index();
Status build_index(const meta::TableFileSchema&);
Status try_build_index();
Status merge_files(const std::string& group_id,
Status merge_files(const std::string& table_id,
const meta::DateT& date,
const meta::TableFilesSchema& files);
Status background_merge_files(const std::string& group_id);
Status background_merge_files(const std::string& table_id);
void try_schedule_compaction();
void start_timer_task(int interval_);
......
cpp/src/db/DBMetaImpl.cpp
浏览文件 @ f2bb20c9
......@@ -29,13 +29,13 @@ inline auto StoragePrototype(const std::string& path) {
return make_storage(path,
make_table("Group",
make_column("id", &TableSchema::id, primary_key()),
make_column("group_id", &TableSchema::group_id, unique()),
make_column("table_id", &TableSchema::table_id, unique()),
make_column("dimension", &TableSchema::dimension),
make_column("created_on", &TableSchema::created_on),
make_column("files_cnt", &TableSchema::files_cnt, default_value(0))),
make_table("GroupFile",
make_column("id", &TableFileSchema::id, primary_key()),
make_column("group_id", &TableFileSchema::group_id),
make_column("table_id", &TableFileSchema::table_id),
make_column("file_id", &TableFileSchema::file_id),
make_column("file_type", &TableFileSchema::file_type),
make_column("size", &TableFileSchema::size, default_value(0)),
......@@ -49,13 +49,13 @@ inline auto StoragePrototype(const std::string& path) {
using ConnectorT = decltype(StoragePrototype(""));
static std::unique_ptr<ConnectorT> ConnectorPtr;
std::string DBMetaImpl::GetGroupPath(const std::string& group_id) {
return _options.path + "/tables/" + group_id;
std::string DBMetaImpl::GetGroupPath(const std::string& table_id) {
return _options.path + "/tables/" + table_id;
}
std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) {
std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) {
std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date;
ss << GetGroupPath(table_id) << "/" << date;
return ss.str();
}
......@@ -64,16 +64,16 @@ void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) {
group_file.date = Meta::GetDate();
}
std::stringstream ss;
ss << GetGroupDatePartitionPath(group_file.group_id, group_file.date)
ss << GetGroupDatePartitionPath(group_file.table_id, group_file.date)
<< "/" << group_file.file_id;
group_file.location = ss.str();
}
Status DBMetaImpl::NextGroupId(std::string& group_id) {
Status DBMetaImpl::NextGroupId(std::string& table_id) {
std::stringstream ss;
SimpleIDGenerator g;
ss << g.getNextIDNumber();
group_id = ss.str();
table_id = ss.str();
return Status::OK();
}
......@@ -111,14 +111,14 @@ Status DBMetaImpl::initialize() {
}
// PXU TODO: Temp solution. Will fix later
Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
Status DBMetaImpl::delete_group_partitions(const std::string& table_id,
const meta::DatesT& dates) {
if (dates.size() == 0) {
return Status::OK();
}
TableSchema group_info;
group_info.group_id = group_id;
group_info.table_id = table_id;
auto status = get_group(group_info);
if (!status.ok()) {
return status;
......@@ -138,7 +138,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
),
where(
c(&TableFileSchema::group_id) == group_id and
c(&TableFileSchema::table_id) == table_id and
in(&TableFileSchema::date, dates)
));
} catch (std::exception & e) {
......@@ -149,8 +149,8 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
}
Status DBMetaImpl::add_group(TableSchema& group_info) {
if (group_info.group_id == "") {
NextGroupId(group_info.group_id);
if (group_info.table_id == "") {
NextGroupId(group_info.table_id);
}
group_info.files_cnt = 0;
group_info.id = -1;
......@@ -165,7 +165,7 @@ Status DBMetaImpl::add_group(TableSchema& group_info) {
}
}
auto group_path = GetGroupPath(group_info.group_id);
auto group_path = GetGroupPath(group_info.table_id);
if (!boost::filesystem::is_directory(group_path)) {
auto ret = boost::filesystem::create_directories(group_path);
......@@ -185,17 +185,17 @@ Status DBMetaImpl::get_group(TableSchema& group_info) {
Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) {
try {
auto groups = ConnectorPtr->select(columns(&TableSchema::id,
&TableSchema::group_id,
&TableSchema::table_id,
&TableSchema::files_cnt,
&TableSchema::dimension),
where(c(&TableSchema::group_id) == group_info.group_id));
where(c(&TableSchema::table_id) == group_info.table_id));
assert(groups.size() <= 1);
if (groups.size() == 1) {
group_info.id = std::get<0>(groups[0]);
group_info.files_cnt = std::get<2>(groups[0]);
group_info.dimension = std::get<3>(groups[0]);
} else {
return Status::NotFound("Group " + group_info.group_id + " not found");
return Status::NotFound("Group " + group_info.table_id + " not found");
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
......@@ -205,10 +205,10 @@ Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) {
return Status::OK();
}
Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
Status DBMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
try {
auto groups = ConnectorPtr->select(columns(&TableSchema::id),
where(c(&TableSchema::group_id) == group_id));
where(c(&TableSchema::table_id) == table_id));
assert(groups.size() <= 1);
if (groups.size() == 1) {
has_or_not = true;
......@@ -227,7 +227,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) {
group_file.date = Meta::GetDate();
}
TableSchema group_info;
group_info.group_id = group_file.group_id;
group_info.table_id = group_file.table_id;
auto status = get_group(group_info);
if (!status.ok()) {
return status;
......@@ -250,7 +250,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) {
}
}
auto partition_path = GetGroupDatePartitionPath(group_file.group_id, group_file.date);
auto partition_path = GetGroupDatePartitionPath(group_file.table_id, group_file.date);
if (!boost::filesystem::is_directory(partition_path)) {
auto ret = boost::filesystem::create_directory(partition_path);
......@@ -268,7 +268,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
......@@ -280,23 +280,23 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) {
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
GetGroupFilePath(group_file);
auto groupItr = groups.find(group_file.group_id);
auto groupItr = groups.find(group_file.table_id);
if (groupItr == groups.end()) {
TableSchema group_info;
group_info.group_id = group_file.group_id;
group_info.table_id = group_file.table_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
return status;
}
groups[group_file.group_id] = group_info;
groups[group_file.table_id] = group_info;
}
group_file.dimension = groups[group_file.group_id].dimension;
group_file.dimension = groups[group_file.table_id].dimension;
files.push_back(group_file);
}
} catch (std::exception & e) {
......@@ -307,7 +307,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) {
return Status::OK();
}
Status DBMetaImpl::files_to_search(const std::string &group_id,
Status DBMetaImpl::files_to_search(const std::string &table_id,
const DatesT& partition,
DatePartionedTableFilesSchema &files) {
files.clear();
......@@ -316,19 +316,19 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::group_id) == group_id and
where(c(&TableFileSchema::table_id) == table_id and
in(&TableFileSchema::date, dates) and
(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX)));
TableSchema group_info;
group_info.group_id = group_id;
group_info.table_id = table_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
return status;
......@@ -337,7 +337,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
......@@ -358,22 +358,22 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
return Status::OK();
}
Status DBMetaImpl::files_to_merge(const std::string& group_id,
Status DBMetaImpl::files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) {
files.clear();
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and
c(&TableFileSchema::group_id) == group_id));
c(&TableFileSchema::table_id) == table_id));
TableSchema group_info;
group_info.group_id = group_id;
group_info.table_id = table_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
return status;
......@@ -382,7 +382,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
......@@ -403,30 +403,30 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
return Status::OK();
}
Status DBMetaImpl::has_group_file(const std::string& group_id_,
Status DBMetaImpl::has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) {
//PXU TODO
return Status::OK();
}
Status DBMetaImpl::get_group_file(const std::string& group_id_,
Status DBMetaImpl::get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) {
try {
auto files = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_id) == file_id_ and
c(&TableFileSchema::group_id) == group_id_
c(&TableFileSchema::table_id) == table_id_
));
assert(files.size() <= 1);
if (files.size() == 1) {
group_file_info_.id = std::get<0>(files[0]);
group_file_info_.group_id = std::get<1>(files[0]);
group_file_info_.table_id = std::get<1>(files[0]);
group_file_info_.file_id = std::get<2>(files[0]);
group_file_info_.file_type = std::get<3>(files[0]);
group_file_info_.size = std::get<4>(files[0]);
......@@ -442,7 +442,7 @@ Status DBMetaImpl::get_group_file(const std::string& group_id_,
return Status::OK();
}
Status DBMetaImpl::get_group_files(const std::string& group_id_,
Status DBMetaImpl::get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) {
// PXU TODO
......@@ -591,7 +591,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
......@@ -604,7 +604,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
......@@ -627,7 +627,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
Status DBMetaImpl::cleanup() {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
......@@ -640,7 +640,7 @@ Status DBMetaImpl::cleanup() {
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
......@@ -660,7 +660,7 @@ Status DBMetaImpl::cleanup() {
return Status::OK();
}
Status DBMetaImpl::count(const std::string& group_id, long& result) {
Status DBMetaImpl::count(const std::string& table_id, long& result) {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
......@@ -668,10 +668,10 @@ Status DBMetaImpl::count(const std::string& group_id, long& result) {
where((c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW or
c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and
c(&TableFileSchema::group_id) == group_id));
c(&TableFileSchema::table_id) == table_id));
TableSchema group_info;
group_info.group_id = group_id;
group_info.table_id = table_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
return status;
......
cpp/src/db/DBMetaImpl.h
浏览文件 @ f2bb20c9
......@@ -21,30 +21,30 @@ public:
virtual Status add_group(TableSchema& group_info) override;
virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
virtual Status delete_group_partitions(const std::string& group_id,
virtual Status delete_group_partitions(const std::string& table_id,
const meta::DatesT& dates) override;
virtual Status has_group_file(const std::string& group_id_,
virtual Status has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_,
virtual Status get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) override;
virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_,
virtual Status get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) override;
virtual Status update_files(TableFilesSchema& files) override;
virtual Status files_to_merge(const std::string& group_id,
virtual Status files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) override;
virtual Status files_to_search(const std::string& group_id,
virtual Status files_to_search(const std::string& table_id,
const DatesT& partition,
DatePartionedTableFilesSchema& files) override;
......@@ -60,17 +60,17 @@ public:
virtual Status drop_all() override;
virtual Status count(const std::string& group_id, long& result) override;
virtual Status count(const std::string& table_id, long& result) override;
virtual ~DBMetaImpl();
private:
Status NextFileId(std::string& file_id);
Status NextGroupId(std::string& group_id);
Status NextGroupId(std::string& table_id);
Status discard_files_of_size(long to_discard_size);
Status get_group_no_lock(TableSchema& group_info);
std::string GetGroupPath(const std::string& group_id);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
std::string GetGroupPath(const std::string& table_id);
std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date);
void GetGroupFilePath(TableFileSchema& group_file);
Status initialize();
......
cpp/src/db/LocalMetaImpl.cpp
浏览文件 @ f2bb20c9
......@@ -27,22 +27,22 @@ long LocalMetaImpl::GetFileSize(const std::string& filename)
return rc == 0 ? stat_buf.st_size : -1;
}
std::string LocalMetaImpl::GetGroupPath(const std::string& group_id) {
return _options.path + "/" + group_id;
std::string LocalMetaImpl::GetGroupPath(const std::string& table_id) {
return _options.path + "/" + table_id;
}
std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) {
std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) {
std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date;
ss << GetGroupPath(table_id) << "/" << date;
return ss.str();
}
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
TableFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index";
SimpleIDGenerator g;
std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
ss << GetGroupPath(table_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
return ss.str();
}
......@@ -50,8 +50,8 @@ std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_
return group_path + "/" + "meta";
}
std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(group_id));
std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(table_id));
}
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) {
......@@ -62,18 +62,18 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchem
/* std::cout << dimension << std::endl; */
/* std::cout << files_cnt << std::endl; */
group_info.id = std::stoi(group_info.group_id);
group_info.id = std::stoi(group_info.table_id);
group_info.files_cnt = std::stoi(files_cnt);
group_info.dimension = std::stoi(dimension);
group_info.location = GetGroupPath(group_info.group_id);
group_info.location = GetGroupPath(group_info.table_id);
return Status::OK();
}
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) {
group_info.group_id = group_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info);
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info) {
group_info.table_id = table_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), group_info);
}
LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_)
......@@ -93,12 +93,12 @@ Status LocalMetaImpl::initialize() {
Status LocalMetaImpl::add_group(TableSchema& group_info) {
std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber();
if (group_info.group_id == "") {
if (group_info.table_id == "") {
std::stringstream ss;
ss << id;
real_gid = ss.str();
} else {
real_gid = group_info.group_id;
real_gid = group_info.table_id;
}
bool group_exist;
......@@ -110,7 +110,7 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) {
return Status::GroupError("Cannot Create Group " + real_gid);
}
group_info.group_id = real_gid;
group_info.table_id = real_gid;
group_info.files_cnt = 0;
group_info.id = 0;
group_info.location = GetGroupPath(real_gid);
......@@ -125,16 +125,16 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) {
Status LocalMetaImpl::get_group(TableSchema& group_info) {
bool group_exist;
has_group(group_info.group_id, group_exist);
has_group(group_info.table_id, group_exist);
if (!group_exist) {
return Status::NotFound("Group " + group_info.group_id + " Not Found");
return Status::NotFound("Group " + group_info.table_id + " Not Found");
}
return GetGroupMetaInfo(group_info.group_id, group_info);
return GetGroupMetaInfo(group_info.table_id, group_info);
}
Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
has_or_not = boost::filesystem::is_directory(GetGroupPath(group_id));
Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
has_or_not = boost::filesystem::is_directory(GetGroupPath(table_id));
return Status::OK();
}
......@@ -144,8 +144,8 @@ Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
/* if (!status.ok()) { */
/* return status; */
/* } */
/* auto location = GetNextGroupFileLocationByPartition(group_id, date, file_type); */
/* group_file_info.group_id = group_id; */
/* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */
/* group_file_info.table_id = table_id; */
/* group_file_info.dimension = group_info.dimension; */
/* group_file_info.location = location; */
/* group_file_info.date = date; */
......@@ -171,7 +171,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) {
std::cout << "[About to index] " << location << std::endl;
TableFileSchema f;
f.location = location;
/* f.group_id = group_id; */
/* f.table_id = table_id; */
f.dimension = group_info.dimension;
files.push_back(f);
}
......@@ -181,7 +181,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) {
return Status::OK();
}
Status LocalMetaImpl::files_to_merge(const std::string& group_id,
Status LocalMetaImpl::files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) {
files.clear();
/* std::string suffix; */
......@@ -200,7 +200,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* std::cout << "[About to index] " << location << std::endl; */
/* TableFileSchema f; */
/* f.location = location; */
/* f.group_id = group_id; */
/* f.table_id = table_id; */
/* f.dimension = group_info.dimension; */
/* files.push_back(f); */
/* } */
......@@ -210,21 +210,21 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
return Status::OK();
}
Status LocalMetaImpl::has_group_file(const std::string& group_id_,
Status LocalMetaImpl::has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::get_group_file(const std::string& group_id_,
Status LocalMetaImpl::get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::get_group_files(const std::string& group_id_,
Status LocalMetaImpl::get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) {
// PXU TODO
......@@ -266,7 +266,7 @@ Status LocalMetaImpl::size(long& result) {
return Status::OK();
}
Status LocalMetaImpl::count(const std::string& group_id, long& result) {
Status LocalMetaImpl::count(const std::string& table_id, long& result) {
// PXU TODO
return Status::OK();
}
......
cpp/src/db/LocalMetaImpl.h
浏览文件 @ f2bb20c9
......@@ -20,21 +20,21 @@ public:
virtual Status add_group(TableSchema& group_info_) override;
virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
/* virtual Status delete_group_partitions(const std::string& group_id, */
/* virtual Status delete_group_partitions(const std::string& table_id, */
/* const meta::DatesT& dates) override; */
virtual Status has_group_file(const std::string& group_id_,
virtual Status has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_,
virtual Status get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) override;
virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_,
virtual Status get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) override;
......@@ -42,7 +42,7 @@ public:
virtual Status cleanup() override;
virtual Status files_to_merge(const std::string& group_id,
virtual Status files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) override;
virtual Status files_to_index(TableFilesSchema&) override;
......@@ -51,7 +51,7 @@ public:
virtual Status cleanup_ttl_files(uint16_t seconds) override;
virtual Status count(const std::string& group_id, long& result) override;
virtual Status count(const std::string& table_id, long& result) override;
virtual Status drop_all() override;
......@@ -61,12 +61,12 @@ private:
Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info);
std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
Status GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info);
std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
TableFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
std::string GetGroupPath(const std::string& group_id);
std::string GetGroupMetaPath(const std::string& group_id);
std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date);
std::string GetGroupPath(const std::string& table_id);
std::string GetGroupMetaPath(const std::string& table_id);
Status CreateGroupMeta(const TableSchema& group_schema);
long GetFileSize(const std::string& filename);
......
cpp/src/db/MemManager.cpp
浏览文件 @ f2bb20c9
......@@ -47,8 +47,8 @@ size_t MemVectors<EngineT>::approximate_size() const {
}
template<typename EngineT>
Status MemVectors<EngineT>::serialize(std::string& group_id) {
group_id = schema_.group_id;
Status MemVectors<EngineT>::serialize(std::string& table_id) {
table_id = schema_.table_id;
auto size = approximate_size();
pEE_->Serialize();
schema_.size = size;
......@@ -79,40 +79,40 @@ MemVectors<EngineT>::~MemVectors() {
template<typename EngineT>
typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::get_mem_by_group(
const std::string& group_id) {
auto memIt = _memMap.find(group_id);
const std::string& table_id) {
auto memIt = _memMap.find(table_id);
if (memIt != _memMap.end()) {
return memIt->second;
}
meta::TableFileSchema group_file;
group_file.group_id = group_id;
group_file.table_id = table_id;
auto status = _pMeta->add_group_file(group_file);
if (!status.ok()) {
return nullptr;
}
_memMap[group_id] = MemVectorsPtr(new MemVectors<EngineT>(_pMeta, group_file, options_));
return _memMap[group_id];
_memMap[table_id] = MemVectorsPtr(new MemVectors<EngineT>(_pMeta, group_file, options_));
return _memMap[table_id];
}
template<typename EngineT>
Status MemManager<EngineT>::add_vectors(const std::string& group_id_,
Status MemManager<EngineT>::add_vectors(const std::string& table_id_,
size_t n_,
const float* vectors_,
IDNumbers& vector_ids_) {
std::unique_lock<std::mutex> lock(_mutex);
return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_);
return add_vectors_no_lock(table_id_, n_, vectors_, vector_ids_);
}
template<typename EngineT>
Status MemManager<EngineT>::add_vectors_no_lock(const std::string& group_id,
Status MemManager<EngineT>::add_vectors_no_lock(const std::string& table_id,
size_t n,
const float* vectors,
IDNumbers& vector_ids) {
MemVectorsPtr mem = get_mem_by_group(group_id);
MemVectorsPtr mem = get_mem_by_group(table_id);
if (mem == nullptr) {
return Status::NotFound("Group " + group_id + " not found!");
return Status::NotFound("Group " + table_id + " not found!");
}
mem->add(n, vectors, vector_ids);
......@@ -131,14 +131,14 @@ Status MemManager<EngineT>::mark_memory_as_immutable() {
}
template<typename EngineT>
Status MemManager<EngineT>::serialize(std::vector<std::string>& group_ids) {
Status MemManager<EngineT>::serialize(std::vector<std::string>& table_ids) {
mark_memory_as_immutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
std::string group_id;
group_ids.clear();
std::string table_id;
table_ids.clear();
for (auto& mem : _immMems) {
mem->serialize(group_id);
group_ids.push_back(group_id);
mem->serialize(table_id);
table_ids.push_back(table_id);
}
_immMems.clear();
return Status::OK();
......
cpp/src/db/MemManager.h
浏览文件 @ f2bb20c9
......@@ -39,7 +39,7 @@ public:
size_t approximate_size() const;
Status serialize(std::string& group_id);
Status serialize(std::string& table_id);
~MemVectors();
......@@ -70,15 +70,15 @@ public:
MemManager(const std::shared_ptr<meta::Meta>& meta_, const Options& options)
: _pMeta(meta_), options_(options) {}
MemVectorsPtr get_mem_by_group(const std::string& group_id_);
MemVectorsPtr get_mem_by_group(const std::string& table_id_);
Status add_vectors(const std::string& group_id_,
Status add_vectors(const std::string& table_id_,
size_t n_, const float* vectors_, IDNumbers& vector_ids_);
Status serialize(std::vector<std::string>& group_ids);
Status serialize(std::vector<std::string>& table_ids);
private:
Status add_vectors_no_lock(const std::string& group_id_,
Status add_vectors_no_lock(const std::string& table_id_,
size_t n_, const float* vectors_, IDNumbers& vector_ids_);
Status mark_memory_as_immutable();
......
cpp/src/db/Meta.h
浏览文件 @ f2bb20c9
......@@ -24,31 +24,31 @@ public:
virtual Status add_group(TableSchema& group_info) = 0;
virtual Status get_group(TableSchema& group_info) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0;
virtual Status add_group_file(TableFileSchema& group_file_info) = 0;
virtual Status delete_group_partitions(const std::string& group_id,
virtual Status delete_group_partitions(const std::string& table_id,
const meta::DatesT& dates) = 0;
virtual Status has_group_file(const std::string& group_id_,
virtual Status has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) = 0;
virtual Status get_group_file(const std::string& group_id_,
virtual Status get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) = 0;
virtual Status update_group_file(TableFileSchema& group_file_) = 0;
virtual Status get_group_files(const std::string& group_id_,
virtual Status get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) = 0;
virtual Status update_files(TableFilesSchema& files) = 0;
virtual Status files_to_search(const std::string& group_id,
virtual Status files_to_search(const std::string& table_id,
const DatesT& partition,
DatePartionedTableFilesSchema& files) = 0;
virtual Status files_to_merge(const std::string& group_id,
virtual Status files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) = 0;
virtual Status size(long& result) = 0;
......@@ -62,7 +62,7 @@ public:
virtual Status drop_all() = 0;
virtual Status count(const std::string& group_id, long& result) = 0;
virtual Status count(const std::string& table_id, long& result) = 0;
static DateT GetDate(const std::time_t& t, int day_delta = 0);
static DateT GetDate();
......
cpp/src/db/MetaTypes.h
浏览文件 @ f2bb20c9
......@@ -20,7 +20,7 @@ typedef std::vector<DateT> DatesT;
struct TableSchema {
size_t id;
std::string group_id;
std::string table_id;
size_t files_cnt = 0;
uint16_t dimension;
std::string location;
......@@ -37,7 +37,7 @@ struct TableFileSchema {
} FILE_TYPE;
size_t id;
std::string group_id;
std::string table_id;
std::string file_id;
int file_type = NEW;
size_t size;
......
cpp/src/db/db_connection.cpp 已删除 100644 → 0
浏览文件 @ 209d0e48
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "db_connection.h"
namespace zilliz {
namespace vecwise {
namespace engine {
using std::string;
using namespace sqlite_orm;
string storage_file_name = "default.sqlite";
SqliteDBPtr connect() {
SqliteDBPtr temp = std::make_shared<SqliteDB>(initStorage(storage_file_name));
temp->sync_schema();
temp->open_forever(); // thread safe option
//temp->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
return temp;
}
/* SqliteDBPtr Connection::connect_ = connect(); */
}
}
}
cpp/src/db/db_connection.h 已删除 100644 → 0
浏览文件 @ 209d0e48
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <string>
#include <memory>
#include <sqlite_orm.h>
namespace zilliz {
namespace vecwise {
namespace engine {
struct GroupSchema {
size_t id;
std::string group_id;
size_t files_cnt = 0;
uint16_t dimension;
std::string location = "";
std::string next_file_location = "";
}; // GroupSchema
struct GroupFileSchema {
typedef enum {
RAW,
INDEX
} FILE_TYPE;
size_t id;
std::string group_id;
std::string file_id;
int files_type = RAW;
size_t rows;
std::string location = "";
}; // GroupFileSchema
inline auto initStorage(const std::string &path) {
using namespace sqlite_orm;
return make_storage(path,
// Add table below
make_table("Groups",
make_column("id", &GroupSchema::id, primary_key()),
make_column("group_id", &GroupSchema::group_id, unique()),
make_column("dimension", &GroupSchema::dimension),
make_column("files_cnt", &GroupSchema::files_cnt, default_value(0))));
}
using SqliteDB = decltype(initStorage(""));
using SqliteDBPtr= std::shared_ptr<SqliteDB>;
class Connection {
protected:
static SqliteDBPtr connect_;
};
}
}
}
cpp/unittest/db/db_tests.cpp
浏览文件 @ f2bb20c9
......@@ -68,11 +68,11 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.group_id = group_name;
group_info.table_id = group_name;
engine::Status stat = db_->add_group(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name;
group_info_get.table_id = group_name;
stat = db_->get_group(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
......@@ -113,11 +113,11 @@ TEST_F(DBTest, DB_TEST) {
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.group_id = group_name;
group_info.table_id = group_name;
engine::Status stat = db_->add_group(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name;
group_info_get.table_id = group_name;
stat = db_->get_group(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
......@@ -199,11 +199,11 @@ TEST_F(DBTest, SEARCH_TEST) {
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.group_id = group_name;
group_info.table_id = group_name;
engine::Status stat = db_->add_group(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name;
group_info_get.table_id = group_name;
stat = db_->get_group(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
......
cpp/unittest/db/meta_tests.cpp
浏览文件 @ f2bb20c9
......@@ -18,10 +18,10 @@
using namespace zilliz::vecwise::engine;
TEST_F(MetaTest, GROUP_TEST) {
auto group_id = "meta_test_group";
auto table_id = "meta_test_group";
meta::TableSchema group;
group.group_id = group_id;
group.table_id = table_id;
auto status = impl_->add_group(group);
ASSERT_TRUE(status.ok());
......@@ -30,26 +30,26 @@ TEST_F(MetaTest, GROUP_TEST) {
status = impl_->get_group(group);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group.id, gid);
ASSERT_EQ(group.group_id, group_id);
ASSERT_EQ(group.table_id, table_id);
group.group_id = "not_found";
group.table_id = "not_found";
status = impl_->get_group(group);
ASSERT_TRUE(!status.ok());
group.group_id = group_id;
group.table_id = table_id;
status = impl_->add_group(group);
ASSERT_TRUE(!status.ok());
}
TEST_F(MetaTest, GROUP_FILE_TEST) {
auto group_id = "meta_test_group";
auto table_id = "meta_test_group";
meta::TableSchema group;
group.group_id = group_id;
group.table_id = table_id;
auto status = impl_->add_group(group);
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
group_file.table_id = group.table_id;
status = impl_->add_group_file(group_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW);
......@@ -65,14 +65,14 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
meta::DatesT dates;
dates.push_back(meta::Meta::GetDate());
status = impl_->delete_group_partitions(group_file.group_id, dates);
status = impl_->delete_group_partitions(group_file.table_id, dates);
ASSERT_FALSE(status.ok());
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(meta::Meta::GetDateWithDelta(-1*i));
}
status = impl_->delete_group_partitions(group_file.group_id, dates);
status = impl_->delete_group_partitions(group_file.table_id, dates);
ASSERT_TRUE(status.ok());
group_file.date = meta::Meta::GetDateWithDelta(-2);
......@@ -83,9 +83,9 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
dates.clear();
dates.push_back(group_file.date);
status = impl_->delete_group_partitions(group_file.group_id, dates);
status = impl_->delete_group_partitions(group_file.table_id, dates);
ASSERT_TRUE(status.ok());
status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file);
status = impl_->get_group_file(group_file.table_id, group_file.file_id, group_file);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
}
......@@ -100,15 +100,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
options.archive_conf = ArchiveConf("delete", ss.str());
auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group";
auto table_id = "meta_test_group";
meta::TableSchema group;
group.group_id = group_id;
group.table_id = table_id;
auto status = impl.add_group(group);
meta::TableFilesSchema files;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
group_file.table_id = group.table_id;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
......@@ -127,7 +127,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
int i = 0;
for (auto file : files) {
status = impl.get_group_file(file.group_id, file.file_id, file);
status = impl.get_group_file(file.table_id, file.file_id, file);
ASSERT_TRUE(status.ok());
if (days[i] < days_num) {
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
......@@ -146,15 +146,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
options.archive_conf = ArchiveConf("delete", "disk:11");
auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group";
auto table_id = "meta_test_group";
meta::TableSchema group;
group.group_id = group_id;
group.table_id = table_id;
auto status = impl.add_group(group);
meta::TableFilesSchema files;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
group_file.table_id = group.table_id;
auto cnt = 10;
auto each_size = 2UL;
......@@ -170,7 +170,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
int i = 0;
for (auto file : files) {
status = impl.get_group_file(file.group_id, file.file_id, file);
status = impl.get_group_file(file.table_id, file.file_id, file);
ASSERT_TRUE(status.ok());
if (i < 5) {
ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
......@@ -184,10 +184,10 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
}
TEST_F(MetaTest, GROUP_FILES_TEST) {
auto group_id = "meta_test_group";
auto table_id = "meta_test_group";
meta::TableSchema group;
group.group_id = group_id;
group.table_id = table_id;
auto status = impl_->add_group(group);
int new_files_cnt = 4;
......@@ -196,7 +196,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
int index_files_cnt = 7;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
group_file.table_id = group.table_id;
for (auto i=0; i<new_files_cnt; ++i) {
status = impl_->add_group_file(group_file);
......@@ -229,7 +229,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
status = impl_->files_to_merge(group.group_id, dated_files);
status = impl_->files_to_merge(group.table_id, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt);
......@@ -238,7 +238,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {group_file.date};
status = impl_->files_to_search(group_id, dates, dated_files);
status = impl_->files_to_search(table_id, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[group_file.date].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或