From f2bb20c90d4c27f328180c9aa02c35735bcca736 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:30:17 +0800 Subject: [PATCH] refactor(db): change group_id to table_id Former-commit-id: bf0cec2552fcad95bf1b77cdfbdd423984e6f18f --- cpp/src/db/DB.h | 14 ++--- cpp/src/db/DBImpl.cpp | 50 ++++++++-------- cpp/src/db/DBImpl.h | 18 +++--- cpp/src/db/DBMetaImpl.cpp | 102 ++++++++++++++++----------------- cpp/src/db/DBMetaImpl.h | 22 +++---- cpp/src/db/LocalMetaImpl.cpp | 60 +++++++++---------- cpp/src/db/LocalMetaImpl.h | 24 ++++---- cpp/src/db/MemManager.cpp | 34 +++++------ cpp/src/db/MemManager.h | 10 ++-- cpp/src/db/Meta.h | 16 +++--- cpp/src/db/MetaTypes.h | 4 +- cpp/src/db/db_connection.cpp | 31 ---------- cpp/src/db/db_connection.h | 64 --------------------- cpp/unittest/db/db_tests.cpp | 12 ++-- cpp/unittest/db/meta_tests.cpp | 50 ++++++++-------- 15 files changed, 208 insertions(+), 303 deletions(-) delete mode 100644 cpp/src/db/db_connection.cpp delete mode 100644 cpp/src/db/db_connection.h diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index c1f73e5b..20e49240 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -23,27 +23,27 @@ public: virtual Status add_group(meta::TableSchema& group_info_) = 0; virtual Status get_group(meta::TableSchema& group_info_) = 0; - virtual Status delete_vectors(const std::string& group_id, + virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) = 0; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; - virtual Status get_group_files(const std::string& group_id_, + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, meta::TableFilesSchema& group_files_info_) = 0; - virtual Status add_vectors(const std::string& group_id_, + virtual Status add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) = 0; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; virtual Status size(long& result) = 0; virtual Status drop_all() = 0; - virtual Status count(const std::string& group_id, long& result) = 0; + virtual Status count(const std::string& table_id, long& result) = 0; DB() = default; DB(const DB&) = delete; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index c3df297b..54a13800 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -45,46 +45,46 @@ Status DBImpl::get_group(meta::TableSchema& group_info) { } template -Status DBImpl::delete_vectors(const std::string& group_id, +Status DBImpl::delete_vectors(const std::string& table_id, const meta::DatesT& dates) { - return _pMeta->delete_group_partitions(group_id, dates); + return _pMeta->delete_group_partitions(table_id, dates); } template -Status DBImpl::has_group(const std::string& group_id_, bool& has_or_not_) { - return _pMeta->has_group(group_id_, has_or_not_); +Status DBImpl::has_group(const std::string& table_id_, bool& has_or_not_) { + return _pMeta->has_group(table_id_, has_or_not_); } template -Status DBImpl::get_group_files(const std::string& group_id, +Status DBImpl::get_group_files(const std::string& table_id, const int date_delta, meta::TableFilesSchema& group_files_info) { - return _pMeta->get_group_files(group_id, date_delta, group_files_info); + return _pMeta->get_group_files(table_id, date_delta, group_files_info); } template -Status DBImpl::add_vectors(const std::string& group_id_, +Status DBImpl::add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { - Status status = _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_); + Status status = _pMemMgr->add_vectors(table_id_, n, vectors, vector_ids_); if (!status.ok()) { return status; } } template -Status DBImpl::search(const std::string &group_id, size_t k, size_t nq, +Status DBImpl::search(const std::string &table_id, size_t k, size_t nq, const float *vectors, QueryResults &results) { meta::DatesT dates = {meta::Meta::GetDate()}; - return search(group_id, k, nq, vectors, dates, results); + return search(table_id, k, nq, vectors, dates, results); } template -Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, +Status DBImpl::search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { meta::DatePartionedTableFilesSchema files; - auto status = _pMeta->files_to_search(group_id, dates, files); + auto status = _pMeta->files_to_search(table_id, dates, files); if (!status.ok()) { return status; } LOG(DEBUG) << "Search DateT Size=" << files.size(); @@ -204,7 +204,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, } if (results.empty()) { - return Status::NotFound("Group " + group_id + ", search result not found!"); + return Status::NotFound("Group " + table_id + ", search result not found!"); } return Status::OK(); } @@ -257,10 +257,10 @@ void DBImpl::background_call() { template -Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, +Status DBImpl::merge_files(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { meta::TableFileSchema group_file; - group_file.group_id = group_id; + group_file.table_id = table_id; group_file.date = date; Status status = _pMeta->add_group_file(group_file); @@ -304,9 +304,9 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::Dat } template -Status DBImpl::background_merge_files(const std::string& group_id) { +Status DBImpl::background_merge_files(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; - auto status = _pMeta->files_to_merge(group_id, raw_files); + auto status = _pMeta->files_to_merge(table_id, raw_files); if (!status.ok()) { return status; } @@ -323,7 +323,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) { continue; } has_merge = true; - merge_files(group_id, kv.first, kv.second); + merge_files(table_id, kv.first, kv.second); } _pMeta->archive_files(); @@ -338,7 +338,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) { template Status DBImpl::build_index(const meta::TableFileSchema& file) { meta::TableFileSchema group_file; - group_file.group_id = file.group_id; + group_file.table_id = file.table_id; group_file.date = file.date; Status status = _pMeta->add_group_file(group_file); if (!status.ok()) { @@ -402,12 +402,12 @@ Status DBImpl::try_build_index() { template void DBImpl::background_compaction() { - std::vector group_ids; - _pMemMgr->serialize(group_ids); + std::vector table_ids; + _pMemMgr->serialize(table_ids); Status status; - for (auto group_id : group_ids) { - status = background_merge_files(group_id); + for (auto table_id : table_ids) { + status = background_merge_files(table_id); if (!status.ok()) { _bg_error = status; return; @@ -421,8 +421,8 @@ Status DBImpl::drop_all() { } template -Status DBImpl::count(const std::string& group_id, long& result) { - return _pMeta->count(group_id, result); +Status DBImpl::count(const std::string& table_id, long& result) { + return _pMeta->count(table_id, result); } template diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index a727bc6e..35a4a93e 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -35,25 +35,25 @@ public: virtual Status add_group(meta::TableSchema& group_info) override; virtual Status get_group(meta::TableSchema& group_info) override; - virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, meta::TableFilesSchema& group_files_info_) override; - virtual Status add_vectors(const std::string& group_id_, + virtual Status add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) override; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) override; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) override; virtual Status drop_all() override; - virtual Status count(const std::string& group_id, long& result) override; + virtual Status count(const std::string& table_id, long& result) override; virtual Status size(long& result) override; @@ -64,10 +64,10 @@ private: void background_build_index(); Status build_index(const meta::TableFileSchema&); Status try_build_index(); - Status merge_files(const std::string& group_id, + Status merge_files(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files); - Status background_merge_files(const std::string& group_id); + Status background_merge_files(const std::string& table_id); void try_schedule_compaction(); void start_timer_task(int interval_); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 15c3a5ed..41fceb5c 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -29,13 +29,13 @@ inline auto StoragePrototype(const std::string& path) { return make_storage(path, make_table("Group", make_column("id", &TableSchema::id, primary_key()), - make_column("group_id", &TableSchema::group_id, unique()), + make_column("table_id", &TableSchema::table_id, unique()), make_column("dimension", &TableSchema::dimension), make_column("created_on", &TableSchema::created_on), make_column("files_cnt", &TableSchema::files_cnt, default_value(0))), make_table("GroupFile", make_column("id", &TableFileSchema::id, primary_key()), - make_column("group_id", &TableFileSchema::group_id), + make_column("table_id", &TableFileSchema::table_id), make_column("file_id", &TableFileSchema::file_id), make_column("file_type", &TableFileSchema::file_type), make_column("size", &TableFileSchema::size, default_value(0)), @@ -49,13 +49,13 @@ inline auto StoragePrototype(const std::string& path) { using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -std::string DBMetaImpl::GetGroupPath(const std::string& group_id) { - return _options.path + "/tables/" + group_id; +std::string DBMetaImpl::GetGroupPath(const std::string& table_id) { + return _options.path + "/tables/" + table_id; } -std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) { +std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) { std::stringstream ss; - ss << GetGroupPath(group_id) << "/" << date; + ss << GetGroupPath(table_id) << "/" << date; return ss.str(); } @@ -64,16 +64,16 @@ void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) { group_file.date = Meta::GetDate(); } std::stringstream ss; - ss << GetGroupDatePartitionPath(group_file.group_id, group_file.date) + ss << GetGroupDatePartitionPath(group_file.table_id, group_file.date) << "/" << group_file.file_id; group_file.location = ss.str(); } -Status DBMetaImpl::NextGroupId(std::string& group_id) { +Status DBMetaImpl::NextGroupId(std::string& table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.getNextIDNumber(); - group_id = ss.str(); + table_id = ss.str(); return Status::OK(); } @@ -111,14 +111,14 @@ Status DBMetaImpl::initialize() { } // PXU TODO: Temp solution. Will fix later -Status DBMetaImpl::delete_group_partitions(const std::string& group_id, +Status DBMetaImpl::delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) { if (dates.size() == 0) { return Status::OK(); } TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group(group_info); if (!status.ok()) { return status; @@ -138,7 +138,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE ), where( - c(&TableFileSchema::group_id) == group_id and + c(&TableFileSchema::table_id) == table_id and in(&TableFileSchema::date, dates) )); } catch (std::exception & e) { @@ -149,8 +149,8 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, } Status DBMetaImpl::add_group(TableSchema& group_info) { - if (group_info.group_id == "") { - NextGroupId(group_info.group_id); + if (group_info.table_id == "") { + NextGroupId(group_info.table_id); } group_info.files_cnt = 0; group_info.id = -1; @@ -165,7 +165,7 @@ Status DBMetaImpl::add_group(TableSchema& group_info) { } } - auto group_path = GetGroupPath(group_info.group_id); + auto group_path = GetGroupPath(group_info.table_id); if (!boost::filesystem::is_directory(group_path)) { auto ret = boost::filesystem::create_directories(group_path); @@ -185,17 +185,17 @@ Status DBMetaImpl::get_group(TableSchema& group_info) { Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id, - &TableSchema::group_id, + &TableSchema::table_id, &TableSchema::files_cnt, &TableSchema::dimension), - where(c(&TableSchema::group_id) == group_info.group_id)); + where(c(&TableSchema::table_id) == group_info.table_id)); assert(groups.size() <= 1); if (groups.size() == 1) { group_info.id = std::get<0>(groups[0]); group_info.files_cnt = std::get<2>(groups[0]); group_info.dimension = std::get<3>(groups[0]); } else { - return Status::NotFound("Group " + group_info.group_id + " not found"); + return Status::NotFound("Group " + group_info.table_id + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -205,10 +205,10 @@ Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { return Status::OK(); } -Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { +Status DBMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id), - where(c(&TableSchema::group_id) == group_id)); + where(c(&TableSchema::table_id) == table_id)); assert(groups.size() <= 1); if (groups.size() == 1) { has_or_not = true; @@ -227,7 +227,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { group_file.date = Meta::GetDate(); } TableSchema group_info; - group_info.group_id = group_file.group_id; + group_info.table_id = group_file.table_id; auto status = get_group(group_info); if (!status.ok()) { return status; @@ -250,7 +250,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { } } - auto partition_path = GetGroupDatePartitionPath(group_file.group_id, group_file.date); + auto partition_path = GetGroupDatePartitionPath(group_file.table_id, group_file.date); if (!boost::filesystem::is_directory(partition_path)) { auto ret = boost::filesystem::create_directory(partition_path); @@ -268,7 +268,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, @@ -280,23 +280,23 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); GetGroupFilePath(group_file); - auto groupItr = groups.find(group_file.group_id); + auto groupItr = groups.find(group_file.table_id); if (groupItr == groups.end()) { TableSchema group_info; - group_info.group_id = group_file.group_id; + group_info.table_id = group_file.table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; } - groups[group_file.group_id] = group_info; + groups[group_file.table_id] = group_info; } - group_file.dimension = groups[group_file.group_id].dimension; + group_file.dimension = groups[group_file.table_id].dimension; files.push_back(group_file); } } catch (std::exception & e) { @@ -307,7 +307,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { return Status::OK(); } -Status DBMetaImpl::files_to_search(const std::string &group_id, +Status DBMetaImpl::files_to_search(const std::string &table_id, const DatesT& partition, DatePartionedTableFilesSchema &files) { files.clear(); @@ -316,19 +316,19 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), - where(c(&TableFileSchema::group_id) == group_id and + where(c(&TableFileSchema::table_id) == table_id and in(&TableFileSchema::date, dates) and (c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX))); TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; @@ -337,7 +337,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -358,22 +358,22 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, return Status::OK(); } -Status DBMetaImpl::files_to_merge(const std::string& group_id, +Status DBMetaImpl::files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and - c(&TableFileSchema::group_id) == group_id)); + c(&TableFileSchema::table_id) == table_id)); TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; @@ -382,7 +382,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -403,30 +403,30 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, return Status::OK(); } -Status DBMetaImpl::has_group_file(const std::string& group_id_, +Status DBMetaImpl::has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) { //PXU TODO return Status::OK(); } -Status DBMetaImpl::get_group_file(const std::string& group_id_, +Status DBMetaImpl::get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) { try { auto files = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), where(c(&TableFileSchema::file_id) == file_id_ and - c(&TableFileSchema::group_id) == group_id_ + c(&TableFileSchema::table_id) == table_id_ )); assert(files.size() <= 1); if (files.size() == 1) { group_file_info_.id = std::get<0>(files[0]); - group_file_info_.group_id = std::get<1>(files[0]); + group_file_info_.table_id = std::get<1>(files[0]); group_file_info_.file_id = std::get<2>(files[0]); group_file_info_.file_type = std::get<3>(files[0]); group_file_info_.size = std::get<4>(files[0]); @@ -442,7 +442,7 @@ Status DBMetaImpl::get_group_file(const std::string& group_id_, return Status::OK(); } -Status DBMetaImpl::get_group_files(const std::string& group_id_, +Status DBMetaImpl::get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) { // PXU TODO @@ -591,7 +591,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, @@ -604,7 +604,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -627,7 +627,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { Status DBMetaImpl::cleanup() { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, @@ -640,7 +640,7 @@ Status DBMetaImpl::cleanup() { for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -660,7 +660,7 @@ Status DBMetaImpl::cleanup() { return Status::OK(); } -Status DBMetaImpl::count(const std::string& group_id, long& result) { +Status DBMetaImpl::count(const std::string& table_id, long& result) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::size, @@ -668,10 +668,10 @@ Status DBMetaImpl::count(const std::string& group_id, long& result) { where((c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW or c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and - c(&TableFileSchema::group_id) == group_id)); + c(&TableFileSchema::table_id) == table_id)); TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 1479396e..b278bee2 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -21,30 +21,30 @@ public: virtual Status add_group(TableSchema& group_info) override; virtual Status get_group(TableSchema& group_info_) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; - virtual Status delete_group_partitions(const std::string& group_id, + virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) override; - virtual Status has_group_file(const std::string& group_id_, + virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) override; - virtual Status get_group_file(const std::string& group_id_, + virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) override; virtual Status update_group_file(TableFileSchema& group_file_) override; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) override; virtual Status update_files(TableFilesSchema& files) override; - virtual Status files_to_merge(const std::string& group_id, + virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - virtual Status files_to_search(const std::string& group_id, + virtual Status files_to_search(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) override; @@ -60,17 +60,17 @@ public: virtual Status drop_all() override; - virtual Status count(const std::string& group_id, long& result) override; + virtual Status count(const std::string& table_id, long& result) override; virtual ~DBMetaImpl(); private: Status NextFileId(std::string& file_id); - Status NextGroupId(std::string& group_id); + Status NextGroupId(std::string& table_id); Status discard_files_of_size(long to_discard_size); Status get_group_no_lock(TableSchema& group_info); - std::string GetGroupPath(const std::string& group_id); - std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); + std::string GetGroupPath(const std::string& table_id); + std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); void GetGroupFilePath(TableFileSchema& group_file); Status initialize(); diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index f254d238..70a07350 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -27,22 +27,22 @@ long LocalMetaImpl::GetFileSize(const std::string& filename) return rc == 0 ? stat_buf.st_size : -1; } -std::string LocalMetaImpl::GetGroupPath(const std::string& group_id) { - return _options.path + "/" + group_id; +std::string LocalMetaImpl::GetGroupPath(const std::string& table_id) { + return _options.path + "/" + table_id; } -std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) { +std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) { std::stringstream ss; - ss << GetGroupPath(group_id) << "/" << date; + ss << GetGroupPath(table_id) << "/" << date; return ss.str(); } -std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, +std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, TableFileSchema::FILE_TYPE file_type) { std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index"; SimpleIDGenerator g; std::stringstream ss; - ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; + ss << GetGroupPath(table_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; return ss.str(); } @@ -50,8 +50,8 @@ std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_ return group_path + "/" + "meta"; } -std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) { - return GetGroupMetaPathByGroupPath(GetGroupPath(group_id)); +std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) { + return GetGroupMetaPathByGroupPath(GetGroupPath(table_id)); } Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) { @@ -62,18 +62,18 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchem /* std::cout << dimension << std::endl; */ /* std::cout << files_cnt << std::endl; */ - group_info.id = std::stoi(group_info.group_id); + group_info.id = std::stoi(group_info.table_id); group_info.files_cnt = std::stoi(files_cnt); group_info.dimension = std::stoi(dimension); - group_info.location = GetGroupPath(group_info.group_id); + group_info.location = GetGroupPath(group_info.table_id); return Status::OK(); } -Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) { - group_info.group_id = group_id; - return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info); +Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info) { + group_info.table_id = table_id; + return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), group_info); } LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_) @@ -93,12 +93,12 @@ Status LocalMetaImpl::initialize() { Status LocalMetaImpl::add_group(TableSchema& group_info) { std::string real_gid; size_t id = SimpleIDGenerator().getNextIDNumber(); - if (group_info.group_id == "") { + if (group_info.table_id == "") { std::stringstream ss; ss << id; real_gid = ss.str(); } else { - real_gid = group_info.group_id; + real_gid = group_info.table_id; } bool group_exist; @@ -110,7 +110,7 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) { return Status::GroupError("Cannot Create Group " + real_gid); } - group_info.group_id = real_gid; + group_info.table_id = real_gid; group_info.files_cnt = 0; group_info.id = 0; group_info.location = GetGroupPath(real_gid); @@ -125,16 +125,16 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) { Status LocalMetaImpl::get_group(TableSchema& group_info) { bool group_exist; - has_group(group_info.group_id, group_exist); + has_group(group_info.table_id, group_exist); if (!group_exist) { - return Status::NotFound("Group " + group_info.group_id + " Not Found"); + return Status::NotFound("Group " + group_info.table_id + " Not Found"); } - return GetGroupMetaInfo(group_info.group_id, group_info); + return GetGroupMetaInfo(group_info.table_id, group_info); } -Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { - has_or_not = boost::filesystem::is_directory(GetGroupPath(group_id)); +Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { + has_or_not = boost::filesystem::is_directory(GetGroupPath(table_id)); return Status::OK(); } @@ -144,8 +144,8 @@ Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { /* if (!status.ok()) { */ /* return status; */ /* } */ - /* auto location = GetNextGroupFileLocationByPartition(group_id, date, file_type); */ - /* group_file_info.group_id = group_id; */ + /* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */ + /* group_file_info.table_id = table_id; */ /* group_file_info.dimension = group_info.dimension; */ /* group_file_info.location = location; */ /* group_file_info.date = date; */ @@ -171,7 +171,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { std::cout << "[About to index] " << location << std::endl; TableFileSchema f; f.location = location; - /* f.group_id = group_id; */ + /* f.table_id = table_id; */ f.dimension = group_info.dimension; files.push_back(f); } @@ -181,7 +181,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { return Status::OK(); } -Status LocalMetaImpl::files_to_merge(const std::string& group_id, +Status LocalMetaImpl::files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); /* std::string suffix; */ @@ -200,7 +200,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, /* std::cout << "[About to index] " << location << std::endl; */ /* TableFileSchema f; */ /* f.location = location; */ - /* f.group_id = group_id; */ + /* f.table_id = table_id; */ /* f.dimension = group_info.dimension; */ /* files.push_back(f); */ /* } */ @@ -210,21 +210,21 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, return Status::OK(); } -Status LocalMetaImpl::has_group_file(const std::string& group_id_, +Status LocalMetaImpl::has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) { //PXU TODO return Status::OK(); } -Status LocalMetaImpl::get_group_file(const std::string& group_id_, +Status LocalMetaImpl::get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) { //PXU TODO return Status::OK(); } -Status LocalMetaImpl::get_group_files(const std::string& group_id_, +Status LocalMetaImpl::get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) { // PXU TODO @@ -266,7 +266,7 @@ Status LocalMetaImpl::size(long& result) { return Status::OK(); } -Status LocalMetaImpl::count(const std::string& group_id, long& result) { +Status LocalMetaImpl::count(const std::string& table_id, long& result) { // PXU TODO return Status::OK(); } diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index 952e85b1..e57f0a97 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -20,21 +20,21 @@ public: virtual Status add_group(TableSchema& group_info_) override; virtual Status get_group(TableSchema& group_info_) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; - /* virtual Status delete_group_partitions(const std::string& group_id, */ + /* virtual Status delete_group_partitions(const std::string& table_id, */ /* const meta::DatesT& dates) override; */ - virtual Status has_group_file(const std::string& group_id_, + virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) override; - virtual Status get_group_file(const std::string& group_id_, + virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) override; virtual Status update_group_file(TableFileSchema& group_file_) override; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) override; @@ -42,7 +42,7 @@ public: virtual Status cleanup() override; - virtual Status files_to_merge(const std::string& group_id, + virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; virtual Status files_to_index(TableFilesSchema&) override; @@ -51,7 +51,7 @@ public: virtual Status cleanup_ttl_files(uint16_t seconds) override; - virtual Status count(const std::string& group_id, long& result) override; + virtual Status count(const std::string& table_id, long& result) override; virtual Status drop_all() override; @@ -61,12 +61,12 @@ private: Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info); std::string GetGroupMetaPathByGroupPath(const std::string& group_path); - Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info); - std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, + Status GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info); + std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, TableFileSchema::FILE_TYPE file_type); - std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); - std::string GetGroupPath(const std::string& group_id); - std::string GetGroupMetaPath(const std::string& group_id); + std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); + std::string GetGroupPath(const std::string& table_id); + std::string GetGroupMetaPath(const std::string& table_id); Status CreateGroupMeta(const TableSchema& group_schema); long GetFileSize(const std::string& filename); diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index 8ba0d7dc..bd69bbc2 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -47,8 +47,8 @@ size_t MemVectors::approximate_size() const { } template -Status MemVectors::serialize(std::string& group_id) { - group_id = schema_.group_id; +Status MemVectors::serialize(std::string& table_id) { + table_id = schema_.table_id; auto size = approximate_size(); pEE_->Serialize(); schema_.size = size; @@ -79,40 +79,40 @@ MemVectors::~MemVectors() { template typename MemManager::MemVectorsPtr MemManager::get_mem_by_group( - const std::string& group_id) { - auto memIt = _memMap.find(group_id); + const std::string& table_id) { + auto memIt = _memMap.find(table_id); if (memIt != _memMap.end()) { return memIt->second; } meta::TableFileSchema group_file; - group_file.group_id = group_id; + group_file.table_id = table_id; auto status = _pMeta->add_group_file(group_file); if (!status.ok()) { return nullptr; } - _memMap[group_id] = MemVectorsPtr(new MemVectors(_pMeta, group_file, options_)); - return _memMap[group_id]; + _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, group_file, options_)); + return _memMap[table_id]; } template -Status MemManager::add_vectors(const std::string& group_id_, +Status MemManager::add_vectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { std::unique_lock lock(_mutex); - return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); + return add_vectors_no_lock(table_id_, n_, vectors_, vector_ids_); } template -Status MemManager::add_vectors_no_lock(const std::string& group_id, +Status MemManager::add_vectors_no_lock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) { - MemVectorsPtr mem = get_mem_by_group(group_id); + MemVectorsPtr mem = get_mem_by_group(table_id); if (mem == nullptr) { - return Status::NotFound("Group " + group_id + " not found!"); + return Status::NotFound("Group " + table_id + " not found!"); } mem->add(n, vectors, vector_ids); @@ -131,14 +131,14 @@ Status MemManager::mark_memory_as_immutable() { } template -Status MemManager::serialize(std::vector& group_ids) { +Status MemManager::serialize(std::vector& table_ids) { mark_memory_as_immutable(); std::unique_lock lock(serialization_mtx_); - std::string group_id; - group_ids.clear(); + std::string table_id; + table_ids.clear(); for (auto& mem : _immMems) { - mem->serialize(group_id); - group_ids.push_back(group_id); + mem->serialize(table_id); + table_ids.push_back(table_id); } _immMems.clear(); return Status::OK(); diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index a1b6e1c7..b6e4b86c 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -39,7 +39,7 @@ public: size_t approximate_size() const; - Status serialize(std::string& group_id); + Status serialize(std::string& table_id); ~MemVectors(); @@ -70,15 +70,15 @@ public: MemManager(const std::shared_ptr& meta_, const Options& options) : _pMeta(meta_), options_(options) {} - MemVectorsPtr get_mem_by_group(const std::string& group_id_); + MemVectorsPtr get_mem_by_group(const std::string& table_id_); - Status add_vectors(const std::string& group_id_, + Status add_vectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); - Status serialize(std::vector& group_ids); + Status serialize(std::vector& table_ids); private: - Status add_vectors_no_lock(const std::string& group_id_, + Status add_vectors_no_lock(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); Status mark_memory_as_immutable(); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index afa9eee2..90b3384c 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -24,31 +24,31 @@ public: virtual Status add_group(TableSchema& group_info) = 0; virtual Status get_group(TableSchema& group_info) = 0; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0; - virtual Status delete_group_partitions(const std::string& group_id, + virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) = 0; - virtual Status has_group_file(const std::string& group_id_, + virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) = 0; - virtual Status get_group_file(const std::string& group_id_, + virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) = 0; virtual Status update_group_file(TableFileSchema& group_file_) = 0; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) = 0; virtual Status update_files(TableFilesSchema& files) = 0; - virtual Status files_to_search(const std::string& group_id, + virtual Status files_to_search(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) = 0; - virtual Status files_to_merge(const std::string& group_id, + virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; virtual Status size(long& result) = 0; @@ -62,7 +62,7 @@ public: virtual Status drop_all() = 0; - virtual Status count(const std::string& group_id, long& result) = 0; + virtual Status count(const std::string& table_id, long& result) = 0; static DateT GetDate(const std::time_t& t, int day_delta = 0); static DateT GetDate(); diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index 88bb112d..b2fe7833 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -20,7 +20,7 @@ typedef std::vector DatesT; struct TableSchema { size_t id; - std::string group_id; + std::string table_id; size_t files_cnt = 0; uint16_t dimension; std::string location; @@ -37,7 +37,7 @@ struct TableFileSchema { } FILE_TYPE; size_t id; - std::string group_id; + std::string table_id; std::string file_id; int file_type = NEW; size_t size; diff --git a/cpp/src/db/db_connection.cpp b/cpp/src/db/db_connection.cpp deleted file mode 100644 index 74779daf..00000000 --- a/cpp/src/db/db_connection.cpp +++ /dev/null @@ -1,31 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "db_connection.h" - - -namespace zilliz { -namespace vecwise { -namespace engine { - -using std::string; -using namespace sqlite_orm; - -string storage_file_name = "default.sqlite"; - -SqliteDBPtr connect() { - SqliteDBPtr temp = std::make_shared(initStorage(storage_file_name)); - temp->sync_schema(); - temp->open_forever(); // thread safe option - //temp->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log - return temp; -} - -/* SqliteDBPtr Connection::connect_ = connect(); */ - -} -} -} diff --git a/cpp/src/db/db_connection.h b/cpp/src/db/db_connection.h deleted file mode 100644 index cfd34461..00000000 --- a/cpp/src/db/db_connection.h +++ /dev/null @@ -1,64 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#pragma once - -#include -#include - -#include - - -namespace zilliz { -namespace vecwise { -namespace engine { - -struct GroupSchema { - size_t id; - std::string group_id; - size_t files_cnt = 0; - uint16_t dimension; - std::string location = ""; - std::string next_file_location = ""; -}; // GroupSchema - - -struct GroupFileSchema { - typedef enum { - RAW, - INDEX - } FILE_TYPE; - - size_t id; - std::string group_id; - std::string file_id; - int files_type = RAW; - size_t rows; - std::string location = ""; -}; // GroupFileSchema - -inline auto initStorage(const std::string &path) { - using namespace sqlite_orm; - return make_storage(path, - // Add table below - make_table("Groups", - make_column("id", &GroupSchema::id, primary_key()), - make_column("group_id", &GroupSchema::group_id, unique()), - make_column("dimension", &GroupSchema::dimension), - make_column("files_cnt", &GroupSchema::files_cnt, default_value(0)))); -} - -using SqliteDB = decltype(initStorage("")); -using SqliteDBPtr= std::shared_ptr; - -class Connection { - protected: - static SqliteDBPtr connect_; -}; - -} -} -} diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index c2d37a5e..381fa28e 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -68,11 +68,11 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; - group_info.group_id = group_name; + group_info.table_id = group_name; engine::Status stat = db_->add_group(group_info); engine::meta::TableSchema group_info_get; - group_info_get.group_id = group_name; + group_info_get.table_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); @@ -113,11 +113,11 @@ TEST_F(DBTest, DB_TEST) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; - group_info.group_id = group_name; + group_info.table_id = group_name; engine::Status stat = db_->add_group(group_info); engine::meta::TableSchema group_info_get; - group_info_get.group_id = group_name; + group_info_get.table_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); @@ -199,11 +199,11 @@ TEST_F(DBTest, SEARCH_TEST) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; - group_info.group_id = group_name; + group_info.table_id = group_name; engine::Status stat = db_->add_group(group_info); engine::meta::TableSchema group_info_get; - group_info_get.group_id = group_name; + group_info_get.table_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 62b1fce1..2f1bd4fb 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -18,10 +18,10 @@ using namespace zilliz::vecwise::engine; TEST_F(MetaTest, GROUP_TEST) { - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl_->add_group(group); ASSERT_TRUE(status.ok()); @@ -30,26 +30,26 @@ TEST_F(MetaTest, GROUP_TEST) { status = impl_->get_group(group); ASSERT_TRUE(status.ok()); ASSERT_EQ(group.id, gid); - ASSERT_EQ(group.group_id, group_id); + ASSERT_EQ(group.table_id, table_id); - group.group_id = "not_found"; + group.table_id = "not_found"; status = impl_->get_group(group); ASSERT_TRUE(!status.ok()); - group.group_id = group_id; + group.table_id = table_id; status = impl_->add_group(group); ASSERT_TRUE(!status.ok()); } TEST_F(MetaTest, GROUP_FILE_TEST) { - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl_->add_group(group); meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; status = impl_->add_group_file(group_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW); @@ -65,14 +65,14 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { meta::DatesT dates; dates.push_back(meta::Meta::GetDate()); - status = impl_->delete_group_partitions(group_file.group_id, dates); + status = impl_->delete_group_partitions(group_file.table_id, dates); ASSERT_FALSE(status.ok()); dates.clear(); for (auto i=2; i < 10; ++i) { dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); } - status = impl_->delete_group_partitions(group_file.group_id, dates); + status = impl_->delete_group_partitions(group_file.table_id, dates); ASSERT_TRUE(status.ok()); group_file.date = meta::Meta::GetDateWithDelta(-2); @@ -83,9 +83,9 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { dates.clear(); dates.push_back(group_file.date); - status = impl_->delete_group_partitions(group_file.group_id, dates); + status = impl_->delete_group_partitions(group_file.table_id, dates); ASSERT_TRUE(status.ok()); - status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); + status = impl_->get_group_file(group_file.table_id, group_file.file_id, group_file); ASSERT_TRUE(status.ok()); ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE); } @@ -100,15 +100,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { options.archive_conf = ArchiveConf("delete", ss.str()); auto impl = meta::DBMetaImpl(options); - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl.add_group(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -127,7 +127,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { int i = 0; for (auto file : files) { - status = impl.get_group_file(file.group_id, file.file_id, file); + status = impl.get_group_file(file.table_id, file.file_id, file); ASSERT_TRUE(status.ok()); if (days[i] < days_num) { ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW); @@ -146,15 +146,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { options.archive_conf = ArchiveConf("delete", "disk:11"); auto impl = meta::DBMetaImpl(options); - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl.add_group(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; auto cnt = 10; auto each_size = 2UL; @@ -170,7 +170,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { int i = 0; for (auto file : files) { - status = impl.get_group_file(file.group_id, file.file_id, file); + status = impl.get_group_file(file.table_id, file.file_id, file); ASSERT_TRUE(status.ok()); if (i < 5) { ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE); @@ -184,10 +184,10 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { } TEST_F(MetaTest, GROUP_FILES_TEST) { - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl_->add_group(group); int new_files_cnt = 4; @@ -196,7 +196,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; for (auto i=0; iadd_group_file(group_file); @@ -229,7 +229,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->files_to_merge(group.group_id, dated_files); + status = impl_->files_to_merge(group.table_id, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt); @@ -238,7 +238,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatesT dates = {group_file.date}; - status = impl_->files_to_search(group_id, dates, dated_files); + status = impl_->files_to_search(table_id, dates, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[group_file.date].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); -- GitLab