From e9872c2ce8c112c0de96d8dcb00590cd722548d3 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 9 Sep 2019 12:42:40 +0800 Subject: [PATCH] add testcase for meta Former-commit-id: ac6954959252f747d89a05f7af2b09a8d8e224ab --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/meta/Meta.h | 2 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 35 +++--- cpp/src/db/meta/MySQLMetaImpl.h | 2 +- cpp/src/db/meta/SqliteMetaImpl.cpp | 2 +- cpp/src/db/meta/SqliteMetaImpl.h | 2 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 12 +- cpp/unittest/db/meta_tests.cpp | 110 ++++++++++++++++++- cpp/unittest/db/mysql_meta_test.cpp | 109 +++++++++++++++++- 9 files changed, 239 insertions(+), 37 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a443e81f..add4b898 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -257,7 +257,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) if(!utils::IsSameIndex(old_index, new_index)) { DropIndex(table_id); - status = meta_ptr_->UpdateTableIndexParam(table_id, new_index); + status = meta_ptr_->UpdateTableIndex(table_id, new_index); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id; return status; diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 8d5dd63a..b59ca3de 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -32,7 +32,7 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; - virtual Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) = 0; + virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 13c3597b..d62e5235 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -419,7 +419,7 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, return Status::OK(); } -Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) { +Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) { try { server::MetricCollector metric; @@ -436,7 +436,7 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndexParam: " << updateTableIndexParamQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); StoreQueryResult res = updateTableIndexParamQuery.store(); @@ -453,12 +453,12 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T "state = " << state << ", " << "dimension = " << dimension << ", " << "created_on = " << created_on << ", " << - "engine_type_ = " << index.engine_type_ << ", " << + "engine_type = " << index.engine_type_ << ", " << "nlist = " << index.nlist_ << ", " << "metric_type = " << index.metric_type_ << " " << - "WHERE id = " << quote << table_id << ";"; + "WHERE table_id = " << quote << table_id << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndexParam: " << updateTableIndexParamQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); if (!updateTableIndexParamQuery.exec()) { @@ -497,7 +497,7 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) Query updateTableFlagQuery = connectionPtr->query(); updateTableFlagQuery << "UPDATE Tables " << "SET flag = " << flag << " " << - "WHERE id = " << quote << table_id << ";"; + "WHERE table_id = " << quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); @@ -608,7 +608,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { //set table index type to raw dropTableIndexQuery << "UPDATE Tables " << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << - "nlist = " << std::to_string(DEFAULT_NLIST) << " " << + "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -725,7 +725,8 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, state, dimension, engine_type, nlist, index_file_size, metric_type " << + describeTableQuery << "SELECT id, state, dimension, created_on, " << + "flag, index_file_size, engine_type, nlist, metric_type " << "FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -744,6 +745,10 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.dimension_ = resRow["dimension"]; + table_schema.created_on_ = resRow["created_on"]; + + table_schema.flag_ = resRow["flag"]; + table_schema.index_file_size_ = resRow["index_file_size"]; table_schema.engine_type_ = resRow["engine_type"]; @@ -1927,7 +1932,7 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { Query countQuery = connectionPtr->query(); - countQuery << "SELECT size " << + countQuery << "SELECT row_count " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << @@ -1941,20 +1946,10 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { result = 0; for (auto &resRow : res) { - size_t size = resRow["size"]; + size_t size = resRow["row_count"]; result += size; } - if (table_schema.dimension_ <= 0) { - std::stringstream errorMsg; - errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) - << ", table_id = " << table_id; - ENGINE_LOG_ERROR << errorMsg.str(); - return Status(DB_ERROR, errorMsg.str()); - } - result /= table_schema.dimension_; - result /= sizeof(float); - } catch (const BadQuery &e) { // Handle any query errors return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 08897cbe..64c703b6 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -51,7 +51,7 @@ class MySQLMetaImpl : public Meta { const std::vector &file_types, std::vector &file_ids) override; - Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag); diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 51297a76..559ffd29 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -339,7 +339,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, return Status::OK(); } -Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) { +Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) { try { server::MetricCollector metric; diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index ad0d21f5..641dcc40 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -46,7 +46,7 @@ class SqliteMetaImpl : public Meta { const std::vector &file_types, std::vector &file_ids) override; - Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 69cc0429..a29ceb88 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -953,8 +953,18 @@ DropIndexTask::OnExecute() { return SetError(res, "Invalid table name: " + table_name_); } + bool has_table = false; + auto stat = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + } + + if (!has_table) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } + //step 2: check table existence - auto stat = DBWrapper::DB()->DropIndex(table_name_); + stat = DBWrapper::DB()->DropIndex(table_name_); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 193e1ac7..50d92079 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -213,14 +213,36 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { table.table_id_ = table_id; auto status = impl_->CreateTable(table); - int new_files_cnt = 4; - int raw_files_cnt = 5; - int to_index_files_cnt = 6; - int index_files_cnt = 7; + uint64_t new_merge_files_cnt = 1; + uint64_t new_index_files_cnt = 2; + uint64_t backup_files_cnt = 3; + uint64_t new_files_cnt = 4; + uint64_t raw_files_cnt = 5; + uint64_t to_index_files_cnt = 6; + uint64_t index_files_cnt = 7; meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; + for (auto i=0; iCreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::NEW_MERGE; + status = impl_->UpdateTableFile(table_file); + } + + for (auto i=0; iCreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; + status = impl_->UpdateTableFile(table_file); + } + + for (auto i=0; iCreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::BACKUP; + table_file.row_count_ = 1; + status = impl_->UpdateTableFile(table_file); + } + for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::NEW; @@ -230,23 +252,30 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::RAW; + table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::TO_INDEX; + table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::INDEX; + table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - meta::TableFilesSchema files; + uint64_t total_row_count = 0; + status = impl_->Count(table_id, total_row_count); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt); + meta::TableFilesSchema files; status = impl_->FilesToIndex(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); @@ -281,4 +310,75 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { status = impl_->FilesToSearch(table_id, ids, dates, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(),0); + + std::vector file_types; + std::vector file_ids; + status = impl_->FilesByType(table.table_id_, file_types, file_ids); + ASSERT_TRUE(file_ids.empty()); + ASSERT_FALSE(status.ok()); + + file_types = { + meta::TableFileSchema::NEW, + meta::TableFileSchema::NEW_MERGE, + meta::TableFileSchema::NEW_INDEX, + meta::TableFileSchema::TO_INDEX, + meta::TableFileSchema::INDEX, + meta::TableFileSchema::RAW, + meta::TableFileSchema::BACKUP, + }; + status = impl_->FilesByType(table.table_id_, file_types, file_ids); + ASSERT_TRUE(status.ok()); + uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + + backup_files_cnt + new_files_cnt + raw_files_cnt + + to_index_files_cnt + index_files_cnt; + ASSERT_EQ(file_ids.size(), total_cnt); + + status = impl_->DeleteTableFiles(table_id); + ASSERT_TRUE(status.ok()); + + status = impl_->DeleteTable(table_id); + ASSERT_TRUE(status.ok()); + + status = impl_->CleanUpFilesWithTTL(1UL); + ASSERT_TRUE(status.ok()); } + +TEST_F(MetaTest, INDEX_TEST) { + auto table_id = "index_test"; + + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); + + TableIndex index; + index.metric_type_ = 2; + index.nlist_ = 1234; + index.engine_type_ = 3; + status = impl_->UpdateTableIndex(table_id, index); + ASSERT_TRUE(status.ok()); + + int64_t flag = 65536; + status = impl_->UpdateTableFlag(table_id, flag); + ASSERT_TRUE(status.ok()); + + engine::meta::TableSchema table_info; + table_info.table_id_ = table_id; + status = impl_->DescribeTable(table_info); + ASSERT_EQ(table_info.flag_, flag); + + TableIndex index_out; + status = impl_->DescribeTableIndex(table_id, index_out); + ASSERT_EQ(index_out.metric_type_, index.metric_type_); + ASSERT_EQ(index_out.nlist_, index.nlist_); + ASSERT_EQ(index_out.engine_type_, index.engine_type_); + + status = impl_->DropTableIndex(table_id); + ASSERT_TRUE(status.ok()); + status = impl_->DescribeTableIndex(table_id, index_out); + ASSERT_NE(index_out.metric_type_, index.metric_type_); + ASSERT_NE(index_out.nlist_, index.nlist_); + ASSERT_NE(index_out.engine_type_, index.engine_type_); + + status = impl_->UpdateTableFilesToIndex(table_id); + ASSERT_TRUE(status.ok()); +} \ No newline at end of file diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index aa0fa176..5d692347 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -238,14 +238,36 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { table.table_id_ = table_id; auto status = impl_->CreateTable(table); - int new_files_cnt = 4; - int raw_files_cnt = 5; - int to_index_files_cnt = 6; - int index_files_cnt = 7; + uint64_t new_merge_files_cnt = 1; + uint64_t new_index_files_cnt = 2; + uint64_t backup_files_cnt = 3; + uint64_t new_files_cnt = 4; + uint64_t raw_files_cnt = 5; + uint64_t to_index_files_cnt = 6; + uint64_t index_files_cnt = 7; meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; + for (auto i=0; iCreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::NEW_MERGE; + status = impl_->UpdateTableFile(table_file); + } + + for (auto i=0; iCreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; + status = impl_->UpdateTableFile(table_file); + } + + for (auto i=0; iCreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::BACKUP; + table_file.row_count_ = 1; + status = impl_->UpdateTableFile(table_file); + } + for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::NEW; @@ -255,23 +277,30 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::RAW; + table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::TO_INDEX; + table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type_ = meta::TableFileSchema::INDEX; + table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - meta::TableFilesSchema files; + uint64_t total_row_count = 0; + status = impl_->Count(table_id, total_row_count); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt); + meta::TableFilesSchema files; status = impl_->FilesToIndex(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); @@ -307,6 +336,74 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(),0); - status = impl_->DropAll(); + std::vector file_types; + std::vector file_ids; + status = impl_->FilesByType(table.table_id_, file_types, file_ids); + ASSERT_TRUE(file_ids.empty()); + ASSERT_FALSE(status.ok()); + + file_types = { + meta::TableFileSchema::NEW, + meta::TableFileSchema::NEW_MERGE, + meta::TableFileSchema::NEW_INDEX, + meta::TableFileSchema::TO_INDEX, + meta::TableFileSchema::INDEX, + meta::TableFileSchema::RAW, + meta::TableFileSchema::BACKUP, + }; + status = impl_->FilesByType(table.table_id_, file_types, file_ids); + ASSERT_TRUE(status.ok()); + uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + + backup_files_cnt + new_files_cnt + raw_files_cnt + + to_index_files_cnt + index_files_cnt; + ASSERT_EQ(file_ids.size(), total_cnt); + + status = impl_->DeleteTableFiles(table_id); + ASSERT_TRUE(status.ok()); + + status = impl_->DeleteTable(table_id); + ASSERT_TRUE(status.ok()); + + status = impl_->CleanUpFilesWithTTL(1UL); + ASSERT_TRUE(status.ok()); +} + +TEST_F(MySqlMetaTest, INDEX_TEST) { + auto table_id = "index_test"; + + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); + + TableIndex index; + index.metric_type_ = 2; + index.nlist_ = 1234; + index.engine_type_ = 3; + status = impl_->UpdateTableIndex(table_id, index); + ASSERT_TRUE(status.ok()); + + int64_t flag = 65536; + status = impl_->UpdateTableFlag(table_id, flag); + ASSERT_TRUE(status.ok()); + + engine::meta::TableSchema table_info; + table_info.table_id_ = table_id; + status = impl_->DescribeTable(table_info); + ASSERT_EQ(table_info.flag_, flag); + + TableIndex index_out; + status = impl_->DescribeTableIndex(table_id, index_out); + ASSERT_EQ(index_out.metric_type_, index.metric_type_); + ASSERT_EQ(index_out.nlist_, index.nlist_); + ASSERT_EQ(index_out.engine_type_, index.engine_type_); + + status = impl_->DropTableIndex(table_id); + ASSERT_TRUE(status.ok()); + status = impl_->DescribeTableIndex(table_id, index_out); + ASSERT_NE(index_out.metric_type_, index.metric_type_); + ASSERT_NE(index_out.nlist_, index.nlist_); + ASSERT_NE(index_out.engine_type_, index.engine_type_); + + status = impl_->UpdateTableFilesToIndex(table_id); ASSERT_TRUE(status.ok()); } -- GitLab