Merge remote-tracking branch 'main/branch-0.4.0' into branch-0.4.0

Former-commit-id: 7b28c30b2b49f448383357a889419ab18596e86e

cpp/src/db/DBImpl.cpp

@@ -257,7 +257,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
         if(!utils::IsSameIndex(old_index, new_index)) {
             DropIndex(table_id);
 
-            status = meta_ptr_->UpdateTableIndexParam(table_id, new_index);
+            status = meta_ptr_->UpdateTableIndex(table_id, new_index);
             if (!status.ok()) {
                 ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
                 return status;

cpp/src/db/meta/Meta.h

@@ -32,7 +32,7 @@ class Meta {
 
     virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0;
 
-    virtual Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) = 0;
+    virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0;
 
     virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
 

cpp/src/db/meta/MySQLMetaImpl.h

@@ -51,7 +51,7 @@ class MySQLMetaImpl : public Meta {
                        const std::vector<int> &file_types,
                        std::vector<std::string> &file_ids) override;
 
-    Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override;
+    Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override;
 
     Status UpdateTableFlag(const std::string &table_id, int64_t flag);
 

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -339,7 +339,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id,
     return Status::OK();
 }
 
-Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) {
+Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) {
     try {
         server::MetricCollector metric;
 

cpp/src/db/meta/SqliteMetaImpl.h

@@ -46,7 +46,7 @@ class SqliteMetaImpl : public Meta {
                        const std::vector<int> &file_types,
                        std::vector<std::string> &file_ids) override;
 
-    Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override;
+    Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override;
 
     Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
 

cpp/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -953,8 +953,18 @@ DropIndexTask::OnExecute() {
             return SetError(res, "Invalid table name: " + table_name_);
         }
 
+        bool has_table = false;
+        auto stat = DBWrapper::DB()->HasTable(table_name_, has_table);
+        if (!stat.ok()) {
+            return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
+        }
+
+        if (!has_table) {
+            return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+        }
+
         //step 2: check table existence
-        auto stat = DBWrapper::DB()->DropIndex(table_name_);
+        stat = DBWrapper::DB()->DropIndex(table_name_);
         if (!stat.ok()) {
             return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
         }

cpp/unittest/db/meta_tests.cpp

@@ -213,14 +213,36 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
     table.table_id_ = table_id;
     auto status = impl_->CreateTable(table);
 
-    int new_files_cnt = 4;
-    int raw_files_cnt = 5;
-    int to_index_files_cnt = 6;
-    int index_files_cnt = 7;
+    uint64_t new_merge_files_cnt = 1;
+    uint64_t new_index_files_cnt = 2;
+    uint64_t backup_files_cnt = 3;
+    uint64_t new_files_cnt = 4;
+    uint64_t raw_files_cnt = 5;
+    uint64_t to_index_files_cnt = 6;
+    uint64_t index_files_cnt = 7;
 
     meta::TableFileSchema table_file;
     table_file.table_id_ = table.table_id_;
 
+    for (auto i=0; i<new_merge_files_cnt; ++i) {
+        status = impl_->CreateTableFile(table_file);
+        table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
+        status = impl_->UpdateTableFile(table_file);
+    }
+
+    for (auto i=0; i<new_index_files_cnt; ++i) {
+        status = impl_->CreateTableFile(table_file);
+        table_file.file_type_ = meta::TableFileSchema::NEW_INDEX;
+        status = impl_->UpdateTableFile(table_file);
+    }
+
+    for (auto i=0; i<backup_files_cnt; ++i) {
+        status = impl_->CreateTableFile(table_file);
+        table_file.file_type_ = meta::TableFileSchema::BACKUP;
+        table_file.row_count_ = 1;
+        status = impl_->UpdateTableFile(table_file);
+    }
+
     for (auto i=0; i<new_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::NEW;
@@ -230,23 +252,30 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
     for (auto i=0; i<raw_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::RAW;
+        table_file.row_count_ = 1;
         status = impl_->UpdateTableFile(table_file);
     }
 
     for (auto i=0; i<to_index_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
+        table_file.row_count_ = 1;
         status = impl_->UpdateTableFile(table_file);
     }
 
     for (auto i=0; i<index_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::INDEX;
+        table_file.row_count_ = 1;
         status = impl_->UpdateTableFile(table_file);
     }
 
-    meta::TableFilesSchema files;
+    uint64_t total_row_count = 0;
+    status = impl_->Count(table_id, total_row_count);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt);
 
+    meta::TableFilesSchema files;
     status = impl_->FilesToIndex(files);
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(files.size(), to_index_files_cnt);
@@ -281,4 +310,75 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
     status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(dated_files[table_file.date_].size(),0);
+
+    std::vector<int> file_types;
+    std::vector<std::string> file_ids;
+    status = impl_->FilesByType(table.table_id_, file_types, file_ids);
+    ASSERT_TRUE(file_ids.empty());
+    ASSERT_FALSE(status.ok());
+
+    file_types = {
+            meta::TableFileSchema::NEW,
+            meta::TableFileSchema::NEW_MERGE,
+            meta::TableFileSchema::NEW_INDEX,
+            meta::TableFileSchema::TO_INDEX,
+            meta::TableFileSchema::INDEX,
+            meta::TableFileSchema::RAW,
+            meta::TableFileSchema::BACKUP,
+    };
+    status = impl_->FilesByType(table.table_id_, file_types, file_ids);
+    ASSERT_TRUE(status.ok());
+    uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
+                         backup_files_cnt + new_files_cnt + raw_files_cnt +
+                         to_index_files_cnt + index_files_cnt;
+    ASSERT_EQ(file_ids.size(), total_cnt);
+
+    status = impl_->DeleteTableFiles(table_id);
+    ASSERT_TRUE(status.ok());
+
+    status = impl_->DeleteTable(table_id);
+    ASSERT_TRUE(status.ok());
+
+    status = impl_->CleanUpFilesWithTTL(1UL);
+    ASSERT_TRUE(status.ok());
 }
+
+TEST_F(MetaTest, INDEX_TEST) {
+    auto table_id = "index_test";
+
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl_->CreateTable(table);
+
+    TableIndex index;
+    index.metric_type_ = 2;
+    index.nlist_ = 1234;
+    index.engine_type_ = 3;
+    status = impl_->UpdateTableIndex(table_id, index);
+    ASSERT_TRUE(status.ok());
+
+    int64_t flag = 65536;
+    status = impl_->UpdateTableFlag(table_id, flag);
+    ASSERT_TRUE(status.ok());
+
+    engine::meta::TableSchema table_info;
+    table_info.table_id_ = table_id;
+    status = impl_->DescribeTable(table_info);
+    ASSERT_EQ(table_info.flag_, flag);
+
+    TableIndex index_out;
+    status = impl_->DescribeTableIndex(table_id, index_out);
+    ASSERT_EQ(index_out.metric_type_, index.metric_type_);
+    ASSERT_EQ(index_out.nlist_, index.nlist_);
+    ASSERT_EQ(index_out.engine_type_, index.engine_type_);
+
+    status = impl_->DropTableIndex(table_id);
+    ASSERT_TRUE(status.ok());
+    status = impl_->DescribeTableIndex(table_id, index_out);
+    ASSERT_NE(index_out.metric_type_, index.metric_type_);
+    ASSERT_NE(index_out.nlist_, index.nlist_);
+    ASSERT_NE(index_out.engine_type_, index.engine_type_);
+
+    status = impl_->UpdateTableFilesToIndex(table_id);
+    ASSERT_TRUE(status.ok());
+}
\ No newline at end of file

cpp/unittest/db/mysql_meta_test.cpp

@@ -238,14 +238,36 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
     table.table_id_ = table_id;
     auto status = impl_->CreateTable(table);
 
-    int new_files_cnt = 4;
-    int raw_files_cnt = 5;
-    int to_index_files_cnt = 6;
-    int index_files_cnt = 7;
+    uint64_t new_merge_files_cnt = 1;
+    uint64_t new_index_files_cnt = 2;
+    uint64_t backup_files_cnt = 3;
+    uint64_t new_files_cnt = 4;
+    uint64_t raw_files_cnt = 5;
+    uint64_t to_index_files_cnt = 6;
+    uint64_t index_files_cnt = 7;
 
     meta::TableFileSchema table_file;
     table_file.table_id_ = table.table_id_;
 
+    for (auto i=0; i<new_merge_files_cnt; ++i) {
+        status = impl_->CreateTableFile(table_file);
+        table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
+        status = impl_->UpdateTableFile(table_file);
+    }
+
+    for (auto i=0; i<new_index_files_cnt; ++i) {
+        status = impl_->CreateTableFile(table_file);
+        table_file.file_type_ = meta::TableFileSchema::NEW_INDEX;
+        status = impl_->UpdateTableFile(table_file);
+    }
+
+    for (auto i=0; i<backup_files_cnt; ++i) {
+        status = impl_->CreateTableFile(table_file);
+        table_file.file_type_ = meta::TableFileSchema::BACKUP;
+        table_file.row_count_ = 1;
+        status = impl_->UpdateTableFile(table_file);
+    }
+
     for (auto i=0; i<new_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::NEW;
@@ -255,23 +277,30 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
     for (auto i=0; i<raw_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::RAW;
+        table_file.row_count_ = 1;
         status = impl_->UpdateTableFile(table_file);
     }
 
     for (auto i=0; i<to_index_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
+        table_file.row_count_ = 1;
         status = impl_->UpdateTableFile(table_file);
     }
 
     for (auto i=0; i<index_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
         table_file.file_type_ = meta::TableFileSchema::INDEX;
+        table_file.row_count_ = 1;
         status = impl_->UpdateTableFile(table_file);
     }
 
-    meta::TableFilesSchema files;
+    uint64_t total_row_count = 0;
+    status = impl_->Count(table_id, total_row_count);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt);
 
+    meta::TableFilesSchema files;
     status = impl_->FilesToIndex(files);
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(files.size(), to_index_files_cnt);
@@ -307,6 +336,74 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(dated_files[table_file.date_].size(),0);
 
-    status = impl_->DropAll();
+    std::vector<int> file_types;
+    std::vector<std::string> file_ids;
+    status = impl_->FilesByType(table.table_id_, file_types, file_ids);
+    ASSERT_TRUE(file_ids.empty());
+    ASSERT_FALSE(status.ok());
+
+    file_types = {
+            meta::TableFileSchema::NEW,
+            meta::TableFileSchema::NEW_MERGE,
+            meta::TableFileSchema::NEW_INDEX,
+            meta::TableFileSchema::TO_INDEX,
+            meta::TableFileSchema::INDEX,
+            meta::TableFileSchema::RAW,
+            meta::TableFileSchema::BACKUP,
+    };
+    status = impl_->FilesByType(table.table_id_, file_types, file_ids);
+    ASSERT_TRUE(status.ok());
+    uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
+            backup_files_cnt + new_files_cnt + raw_files_cnt +
+            to_index_files_cnt + index_files_cnt;
+    ASSERT_EQ(file_ids.size(), total_cnt);
+
+    status = impl_->DeleteTableFiles(table_id);
+    ASSERT_TRUE(status.ok());
+
+    status = impl_->DeleteTable(table_id);
+    ASSERT_TRUE(status.ok());
+
+    status = impl_->CleanUpFilesWithTTL(1UL);
+    ASSERT_TRUE(status.ok());
+}
+
+TEST_F(MySqlMetaTest, INDEX_TEST) {
+    auto table_id = "index_test";
+
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl_->CreateTable(table);
+
+    TableIndex index;
+    index.metric_type_ = 2;
+    index.nlist_ = 1234;
+    index.engine_type_ = 3;
+    status = impl_->UpdateTableIndex(table_id, index);
+    ASSERT_TRUE(status.ok());
+
+    int64_t flag = 65536;
+    status = impl_->UpdateTableFlag(table_id, flag);
+    ASSERT_TRUE(status.ok());
+
+    engine::meta::TableSchema table_info;
+    table_info.table_id_ = table_id;
+    status = impl_->DescribeTable(table_info);
+    ASSERT_EQ(table_info.flag_, flag);
+
+    TableIndex index_out;
+    status = impl_->DescribeTableIndex(table_id, index_out);
+    ASSERT_EQ(index_out.metric_type_, index.metric_type_);
+    ASSERT_EQ(index_out.nlist_, index.nlist_);
+    ASSERT_EQ(index_out.engine_type_, index.engine_type_);
+
+    status = impl_->DropTableIndex(table_id);
+    ASSERT_TRUE(status.ok());
+    status = impl_->DescribeTableIndex(table_id, index_out);
+    ASSERT_NE(index_out.metric_type_, index.metric_type_);
+    ASSERT_NE(index_out.nlist_, index.nlist_);
+    ASSERT_NE(index_out.engine_type_, index.engine_type_);
+
+    status = impl_->UpdateTableFilesToIndex(table_id);
     ASSERT_TRUE(status.ok());
 }