Merge branch 'branch-0.4.0' into 'branch-0.4.0'

MS-461 Mysql meta unittest failed

See merge request megasearch/milvus!468

Former-commit-id: d9c105ca5fc1d045867b365eb94030c33368a8d1

cpp/CHANGELOG.md

@@ -19,6 +19,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-436 - Delete vectors failed if index created with index_type: IVF_FLAT/IVF_SQ8
 - MS-450 - server hang after run stop_server.sh
 - MS-449 - Add vectors twice success, once with ids, the other no ids
+- MS-461 - Mysql meta unittest failed
 
 ## Improvement
 - MS-327 - Clean code for milvus

cpp/src/db/DBImpl.cpp

@@ -645,14 +645,18 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
 
     try {
         //step 1: load index
-        to_index->Load(options_.insert_cache_immediately_);
+        Status status = to_index->Load(options_.insert_cache_immediately_);
+        if (!status.ok()) {
+            ENGINE_LOG_ERROR << "Failed to load index file: " << status.ToString();
+            return status;
+        }
 
         //step 2: create table file
         meta::TableFileSchema table_file;
         table_file.table_id_ = file.table_id_;
         table_file.date_ = file.date_;
         table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
-        Status status = meta_ptr_->CreateTableFile(table_file);
+        status = meta_ptr_->CreateTableFile(table_file);
         if (!status.ok()) {
             ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
             return status;
@@ -664,6 +668,14 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
         try {
             server::CollectBuildIndexMetrics metrics;
             index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
+            if (index == nullptr) {
+                table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
+                status = meta_ptr_->UpdateTableFile(table_file);
+                ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
+
+                return status;
+            }
+
         } catch (std::exception& ex) {
             //typical error: out of gpu memory
             std::string msg = "BuildIndex encounter exception: " + std::string(ex.what());

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -132,7 +132,9 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
             server::CollectExecutionEngineMetrics metrics(physical_size);
             index_ = read_index(location_);
             if(index_ == nullptr) {
-                ENGINE_LOG_ERROR << "Failed to load index from " << location_;
+                std::string msg = "Failed to load index from " + location_;
+                ENGINE_LOG_ERROR << msg;
+                return Status::Error(msg);
             } else {
                 ENGINE_LOG_DEBUG << "Disk io from: " << location_;
             }

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -144,7 +144,7 @@ Status MySQLMetaImpl::Initialize() {
                                 "dimension SMALLINT NOT NULL, " <<
                                 "created_on BIGINT NOT NULL, " <<
                                 "flag BIGINT DEFAULT 0 NOT NULL, " <<
-                                "index_file_size INT DEFAULT 1024 NOT NULL, " <<
+                                "index_file_size BIGINT DEFAULT 1024 NOT NULL, " <<
                                 "engine_type INT DEFAULT 1 NOT NULL, " <<
                                 "nlist INT DEFAULT 16384 NOT NULL, " <<
                                 "metric_type INT DEFAULT 1 NOT NULL);";
@@ -291,11 +291,16 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) {
             std::string state = std::to_string(table_schema.state_);
             std::string dimension = std::to_string(table_schema.dimension_);
             std::string created_on = std::to_string(table_schema.created_on_);
+            std::string flag = std::to_string(table_schema.flag_);
+            std::string index_file_size = std::to_string(table_schema.index_file_size_);
             std::string engine_type = std::to_string(table_schema.engine_type_);
+            std::string nlist = std::to_string(table_schema.nlist_);
+            std::string metric_type = std::to_string(table_schema.metric_type_);
 
             createTableQuery << "INSERT INTO Tables VALUES" <<
                              "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " <<
-                             created_on << ", " << engine_type << ");";
+                             created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " <<
+                             nlist << ", " << metric_type << ");";
 
             ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
 
@@ -904,6 +909,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         std::string engine_type = std::to_string(file_schema.engine_type_);
         std::string file_id = file_schema.file_id_;
         std::string file_type = std::to_string(file_schema.file_type_);
+        std::string file_size = std::to_string(file_schema.file_size_);
         std::string row_count = std::to_string(file_schema.row_count_);
         std::string updated_time = std::to_string(file_schema.updated_time_);
         std::string created_on = std::to_string(file_schema.created_on_);
@@ -920,8 +926,8 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
 
             createTableFileQuery << "INSERT INTO TableFiles VALUES" <<
                                  "(" << id << ", " << quote << table_id << ", " << engine_type << ", " <<
-                                 quote << file_id << ", " << file_type << ", " << row_count << ", " <<
-                                 updated_time << ", " << created_on << ", " << date << ");";
+                                 quote << file_id << ", " << file_type << ", " << file_size << ", " <<
+                                 row_count << ", " << updated_time << ", " << created_on << ", " << date << ");";
 
             ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str();
 
@@ -1170,7 +1176,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
             }
 
             Query filesToMergeQuery = connectionPtr->query();
-            filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, create_on " <<
+            filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on " <<
                               "FROM TableFiles " <<
                               "WHERE table_id = " << quote << table_id << " AND " <<
                               "file_type = " << std::to_string(TableFileSchema::RAW) << " " <<

cpp/unittest/db/db_tests.cpp

@@ -141,7 +141,6 @@ TEST_F(DBTest, CONFIG_TEST) {
 
 
 TEST_F(DBTest, DB_TEST) {
-    db_->Open(GetOptions(), &db_);
     engine::meta::TableSchema table_info = BuildTableSchema();
     engine::Status stat = db_->CreateTable(table_info);
 

cpp/unittest/db/mysql_db_test.cpp

@@ -46,11 +46,7 @@ namespace {
 }
 
 
-TEST_F(DISABLED_MySQLDBTest, DB_TEST) {
-
-    auto options = GetOptions();
-    auto db_ = engine::DBFactory::Build(options);
-
+TEST_F(MySQLDBTest, DB_TEST) {
     engine::meta::TableSchema table_info = BuildTableSchema();
     engine::Status stat = db_->CreateTable(table_info);
 
@@ -115,6 +111,8 @@ TEST_F(DISABLED_MySQLDBTest, DB_TEST) {
             ASSERT_TRUE(count >= prev_count);
             std::this_thread::sleep_for(std::chrono::seconds(3));
         }
+
+        std::cout << "Search AAA done" << std::endl;
     });
 
     int loop = INSERT_LOOP;
@@ -131,18 +129,9 @@ TEST_F(DISABLED_MySQLDBTest, DB_TEST) {
     }
 
     search.join();
-
-    delete db_;
-
-    auto dummyDB = engine::DBFactory::Build(options);
-    dummyDB->DropAll();
-    delete dummyDB;
 };
 
-TEST_F(DISABLED_MySQLDBTest, SEARCH_TEST) {
-    auto options = GetOptions();
-    auto db_ = engine::DBFactory::Build(options);
-
+TEST_F(MySQLDBTest, SEARCH_TEST) {
     engine::meta::TableSchema table_info = BuildTableSchema();
     engine::Status stat = db_->CreateTable(table_info);
 
@@ -192,22 +181,9 @@ TEST_F(DISABLED_MySQLDBTest, SEARCH_TEST) {
     engine::QueryResults results;
     stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
     ASSERT_STATS(stat);
-
-    delete db_;
-
-    auto dummyDB = engine::DBFactory::Build(options);
-    dummyDB->DropAll();
-    delete dummyDB;
-
-    // TODO(linxj): add groundTruth assert
 };
 
-TEST_F(DISABLED_MySQLDBTest, ARHIVE_DISK_CHECK) {
-
-    auto options = GetOptions();
-    options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1");
-    auto db_ = engine::DBFactory::Build(options);
-
+TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) {
     engine::meta::TableSchema table_info = BuildTableSchema();
     engine::Status stat = db_->CreateTable(table_info);
 
@@ -250,20 +226,9 @@ TEST_F(DISABLED_MySQLDBTest, ARHIVE_DISK_CHECK) {
     db_->Size(size);
     LOG(DEBUG) << "size=" << size;
     ASSERT_LE(size, 1 * engine::meta::G);
-
-    delete db_;
-
-    auto dummyDB = engine::DBFactory::Build(options);
-    dummyDB->DropAll();
-    delete dummyDB;
 };
 
-TEST_F(DISABLED_MySQLDBTest, DELETE_TEST) {
-
-    auto options = GetOptions();
-    options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1");
-    auto db_ = engine::DBFactory::Build(options);
-
+TEST_F(MySQLDBTest, DELETE_TEST) {
     engine::meta::TableSchema table_info = BuildTableSchema();
     engine::Status stat = db_->CreateTable(table_info);
 //    std::cout << stat.ToString() << std::endl;
@@ -301,10 +266,4 @@ TEST_F(DISABLED_MySQLDBTest, DELETE_TEST) {
 
     db_->HasTable(TABLE_NAME, has_table);
     ASSERT_FALSE(has_table);
-
-    delete db_;
-
-    auto dummyDB = engine::DBFactory::Build(options);
-    dummyDB->DropAll();
-    delete dummyDB;
 };

cpp/unittest/db/mysql_meta_test.cpp

@@ -21,7 +21,7 @@
 
 using namespace zilliz::milvus::engine;
 
-TEST_F(DISABLED_MySQLTest, TABLE_TEST) {
+TEST_F(MySQLTest, TABLE_TEST) {
     DBMetaOptions options;
     try {
         options = getDBMetaOptions();
@@ -53,7 +53,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_TEST) {
 
     table.table_id_ = table_id;
     status = impl.CreateTable(table);
-    ASSERT_TRUE(status.ok());
+    ASSERT_TRUE(status.IsAlreadyExist());
 
     table.table_id_ = "";
     status = impl.CreateTable(table);
@@ -63,7 +63,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_TEST) {
     ASSERT_TRUE(status.ok());
 }
 
-TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) {
+TEST_F(MySQLTest, TABLE_FILE_TEST) {
     DBMetaOptions options;
     try {
         options = getDBMetaOptions();
@@ -92,7 +92,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) {
     meta::DatesT dates;
     dates.push_back(utils::GetDate());
     status = impl.DropPartitionsByDates(table_file.table_id_, dates);
-    ASSERT_FALSE(status.ok());
+    ASSERT_TRUE(status.ok());
 
     uint64_t cnt = 0;
     status = impl.Count(table_id, cnt);
@@ -139,7 +139,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) {
     ASSERT_TRUE(status.ok());
 }
 
-TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) {
+TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
     srand(time(0));
     DBMetaOptions options;
     try {
@@ -211,7 +211,7 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) {
     ASSERT_TRUE(status.ok());
 }
 
-TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DISK) {
+TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
     DBMetaOptions options;
     try {
         options = getDBMetaOptions();
@@ -269,7 +269,7 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DISK) {
     ASSERT_TRUE(status.ok());
 }
 
-TEST_F(DISABLED_MySQLTest, TABLE_FILES_TEST) {
+TEST_F(MySQLTest, TABLE_FILES_TEST) {
     DBMetaOptions options;
     try {
         options = getDBMetaOptions();

cpp/unittest/db/utils.cpp

@@ -103,7 +103,7 @@ void MetaTest::TearDown() {
     impl_->DropAll();
 }
 
-zilliz::milvus::engine::DBMetaOptions DISABLED_MySQLTest::getDBMetaOptions() {
+zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() {
 //    std::string path = "/tmp/milvus_test";
 //    engine::DBMetaOptions options = engine::DBMetaOptionsFactory::Build(path);
     zilliz::milvus::engine::DBMetaOptions options;
@@ -111,17 +111,16 @@ zilliz::milvus::engine::DBMetaOptions DISABLED_MySQLTest::getDBMetaOptions() {
     options.backend_uri = DBTestEnvironment::getURI();
 
     if(options.backend_uri.empty()) {
-//        throw std::exception();
         options.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/";
     }
 
     return options;
 }
 
-zilliz::milvus::engine::Options DISABLED_MySQLDBTest::GetOptions() {
+zilliz::milvus::engine::Options MySQLDBTest::GetOptions() {
     auto options = engine::OptionsFactory::Build();
     options.meta.path = "/tmp/milvus_test";
-    options.meta.backend_uri = DBTestEnvironment::getURI();
+    options.meta.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/";
     return options;
 }
 

cpp/unittest/db/utils.h

@@ -79,13 +79,13 @@ class MetaTest : public DBTest {
     virtual void TearDown() override;
 };
 
-class DISABLED_MySQLTest : public ::testing::Test {
+class MySQLTest : public ::testing::Test {
  protected:
 //    std::shared_ptr<zilliz::milvus::engine::meta::MySQLMetaImpl> impl_;
     zilliz::milvus::engine::DBMetaOptions getDBMetaOptions();
 };
 
-class DISABLED_MySQLDBTest : public ::testing::Test {
+class MySQLDBTest : public DBTest {
  protected:
     zilliz::milvus::engine::Options GetOptions();
 };