diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index d0d353cfa7b3881285db2b524bc2740e1e46496c..dfb10ed0cd33e153221abb2e5640b4247792dd5b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -19,6 +19,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-436 - Delete vectors failed if index created with index_type: IVF_FLAT/IVF_SQ8 - MS-450 - server hang after run stop_server.sh - MS-449 - Add vectors twice success, once with ids, the other no ids +- MS-461 - Mysql meta unittest failed ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 8a381b9dbfedf2e443722b09e97c6121079e41ea..64dcc7275eb5bd4a31595e076b2924708cc32b96 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -645,14 +645,18 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { try { //step 1: load index - to_index->Load(options_.insert_cache_immediately_); + Status status = to_index->Load(options_.insert_cache_immediately_); + if (!status.ok()) { + ENGINE_LOG_ERROR << "Failed to load index file: " << status.ToString(); + return status; + } //step 2: create table file meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path - Status status = meta_ptr_->CreateTableFile(table_file); + status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); return status; @@ -664,6 +668,14 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { try { server::CollectBuildIndexMetrics metrics; index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); + if (index == nullptr) { + table_file.file_type_ = meta::TableFileSchema::TO_DELETE; + status = meta_ptr_->UpdateTableFile(table_file); + ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + + return status; + } + } catch (std::exception& ex) { //typical error: out of gpu memory std::string msg = "BuildIndex encounter exception: " + std::string(ex.what()); diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index c2b0b352208f510c24f5bdc72c79f521e938d9d1..098823c482103236fce4569116830afde65813b9 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -132,7 +132,9 @@ Status ExecutionEngineImpl::Load(bool to_cache) { server::CollectExecutionEngineMetrics metrics(physical_size); index_ = read_index(location_); if(index_ == nullptr) { - ENGINE_LOG_ERROR << "Failed to load index from " << location_; + std::string msg = "Failed to load index from " + location_; + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); } else { ENGINE_LOG_DEBUG << "Disk io from: " << location_; } diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index beae3fa1e98a02305b812a63e99c2f4072305d72..dc1f931c031a2724beeffa17c6172ee904ce2f76 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -144,7 +144,7 @@ Status MySQLMetaImpl::Initialize() { "dimension SMALLINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "flag BIGINT DEFAULT 0 NOT NULL, " << - "index_file_size INT DEFAULT 1024 NOT NULL, " << + "index_file_size BIGINT DEFAULT 1024 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "nlist INT DEFAULT 16384 NOT NULL, " << "metric_type INT DEFAULT 1 NOT NULL);"; @@ -291,11 +291,16 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string state = std::to_string(table_schema.state_); std::string dimension = std::to_string(table_schema.dimension_); std::string created_on = std::to_string(table_schema.created_on_); + std::string flag = std::to_string(table_schema.flag_); + std::string index_file_size = std::to_string(table_schema.index_file_size_); std::string engine_type = std::to_string(table_schema.engine_type_); + std::string nlist = std::to_string(table_schema.nlist_); + std::string metric_type = std::to_string(table_schema.metric_type_); createTableQuery << "INSERT INTO Tables VALUES" << "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << engine_type << ");"; + created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " << + nlist << ", " << metric_type << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -904,6 +909,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; std::string file_type = std::to_string(file_schema.file_type_); + std::string file_size = std::to_string(file_schema.file_size_); std::string row_count = std::to_string(file_schema.row_count_); std::string updated_time = std::to_string(file_schema.updated_time_); std::string created_on = std::to_string(file_schema.created_on_); @@ -920,8 +926,8 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { createTableFileQuery << "INSERT INTO TableFiles VALUES" << "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << row_count << ", " << - updated_time << ", " << created_on << ", " << date << ");"; + quote << file_id << ", " << file_type << ", " << file_size << ", " << + row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); @@ -1170,7 +1176,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, } Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, create_on " << + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index f37896df33db7fb822fd9f649caa1a51fd89438a..acf69a9aa1e8c4dbdc3d01cea838ad115bc38fb6 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -141,7 +141,6 @@ TEST_F(DBTest, CONFIG_TEST) { TEST_F(DBTest, DB_TEST) { - db_->Open(GetOptions(), &db_); engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 78adf9f0f51a471b82a311e1e2a6ed5d9abd9819..cdad9b22752c68a4f4b0e1d17ee090de13dbabef 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -46,11 +46,7 @@ namespace { } -TEST_F(DISABLED_MySQLDBTest, DB_TEST) { - - auto options = GetOptions(); - auto db_ = engine::DBFactory::Build(options); - +TEST_F(MySQLDBTest, DB_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -115,6 +111,8 @@ TEST_F(DISABLED_MySQLDBTest, DB_TEST) { ASSERT_TRUE(count >= prev_count); std::this_thread::sleep_for(std::chrono::seconds(3)); } + + std::cout << "Search AAA done" << std::endl; }); int loop = INSERT_LOOP; @@ -131,18 +129,9 @@ TEST_F(DISABLED_MySQLDBTest, DB_TEST) { } search.join(); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; }; -TEST_F(DISABLED_MySQLDBTest, SEARCH_TEST) { - auto options = GetOptions(); - auto db_ = engine::DBFactory::Build(options); - +TEST_F(MySQLDBTest, SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -192,22 +181,9 @@ TEST_F(DISABLED_MySQLDBTest, SEARCH_TEST) { engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); ASSERT_STATS(stat); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; - - // TODO(linxj): add groundTruth assert }; -TEST_F(DISABLED_MySQLDBTest, ARHIVE_DISK_CHECK) { - - auto options = GetOptions(); - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - auto db_ = engine::DBFactory::Build(options); - +TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -250,20 +226,9 @@ TEST_F(DISABLED_MySQLDBTest, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; ASSERT_LE(size, 1 * engine::meta::G); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; }; -TEST_F(DISABLED_MySQLDBTest, DELETE_TEST) { - - auto options = GetOptions(); - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - auto db_ = engine::DBFactory::Build(options); - +TEST_F(MySQLDBTest, DELETE_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); // std::cout << stat.ToString() << std::endl; @@ -301,10 +266,4 @@ TEST_F(DISABLED_MySQLDBTest, DELETE_TEST) { db_->HasTable(TABLE_NAME, has_table); ASSERT_FALSE(has_table); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; }; diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 2ad842a2236aba947f5d0d5ea2a349dc73ec8e98..4960b203095f5366c2787f1d762075d2b936d60e 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,7 +21,7 @@ using namespace zilliz::milvus::engine; -TEST_F(DISABLED_MySQLTest, TABLE_TEST) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -53,7 +53,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_TEST) { table.table_id_ = table_id; status = impl.CreateTable(table); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status.IsAlreadyExist()); table.table_id_ = ""; status = impl.CreateTable(table); @@ -63,7 +63,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_TEST) { ASSERT_TRUE(status.ok()); } -TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -92,7 +92,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) { meta::DatesT dates; dates.push_back(utils::GetDate()); status = impl.DropPartitionsByDates(table_file.table_id_, dates); - ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.ok()); uint64_t cnt = 0; status = impl.Count(table_id, cnt); @@ -139,7 +139,7 @@ TEST_F(DISABLED_MySQLTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); } -TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) { +TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { srand(time(0)); DBMetaOptions options; try { @@ -211,7 +211,7 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) { ASSERT_TRUE(status.ok()); } -TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DISK) { +TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -269,7 +269,7 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DISK) { ASSERT_TRUE(status.ok()); } -TEST_F(DISABLED_MySQLTest, TABLE_FILES_TEST) { +TEST_F(MySQLTest, TABLE_FILES_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 9654245e4028a645cde873f68ec1d3265d394e02..36e2e707872afb9e9ad9270a1d0d0b043b0c695a 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -103,7 +103,7 @@ void MetaTest::TearDown() { impl_->DropAll(); } -zilliz::milvus::engine::DBMetaOptions DISABLED_MySQLTest::getDBMetaOptions() { +zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { // std::string path = "/tmp/milvus_test"; // engine::DBMetaOptions options = engine::DBMetaOptionsFactory::Build(path); zilliz::milvus::engine::DBMetaOptions options; @@ -111,17 +111,16 @@ zilliz::milvus::engine::DBMetaOptions DISABLED_MySQLTest::getDBMetaOptions() { options.backend_uri = DBTestEnvironment::getURI(); if(options.backend_uri.empty()) { -// throw std::exception(); options.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/"; } return options; } -zilliz::milvus::engine::Options DISABLED_MySQLDBTest::GetOptions() { +zilliz::milvus::engine::Options MySQLDBTest::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = DBTestEnvironment::getURI(); + options.meta.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/"; return options; } diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 83f7abef5a15d7fff621b95a050987287169b49a..8432374adaed77cb4cab1bde526504d2c6eb0658 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -79,13 +79,13 @@ class MetaTest : public DBTest { virtual void TearDown() override; }; -class DISABLED_MySQLTest : public ::testing::Test { +class MySQLTest : public ::testing::Test { protected: // std::shared_ptr impl_; zilliz::milvus::engine::DBMetaOptions getDBMetaOptions(); }; -class DISABLED_MySQLDBTest : public ::testing::Test { +class MySQLDBTest : public DBTest { protected: zilliz::milvus::engine::Options GetOptions(); };