From 3610c911305cd6bcbd97e244373cecde4c42f108 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 7 Sep 2019 18:26:13 +0800 Subject: [PATCH] MS-510 unittest out of memory and crashed Former-commit-id: 17e5dd44414299840ccba10ae93ff67d10c80405 --- cpp/CHANGELOG.md | 1 + cpp/src/db/meta/SqliteMetaImpl.cpp | 2 -- cpp/src/db/scheduler/task/IndexLoadTask.cpp | 12 +++++++++++- cpp/src/scheduler/task/SearchTask.cpp | 12 +++++++++++- cpp/src/utils/ValidationUtil.cpp | 14 +++++++------- cpp/src/wrapper/knowhere/vec_index.cpp | 6 +++++- cpp/unittest/db/utils.cpp | 17 ++++++++++++----- 7 files changed, 47 insertions(+), 17 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index ec2fa174..aa53903f 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-492 - Drop index failed if index have been created with index_type: FLAT - MS-493 - Knowhere unittest crash - MS-453 - GPU search error when nprobe set more than 1024 +- MS-510 - unittest out of memory and crashed ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 2b198b61..51297a76 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -192,8 +192,6 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { } catch (std::exception &e) { return HandleException("Encounter exception when create table", e.what()); } - - return Status::OK(); } Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index 561bf07f..65bde802 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -50,7 +50,17 @@ std::shared_ptr IndexLoadTask::Execute() { file_->nlist_); try { - index_ptr->Load(); + auto stat = index_ptr->Load(); + if(!stat.ok()) { + //typical error: file not available + ENGINE_LOG_ERROR << "Failed to load index file: file not available"; + + for(auto& context : search_contexts_) { + context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed + } + + return nullptr; + } } catch (std::exception& ex) { //typical error: out of disk space or permition denied std::string msg = "Failed to load index file: " + std::string(ex.what()); diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index ec603ac3..aa526517 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -99,7 +99,17 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { try { if (type == LoadType::DISK2CPU) { - index_engine_->Load(); + auto stat = index_engine_->Load(); + if(!stat.ok()) { + //typical error: file not available + ENGINE_LOG_ERROR << "Failed to load index file: file not available"; + + for(auto& context : search_contexts_) { + context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed + } + + return; + } } else if (type == LoadType::CPU2GPU) { index_engine_->CopyToGpu(device_id); } else if (type == LoadType::GPU2CPU) { diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index bce8143c..653c0472 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -8,9 +8,9 @@ namespace zilliz { namespace milvus { namespace server { -constexpr size_t table_name_size_limit = 255; -constexpr int64_t table_dimension_limit = 16384; -constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB +constexpr size_t TABLE_NAME_SIZE_LIMIT = 255; +constexpr int64_t TABLE_DIMENSION_LIMIT = 16384; +constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 MB ErrorCode ValidationUtil::ValidateTableName(const std::string &table_name) { @@ -22,7 +22,7 @@ ValidationUtil::ValidateTableName(const std::string &table_name) { } // Table name size shouldn't exceed 16384. - if (table_name.size() > table_name_size_limit) { + if (table_name.size() > TABLE_NAME_SIZE_LIMIT) { SERVER_LOG_ERROR << "Table name size exceed the limitation"; return SERVER_INVALID_TABLE_NAME; } @@ -48,8 +48,8 @@ ValidationUtil::ValidateTableName(const std::string &table_name) { ErrorCode ValidationUtil::ValidateTableDimension(int64_t dimension) { - if (dimension <= 0 || dimension > table_dimension_limit) { - SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; + if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) { + SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT; return SERVER_INVALID_VECTOR_DIMENSION; } else { return SERVER_SUCCESS; @@ -77,7 +77,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { ErrorCode ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { - if(index_file_size <= 0 || index_file_size > index_file_size_limit) { + if(index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { return SERVER_INVALID_INDEX_FILE_SIZE; } diff --git a/cpp/src/wrapper/knowhere/vec_index.cpp b/cpp/src/wrapper/knowhere/vec_index.cpp index 1ee8697d..d5f08136 100644 --- a/cpp/src/wrapper/knowhere/vec_index.cpp +++ b/cpp/src/wrapper/knowhere/vec_index.cpp @@ -139,7 +139,11 @@ VecIndexPtr read_index(const std::string &location) { knowhere::BinarySet load_data_list; FileIOReader reader(location); reader.fs.seekg(0, reader.fs.end); - size_t length = reader.fs.tellg(); + int64_t length = reader.fs.tellg(); + if(length <= 0) { + return nullptr; + } + reader.fs.seekg(0); size_t rp = 0; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 6a13ba52..d3473887 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -94,7 +94,8 @@ void DBTest::TearDown() { engine::ResMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); - boost::filesystem::remove_all("/tmp/milvus_test"); + auto options = GetOptions(); + boost::filesystem::remove_all(options.meta.path); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -110,11 +111,15 @@ engine::Options DBTest2::GetOptions() { void MetaTest::SetUp() { BaseTest::SetUp(); - impl_ = engine::DBMetaImplFactory::Build(); + auto options = GetOptions(); + impl_ = std::make_shared(options.meta); } void MetaTest::TearDown() { impl_->DropAll(); + + auto options = GetOptions(); + boost::filesystem::remove_all(options.meta.path); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -134,13 +139,15 @@ engine::Options MySqlDBTest::GetOptions() { void MySqlMetaTest::SetUp() { BaseTest::SetUp(); - engine::DBMetaOptions options = GetOptions().meta; - int mode = engine::Options::MODE::SINGLE; - impl_ = std::make_shared(options, mode); + auto options = GetOptions(); + impl_ = std::make_shared(options.meta, options.mode); } void MySqlMetaTest::TearDown() { impl_->DropAll(); + + auto options = GetOptions(); + boost::filesystem::remove_all(options.meta.path); } zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() { -- GitLab