diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index ecfcd25f784fd5c7a6dd41344c55819099824038..51e255ac03465852ba866501684fddab7fd96044 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -39,7 +39,6 @@ cache_config: engine_config: use_blas_threshold: 20 - omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute resource_config: # resource list, length: 0~N diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 2910210a50fd03db10d3a400a87bb29797cc9002..7906e02b7fb7d6a21ca56488ffaaffca7144fdb0 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -881,6 +881,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { res = filesToIndexQuery.store(); } //Scoped Connection + Status ret; std::map groups; TableFileSchema table_file; for (auto &resRow : res) { @@ -925,16 +926,17 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { auto status = utils::GetTableFilePath(options_, table_file); if(!status.ok()) { - return status; + ret = status; } files.push_back(table_file); } + + return ret; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } - - return Status::OK(); } Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, @@ -998,6 +1000,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, return status; } + Status ret; TableFileSchema table_file; for (auto &resRow : res) { @@ -1031,7 +1034,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, auto status = utils::GetTableFilePath(options_, table_file); if(!status.ok()) { - return status; + ret = status; } auto dateItr = files.find(table_file.date_); @@ -1041,11 +1044,11 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, files[table_file.date_].push_back(table_file); } + + return ret; } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); } - - return Status::OK(); } Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, @@ -1083,6 +1086,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, res = filesToMergeQuery.store(); } //Scoped Connection + Status ret; for (auto &resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; @@ -1120,7 +1124,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, auto status = utils::GetTableFilePath(options_, table_file); if(!status.ok()) { - return status; + ret = status; } auto dateItr = files.find(table_file.date_); @@ -1131,11 +1135,11 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } + return ret; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); } - - return Status::OK(); } Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, @@ -1165,7 +1169,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ");"; + "(" << idStr << ") AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); @@ -1174,11 +1179,9 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, TableSchema table_schema; table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } + DescribeTable(table_schema); + Status ret; for (auto &resRow : res) { TableFileSchema file_schema; @@ -1211,18 +1214,16 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, file_schema.dimension_ = table_schema.dimension_; - auto status = utils::GetTableFilePath(options_, file_schema); - if(!status.ok()) { - return status; - } + utils::GetTableFilePath(options_, file_schema); table_files.emplace_back(file_schema); } + + return ret; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); } - - return Status::OK(); } // PXU TODO: Support Swap diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 730a3035ea3396485ca4e354fb872c575a3c3308..a69408c0b2b358c6bea7623636016d7498e27a50 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -603,6 +603,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { std::map groups; TableFileSchema table_file; + Status ret; for (auto &file : selected) { table_file.id_ = std::get<0>(file); table_file.table_id_ = std::get<1>(file); @@ -616,7 +617,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { auto status = utils::GetTableFilePath(options_, table_file); if(!status.ok()) { - return status; + ret = status; } auto groupItr = groups.find(table_file.table_id_); if (groupItr == groups.end()) { @@ -635,11 +636,11 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.push_back(table_file); } + return ret; + } catch (std::exception &e) { return HandleException("Encounter exception when iterate raw files", e.what()); } - - return Status::OK(); } Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, @@ -695,6 +696,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, result = ConnectorPtr->select(select_columns, filter); } + Status ret; TableFileSchema table_file; for (auto &file : result) { table_file.id_ = std::get<0>(file); @@ -712,7 +714,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, auto status = utils::GetTableFilePath(options_, table_file); if(!status.ok()) { - return status; + ret = status; } auto dateItr = files.find(table_file.date_); @@ -724,13 +726,12 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, if(files.empty()) { ENGINE_LOG_ERROR << "No file to search for table: " << table_id; } + + return ret; + } catch (std::exception &e) { return HandleException("Encounter exception when iterate index files", e.what()); } - - - - return Status::OK(); } Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, @@ -761,6 +762,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, c(&TableFileSchema::table_id_) == table_id), order_by(&TableFileSchema::file_size_).desc()); + Status result; for (auto &file : selected) { TableFileSchema table_file; table_file.file_size_ = std::get<4>(file); @@ -782,7 +784,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, auto status = utils::GetTableFilePath(options_, table_file); if(!status.ok()) { - return status; + result = status; } auto dateItr = files.find(table_file.date_); @@ -791,11 +793,12 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, } files[table_file.date_].push_back(table_file); } + + return result; + } catch (std::exception &e) { return HandleException("Encounter exception when iterate merge files", e.what()); } - - return Status::OK(); } Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, @@ -812,7 +815,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, &TableFileSchema::engine_type_, &TableFileSchema::created_on_), where(c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::id_, ids) + in(&TableFileSchema::id_, ids) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE )); TableSchema table_schema; @@ -822,6 +826,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, return status; } + Status result; for (auto &file : files) { TableFileSchema file_schema; file_schema.table_id_ = table_id; @@ -838,18 +843,15 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, file_schema.nlist_ = table_schema.nlist_; file_schema.metric_type_ = table_schema.metric_type_; - auto status = utils::GetTableFilePath(options_, file_schema); - if(!status.ok()) { - return status; - } + utils::GetTableFilePath(options_, file_schema); table_files.emplace_back(file_schema); } + + return result; } catch (std::exception &e) { return HandleException("Encounter exception when lookup table files", e.what()); } - - return Status::OK(); } // PXU TODO: Support Swap diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index 391c6bab1fb0d771f7fdf77aa209591121538452..71a9a64d56d5bb66fe0a9eda931a51baa0f8d176 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -16,7 +16,7 @@ namespace grpc { using namespace ::milvus; namespace { - const std::map &ErrorMap() { + ::milvus::grpc::ErrorCode ErrorMap(ErrorCode code) { static const std::map code_map = { {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, @@ -40,8 +40,9 @@ namespace { {SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, {SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK}, {SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST}, + {SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE}, + {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, @@ -49,7 +50,11 @@ namespace { {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, }; - return code_map; + if(code_map.find(code) != code_map.end()) { + return code_map.at(code); + } else { + return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; + } } } @@ -115,7 +120,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu ErrorCode err = task_ptr->ErrorID(); if (err != SERVER_SUCCESS) { grpc_status->set_reason(task_ptr->ErrorMsg()); - grpc_status->set_error_code(ErrorMap().at(err)); + grpc_status->set_error_code(ErrorMap(err)); } } } diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 50d92079a3e6949cc3de0add8d0ec2f29ee9a0cf..51a6f8e2237360833847568524f007d612ff71a6 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -101,8 +101,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { meta::TableFilesSchema files; status = impl_->GetTableFiles(table_file.table_id_, ids, files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(files.size(), 1UL); - ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(files.size(), 0UL); } TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { @@ -150,8 +149,6 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { for(auto& file : files_get) { if (days[i] < days_num) { ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW); - } else { - ASSERT_EQ(file.file_type_, meta::TableFileSchema::TO_DELETE); } i++; } @@ -195,9 +192,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { ASSERT_TRUE(status.ok()); for(auto& file : files_get) { - if (i < 5) { - ASSERT_TRUE(file.file_type_ == meta::TableFileSchema::TO_DELETE); - } else { + if (i >= 5) { ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW); } ++i; @@ -277,38 +272,31 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { meta::TableFilesSchema files; status = impl_->FilesToIndex(files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; status = impl_->FilesToMerge(table.table_id_, dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatesT dates = {table_file.date_}; std::vector ids; status = impl_->FilesToSearch(table_id, ids, dates, dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); ids.push_back(size_t(9999999999)); status = impl_->FilesToSearch(table_id, ids, dates, dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(),0); std::vector file_types; diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 5d692347e2d22b1adf1e389c175c382b6f811e86..e7529506b0224bad9ff3415183bd69bd33611273 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -108,9 +108,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { std::vector ids = {table_file.id_}; meta::TableFilesSchema files; status = impl_->GetTableFiles(table_file.table_id_, ids, files); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(files.size(), 1UL); - ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(files.size(), 0UL); } TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { @@ -159,8 +157,6 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { for(auto& file : files_get) { if (days[i] < days_num) { ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW); - } else { - ASSERT_EQ(file.file_type_, meta::TableFileSchema::TO_DELETE); } i++; } @@ -219,9 +215,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { ASSERT_TRUE(status.ok()); for(auto& file : files_get) { - if (i < 5) { - ASSERT_TRUE(file.file_type_ == meta::TableFileSchema::TO_DELETE); - } else { + if (i >= 5) { ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW); } ++i; @@ -302,38 +296,31 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { meta::TableFilesSchema files; status = impl_->FilesToIndex(files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; status = impl_->FilesToMerge(table.table_id_, dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatesT dates = {table_file.date_}; std::vector ids; status = impl_->FilesToSearch(table_id, ids, dates, dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); ids.push_back(size_t(9999999999)); status = impl_->FilesToSearch(table_id, ids, dates, dated_files); - ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(),0); std::vector file_types;