Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 7
Star 4
Fork 0
milvus
milvus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 251e9b38 编写于 作者: S starlord
浏览文件
操作

fix unitest failure 


Former-commit-id: ed52c067ac2f4ca76e956f5f8afd3b8bda36bd9c
上级 08ed782a
隐藏空白更改
内联 并排
Showing with 55 addition and 73 deletion
cpp/conf/server_config.template
浏览文件 @ 251e9b38
......@@ -39,7 +39,6 @@ cache_config:
engine_config:
use_blas_threshold: 20
omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute
resource_config:
# resource list, length: 0~N
......
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 251e9b38
......@@ -881,6 +881,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
res = filesToIndexQuery.store();
} //Scoped Connection
Status ret;
std::map<std::string, TableSchema> groups;
TableFileSchema table_file;
for (auto &resRow : res) {
......@@ -925,16 +926,17 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
auto status = utils::GetTableFilePath(options_, table_file);
if(!status.ok()) {
return status;
ret = status;
}
files.push_back(table_file);
}
return ret;
} catch (std::exception &e) {
return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what());
}
return Status::OK();
}
Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
......@@ -998,6 +1000,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
return status;
}
Status ret;
TableFileSchema table_file;
for (auto &resRow : res) {
......@@ -1031,7 +1034,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
auto status = utils::GetTableFilePath(options_, table_file);
if(!status.ok()) {
return status;
ret = status;
}
auto dateItr = files.find(table_file.date_);
......@@ -1041,11 +1044,11 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
files[table_file.date_].push_back(table_file);
}
return ret;
} catch (std::exception &e) {
return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what());
}
return Status::OK();
}
Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
......@@ -1083,6 +1086,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
res = filesToMergeQuery.store();
} //Scoped Connection
Status ret;
for (auto &resRow : res) {
TableFileSchema table_file;
table_file.file_size_ = resRow["file_size"];
......@@ -1120,7 +1124,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
auto status = utils::GetTableFilePath(options_, table_file);
if(!status.ok()) {
return status;
ret = status;
}
auto dateItr = files.find(table_file.date_);
......@@ -1131,11 +1135,11 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
files[table_file.date_].push_back(table_file);
}
return ret;
} catch (std::exception &e) {
return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what());
}
return Status::OK();
}
Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
......@@ -1165,7 +1169,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(" << idStr << ");";
"(" << idStr << ") AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str();
......@@ -1174,11 +1179,9 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
TableSchema table_schema;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
DescribeTable(table_schema);
Status ret;
for (auto &resRow : res) {
TableFileSchema file_schema;
......@@ -1211,18 +1214,16 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
file_schema.dimension_ = table_schema.dimension_;
auto status = utils::GetTableFilePath(options_, file_schema);
if(!status.ok()) {
return status;
}
utils::GetTableFilePath(options_, file_schema);
table_files.emplace_back(file_schema);
}
return ret;
} catch (std::exception &e) {
return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what());
}
return Status::OK();
}
// PXU TODO: Support Swap
......
cpp/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ 251e9b38
......@@ -603,6 +603,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
std::map<std::string, TableSchema> groups;
TableFileSchema table_file;
Status ret;
for (auto &file : selected) {
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
......@@ -616,7 +617,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
auto status = utils::GetTableFilePath(options_, table_file);
if(!status.ok()) {
return status;
ret = status;
}
auto groupItr = groups.find(table_file.table_id_);
if (groupItr == groups.end()) {
......@@ -635,11 +636,11 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
files.push_back(table_file);
}
return ret;
} catch (std::exception &e) {
return HandleException("Encounter exception when iterate raw files", e.what());
}
return Status::OK();
}
Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
......@@ -695,6 +696,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
result = ConnectorPtr->select(select_columns, filter);
}
Status ret;
TableFileSchema table_file;
for (auto &file : result) {
table_file.id_ = std::get<0>(file);
......@@ -712,7 +714,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
auto status = utils::GetTableFilePath(options_, table_file);
if(!status.ok()) {
return status;
ret = status;
}
auto dateItr = files.find(table_file.date_);
......@@ -724,13 +726,12 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
if(files.empty()) {
ENGINE_LOG_ERROR << "No file to search for table: " << table_id;
}
return ret;
} catch (std::exception &e) {
return HandleException("Encounter exception when iterate index files", e.what());
}
return Status::OK();
}
Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
......@@ -761,6 +762,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
c(&TableFileSchema::table_id_) == table_id),
order_by(&TableFileSchema::file_size_).desc());
Status result;
for (auto &file : selected) {
TableFileSchema table_file;
table_file.file_size_ = std::get<4>(file);
......@@ -782,7 +784,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
auto status = utils::GetTableFilePath(options_, table_file);
if(!status.ok()) {
return status;
result = status;
}
auto dateItr = files.find(table_file.date_);
......@@ -791,11 +793,12 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
}
files[table_file.date_].push_back(table_file);
}
return result;
} catch (std::exception &e) {
return HandleException("Encounter exception when iterate merge files", e.what());
}
return Status::OK();
}
Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
......@@ -812,7 +815,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
&TableFileSchema::engine_type_,
&TableFileSchema::created_on_),
where(c(&TableFileSchema::table_id_) == table_id and
in(&TableFileSchema::id_, ids)
in(&TableFileSchema::id_, ids) and
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
TableSchema table_schema;
......@@ -822,6 +826,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
return status;
}
Status result;
for (auto &file : files) {
TableFileSchema file_schema;
file_schema.table_id_ = table_id;
......@@ -838,18 +843,15 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
file_schema.nlist_ = table_schema.nlist_;
file_schema.metric_type_ = table_schema.metric_type_;
auto status = utils::GetTableFilePath(options_, file_schema);
if(!status.ok()) {
return status;
}
utils::GetTableFilePath(options_, file_schema);
table_files.emplace_back(file_schema);
}
return result;
} catch (std::exception &e) {
return HandleException("Encounter exception when lookup table files", e.what());
}
return Status::OK();
}
// PXU TODO: Support Swap
......
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
浏览文件 @ 251e9b38
......@@ -16,7 +16,7 @@ namespace grpc {
using namespace ::milvus;
namespace {
const std::map<ErrorCode, ::milvus::grpc::ErrorCode> &ErrorMap() {
::milvus::grpc::ErrorCode ErrorMap(ErrorCode code) {
static const std::map<ErrorCode, ::milvus::grpc::ErrorCode> code_map = {
{SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
......@@ -40,8 +40,9 @@ namespace {
{SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK},
{SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST},
{SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE},
{SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID},
{SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT},
{SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED},
......@@ -49,7 +50,11 @@ namespace {
{SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR},
};
return code_map;
if(code_map.find(code) != code_map.end()) {
return code_map.at(code);
} else {
return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR;
}
}
}
......@@ -115,7 +120,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu
ErrorCode err = task_ptr->ErrorID();
if (err != SERVER_SUCCESS) {
grpc_status->set_reason(task_ptr->ErrorMsg());
grpc_status->set_error_code(ErrorMap().at(err));
grpc_status->set_error_code(ErrorMap(err));
}
}
}
......
cpp/unittest/db/meta_tests.cpp
浏览文件 @ 251e9b38
......@@ -101,8 +101,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), 1UL);
ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(files.size(), 0UL);
}
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
......@@ -150,8 +149,6 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
for(auto& file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
} else {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::TO_DELETE);
}
i++;
}
......@@ -195,9 +192,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
if (i < 5) {
ASSERT_TRUE(file.file_type_ == meta::TableFileSchema::TO_DELETE);
} else {
if (i >= 5) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
}
++i;
......@@ -277,38 +272,31 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),0);
std::vector<int> file_types;
......
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @ 251e9b38
......@@ -108,9 +108,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
std::vector<size_t> ids = {table_file.id_};
meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), 1UL);
ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(files.size(), 0UL);
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
......@@ -159,8 +157,6 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
for(auto& file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
} else {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::TO_DELETE);
}
i++;
}
......@@ -219,9 +215,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
if (i < 5) {
ASSERT_TRUE(file.file_type_ == meta::TableFileSchema::TO_DELETE);
} else {
if (i >= 5) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
}
++i;
......@@ -302,38 +296,31 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),0);
std::vector<int> file_types;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或