提交 9a16867b 编写于 作者: W wxyu

Merge remote-tracking branch 'main/branch-0.4.0' into branch-0.4.0


Former-commit-id: d8df294b4af67d6f02ed308aff3fc3552ec13d41
......@@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-492 - Drop index failed if index have been created with index_type: FLAT
- MS-493 - Knowhere unittest crash
- MS-453 - GPU search error when nprobe set more than 1024
- MS-510 - unittest out of memory and crashed
## Improvement
- MS-327 - Clean code for milvus
......
......@@ -192,8 +192,6 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) {
} catch (std::exception &e) {
return HandleException("Encounter exception when create table", e.what());
}
return Status::OK();
}
Status SqliteMetaImpl::DeleteTable(const std::string& table_id) {
......
......@@ -50,7 +50,17 @@ std::shared_ptr<IScheduleTask> IndexLoadTask::Execute() {
file_->nlist_);
try {
index_ptr->Load();
auto stat = index_ptr->Load();
if(!stat.ok()) {
//typical error: file not available
ENGINE_LOG_ERROR << "Failed to load index file: file not available";
for(auto& context : search_contexts_) {
context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed
}
return nullptr;
}
} catch (std::exception& ex) {
//typical error: out of disk space or permition denied
std::string msg = "Failed to load index file: " + std::string(ex.what());
......
......@@ -99,7 +99,17 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
try {
if (type == LoadType::DISK2CPU) {
index_engine_->Load();
auto stat = index_engine_->Load();
if(!stat.ok()) {
//typical error: file not available
ENGINE_LOG_ERROR << "Failed to load index file: file not available";
for(auto& context : search_contexts_) {
context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed
}
return;
}
} else if (type == LoadType::CPU2GPU) {
index_engine_->CopyToGpu(device_id);
} else if (type == LoadType::GPU2CPU) {
......
......@@ -8,9 +8,9 @@ namespace zilliz {
namespace milvus {
namespace server {
constexpr size_t table_name_size_limit = 255;
constexpr int64_t table_dimension_limit = 16384;
constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB
constexpr size_t TABLE_NAME_SIZE_LIMIT = 255;
constexpr int64_t TABLE_DIMENSION_LIMIT = 16384;
constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 MB
ErrorCode
ValidationUtil::ValidateTableName(const std::string &table_name) {
......@@ -22,7 +22,7 @@ ValidationUtil::ValidateTableName(const std::string &table_name) {
}
// Table name size shouldn't exceed 16384.
if (table_name.size() > table_name_size_limit) {
if (table_name.size() > TABLE_NAME_SIZE_LIMIT) {
SERVER_LOG_ERROR << "Table name size exceed the limitation";
return SERVER_INVALID_TABLE_NAME;
}
......@@ -48,8 +48,8 @@ ValidationUtil::ValidateTableName(const std::string &table_name) {
ErrorCode
ValidationUtil::ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > table_dimension_limit) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit;
if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT;
return SERVER_INVALID_VECTOR_DIMENSION;
} else {
return SERVER_SUCCESS;
......@@ -77,7 +77,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
ErrorCode
ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
if(index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
return SERVER_INVALID_INDEX_FILE_SIZE;
}
......
......@@ -139,7 +139,11 @@ VecIndexPtr read_index(const std::string &location) {
knowhere::BinarySet load_data_list;
FileIOReader reader(location);
reader.fs.seekg(0, reader.fs.end);
size_t length = reader.fs.tellg();
int64_t length = reader.fs.tellg();
if(length <= 0) {
return nullptr;
}
reader.fs.seekg(0);
size_t rp = 0;
......
......@@ -147,7 +147,7 @@ TEST_F(DBTest, DB_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
......@@ -181,7 +181,7 @@ TEST_F(DBTest, DB_TEST) {
ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
for (auto k=0; k<qb; ++k) {
ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
......@@ -212,7 +212,7 @@ TEST_F(DBTest, DB_TEST) {
uint64_t count;
stat = db_->GetTableRowCount(TABLE_NAME, count);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_TRUE(count > 0);
};
......@@ -223,7 +223,7 @@ TEST_F(DBTest, SEARCH_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
// prepare raw data
......@@ -258,7 +258,7 @@ TEST_F(DBTest, SEARCH_TEST) {
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids);
if (j == 200){ sleep(1);}
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
}
engine::TableIndex index;
......@@ -268,7 +268,7 @@ TEST_F(DBTest, SEARCH_TEST) {
{
engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
......@@ -276,7 +276,7 @@ TEST_F(DBTest, SEARCH_TEST) {
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
engine::QueryResults results;
stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
}
// TODO(linxj): add groundTruth assert
......@@ -289,7 +289,7 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
int64_t nb = VECTOR_COUNT;
......@@ -309,7 +309,7 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
int64_t cur_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
ASSERT_TRUE(prev_cache_usage < cur_cache_usage);
......@@ -322,7 +322,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
std::vector<engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for(auto& schema : table_schema_array) {
if(schema.table_id_ == TABLE_NAME) {
......@@ -335,7 +335,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
uint64_t size;
......@@ -366,7 +366,7 @@ TEST_F(DBTest2, DELETE_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
bool has_table = false;
db_->HasTable(TABLE_NAME, has_table);
......@@ -405,7 +405,7 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
bool has_table = false;
db_->HasTable(TABLE_NAME, has_table);
......@@ -433,7 +433,7 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
ConvertTimeRangeToDBDates(start_value, end_value, dates);
stat = db_->DeleteTable(TABLE_NAME, dates);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
uint64_t row_count = 0;
db_->GetTableRowCount(TABLE_NAME, row_count);
......
......@@ -3,12 +3,12 @@
#include "db/insert/VectorSource.h"
#include "db/insert/MemTableFile.h"
#include "db/insert/MemTable.h"
#include "utils.h"
#include "db/Factories.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "db/meta/MetaConsts.h"
#include "metrics/Metrics.h"
#include "utils.h"
#include <boost/filesystem.hpp>
#include <thread>
......@@ -223,7 +223,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
std::map<int64_t, std::vector<float>> search_vectors;
......@@ -269,7 +269,7 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
auto start_time = METRICS_NOW_TIME;
......@@ -295,7 +295,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
......@@ -329,7 +329,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
ss << "Search " << j << " With Size " << count / engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
for (auto k = 0; k < qb; ++k) {
ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
......@@ -366,7 +366,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
......@@ -383,7 +383,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], 0);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
nb = 25000;
xb.clear();
......@@ -395,7 +395,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
nb = 262144; //512M
xb.clear();
......@@ -407,14 +407,14 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb/2);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
nb = 65536; //128M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
nb = 100;
xb.clear();
......
......@@ -53,7 +53,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
......@@ -90,7 +90,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
for (auto k=0; k<qb; ++k) {
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
......@@ -138,7 +138,7 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
// prepare raw data
......@@ -173,14 +173,14 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids);
if (j == 200){ sleep(1);}
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
}
sleep(2); // wait until build index finish
engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
};
TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
......@@ -189,7 +189,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
std::vector<engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for(auto& schema : table_schema_array) {
if(schema.table_id_ == TABLE_NAME) {
......@@ -202,7 +202,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
......@@ -236,7 +236,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_TRUE(stat.ok());
bool has_table = false;
db_->HasTable(TABLE_NAME, has_table);
......
......@@ -35,13 +35,6 @@ public:
};
void ASSERT_STATS(engine::Status& stat) {
ASSERT_TRUE(stat.ok());
if(!stat.ok()) {
std::cout << stat.ToString() << std::endl;
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void BaseTest::InitLog() {
el::Configurations defaultConf;
......@@ -94,7 +87,8 @@ void DBTest::TearDown() {
engine::ResMgrInst::GetInstance()->Stop();
engine::SchedInst::GetInstance()->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
auto options = GetOptions();
boost::filesystem::remove_all(options.meta.path);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -110,11 +104,15 @@ engine::Options DBTest2::GetOptions() {
void MetaTest::SetUp() {
BaseTest::SetUp();
impl_ = engine::DBMetaImplFactory::Build();
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::SqliteMetaImpl>(options.meta);
}
void MetaTest::TearDown() {
impl_->DropAll();
auto options = GetOptions();
boost::filesystem::remove_all(options.meta.path);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -134,13 +132,15 @@ engine::Options MySqlDBTest::GetOptions() {
void MySqlMetaTest::SetUp() {
BaseTest::SetUp();
engine::DBMetaOptions options = GetOptions().meta;
int mode = engine::Options::MODE::SINGLE;
impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options, mode);
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options.meta, options.mode);
}
void MySqlMetaTest::TearDown() {
impl_->DropAll();
auto options = GetOptions();
boost::filesystem::remove_all(options.meta.path);
}
zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() {
......
......@@ -32,8 +32,6 @@
#define STOP_TIMER(name)
#endif
void ASSERT_STATS(zilliz::milvus::engine::Status &stat);
class BaseTest : public ::testing::Test {
protected:
void InitLog();
......
......@@ -34,7 +34,6 @@ class KnowhereWrapperTest
std::string generator_type;
std::tie(index_type, generator_type, dim, nb, nq, k, train_cfg, search_cfg) = GetParam();
//auto generator = GetGenerateFactory(generator_type);
auto generator = std::make_shared<DataGenBase>();
generator->GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis);
......
......@@ -9,14 +9,6 @@
#include "utils.h"
DataGenPtr GetGenerateFactory(const std::string &gen_type) {
std::shared_ptr<DataGenBase> generator;
if (gen_type == "default") {
generator = std::make_shared<DataGenBase>();
}
return generator;
}
void DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
float *xb, float *xq, long *ids,
const int &k, long *gt_ids, float *gt_dis) {
......
......@@ -17,8 +17,6 @@ class DataGenBase;
using DataGenPtr = std::shared_ptr<DataGenBase>;
extern DataGenPtr GetGenerateFactory(const std::string &gen_type);
class DataGenBase {
public:
......
......@@ -90,8 +90,6 @@ TEST_F(MetricTest, Metric_Tes) {
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/(1024*1024) << " M";
// ASSERT_STATS(stat);
for (auto k=0; k<qb; ++k) {
// ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
......
......@@ -34,14 +34,6 @@ public:
};
void ASSERT_STATS(engine::Status& stat) {
ASSERT_TRUE(stat.ok());
if(!stat.ok()) {
std::cout << stat.ToString() << std::endl;
}
}
void MetricTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
......
......@@ -36,7 +36,7 @@ public:
const engine::Config &cfg,
const long &nt = 0,
const float *xt = nullptr) {
return 0;
}
engine::VecIndexPtr Clone() override {
......@@ -55,7 +55,7 @@ public:
const float *xb,
const long *ids,
const engine::Config &cfg = engine::Config()) {
return 0;
}
virtual ErrorCode Search(const long &nq,
......@@ -63,15 +63,16 @@ public:
float *dist,
long *ids,
const engine::Config &cfg = engine::Config()) {
return 0;
}
engine::VecIndexPtr CopyToGpu(const int64_t &device_id, const engine::Config &cfg) override {
engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const engine::Config &cfg) override {
return nullptr;
}
engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override {
return nullptr;
}
virtual int64_t Dimension() {
......@@ -88,7 +89,7 @@ public:
}
virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) {
return 0;
}
public:
......@@ -98,6 +99,24 @@ public:
}
TEST(CacheTest, DUMMY_TEST) {
engine::Config cfg;
MockVecIndex mock_index;
mock_index.Dimension();
mock_index.Count();
mock_index.Add(1, nullptr, nullptr);
mock_index.BuildAll(1, nullptr, nullptr, cfg);
mock_index.Search(1, nullptr, nullptr, nullptr, cfg);
mock_index.Clone();
mock_index.CopyToCpu(cfg);
mock_index.CopyToGpu(1, cfg);
mock_index.GetDeviceId();
mock_index.GetType();
zilliz::knowhere::BinarySet index_binary;
mock_index.Load(index_binary);
mock_index.Serialize();
}
TEST(CacheTest, CPU_CACHE_TEST) {
cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance();
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册