Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

从无法访问的项目Fork

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 5986de3f 编写于 作者: P peng.xu
浏览文件
操作

Merge branch 'branch-0.4.0' into 'branch-0.4.0' 

refine code

See merge request megasearch/milvus!483

Former-commit-id: 5323139fbe9f0e080d6287754651ef285cb461a9
上级 62775f49 12bb81c5
隐藏空白更改
内联 并排
Showing with 264 addition and 338 deletion
cpp/src/db/DBImpl.cpp
浏览文件 @ 5986de3f
...@@ -53,11 +53,15 @@ DBImpl::~DBImpl() { ...@@ -53,11 +53,15 @@ DBImpl::~DBImpl() {
Stop(); Stop();
} }
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//external api
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::Start() { Status DBImpl::Start() {
if (!shutting_down_.load(std::memory_order_acquire)){ if (!shutting_down_.load(std::memory_order_acquire)){
return Status::OK(); return Status::OK();
} }
ENGINE_LOG_TRACE << "DB service start";
shutting_down_.store(false, std::memory_order_release); shutting_down_.store(false, std::memory_order_release);
//for distribute version, some nodes are read only //for distribute version, some nodes are read only
...@@ -75,30 +79,40 @@ Status DBImpl::Stop() { ...@@ -75,30 +79,40 @@ Status DBImpl::Stop() {
} }
shutting_down_.store(true, std::memory_order_release); shutting_down_.store(true, std::memory_order_release);
bg_timer_thread_.join();
//makesure all memory data serialized
MemSerialize();
//wait compaction/buildindex finish //wait compaction/buildindex finish
for(auto& result : compact_thread_results_) { bg_timer_thread_.join();
result.wait();
}
for(auto& result : index_thread_results_) { if (options_.mode != Options::MODE::READ_ONLY) {
result.wait(); meta_ptr_->CleanUp();
} }
//makesure all memory data serialized ENGINE_LOG_TRACE << "DB service stop";
MemSerialize();
return Status::OK(); return Status::OK();
} }
Status DBImpl::DropAll() {
return meta_ptr_->DropAll();
}
Status DBImpl::CreateTable(meta::TableSchema& table_schema) { Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
meta::TableSchema temp_schema = table_schema; meta::TableSchema temp_schema = table_schema;
temp_schema.index_file_size_ *= ONE_MB; temp_schema.index_file_size_ *= ONE_MB;
return meta_ptr_->CreateTable(temp_schema); return meta_ptr_->CreateTable(temp_schema);
} }
Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
//dates partly delete files of the table but currently we don't support //dates partly delete files of the table but currently we don't support
ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id; ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id;
...@@ -121,18 +135,34 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date ...@@ -121,18 +135,34 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
} }
Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
return meta_ptr_->DescribeTable(table_schema); return meta_ptr_->DescribeTable(table_schema);
} }
Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
return meta_ptr_->HasTable(table_id, has_or_not); return meta_ptr_->HasTable(table_id, has_or_not);
} }
Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) { Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
return meta_ptr_->AllTables(table_schema_array); return meta_ptr_->AllTables(table_schema_array);
} }
Status DBImpl::PreloadTable(const std::string &table_id) { Status DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
meta::DatePartionedTableFilesSchema files; meta::DatePartionedTableFilesSchema files;
meta::DatesT dates; meta::DatesT dates;
...@@ -174,16 +204,27 @@ Status DBImpl::PreloadTable(const std::string &table_id) { ...@@ -174,16 +204,27 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
} }
Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
return meta_ptr_->UpdateTableFlag(table_id, flag); return meta_ptr_->UpdateTableFlag(table_id, flag);
} }
Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
return meta_ptr_->Count(table_id, row_count); return meta_ptr_->Count(table_id, row_count);
} }
Status DBImpl::InsertVectors(const std::string& table_id_, Status DBImpl::InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) { uint64_t n, const float* vectors, IDNumbers& vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
Status status; Status status;
zilliz::milvus::server::CollectInsertMetrics metrics(n, status); zilliz::milvus::server::CollectInsertMetrics metrics(n, status);
...@@ -196,8 +237,82 @@ Status DBImpl::InsertVectors(const std::string& table_id_, ...@@ -196,8 +237,82 @@ Status DBImpl::InsertVectors(const std::string& table_id_,
return status; return status;
} }
Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
//step 1: check index difference
TableIndex old_index;
auto status = DescribeIndex(table_id, old_index);
if(!status.ok()) {
ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id;
return status;
}
//step 2: update index info
if(!utils::IsSameIndex(old_index, index)) {
DropIndex(table_id);
status = meta_ptr_->UpdateTableIndexParam(table_id, index);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
return status;
}
}
}
//step 3: wait and build index
//for IDMAP type, only wait all NEW file converted to RAW file
//for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
file_types = {
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
};
} else {
file_types = {
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
};
}
std::vector<std::string> file_ids;
auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
int times = 1;
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++;
}
return Status::OK();
}
Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) {
return meta_ptr_->DescribeTableIndex(table_id, index);
}
Status DBImpl::DropIndex(const std::string& table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id);
}
Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) { const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
meta::DatesT dates = {utils::GetDate()}; meta::DatesT dates = {utils::GetDate()};
Status result = Query(table_id, k, nq, nprobe, vectors, dates, results); Status result = Query(table_id, k, nq, nprobe, vectors, dates, results);
...@@ -206,6 +321,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6 ...@@ -206,6 +321,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6
Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) { const float* vectors, const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id; ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id;
//get all table files from table //get all table files from table
...@@ -230,6 +349,10 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6 ...@@ -230,6 +349,10 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6
Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids, Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) { const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id; ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id;
//get specified files //get specified files
...@@ -264,6 +387,18 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string> ...@@ -264,6 +387,18 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
return status; return status;
} }
Status DBImpl::Size(uint64_t& result) {
if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!");
}
return meta_ptr_->Size(result);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//internal methods
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) { const meta::DatesT& dates, QueryResults& results) {
...@@ -563,76 +698,6 @@ void DBImpl::StartBuildIndexTask(bool force) { ...@@ -563,76 +698,6 @@ void DBImpl::StartBuildIndexTask(bool force) {
} }
} }
Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
//step 1: check index difference
TableIndex old_index;
auto status = DescribeIndex(table_id, old_index);
if(!status.ok()) {
ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id;
return status;
}
//step 2: update index info
if(!utils::IsSameIndex(old_index, index)) {
DropIndex(table_id);
status = meta_ptr_->UpdateTableIndexParam(table_id, index);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
return status;
}
}
}
//step 3: wait and build index
//for IDMAP type, only wait all NEW file converted to RAW file
//for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
file_types = {
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
};
} else {
file_types = {
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
};
}
std::vector<std::string> file_ids;
auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
int times = 1;
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++;
}
return Status::OK();
}
Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) {
return meta_ptr_->DescribeTableIndex(table_id, index);
}
Status DBImpl::DropIndex(const std::string& table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id);
}
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
ExecutionEnginePtr to_index = ExecutionEnginePtr to_index =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
...@@ -775,15 +840,6 @@ void DBImpl::BackgroundBuildIndex() { ...@@ -775,15 +840,6 @@ void DBImpl::BackgroundBuildIndex() {
ENGINE_LOG_TRACE << "Background build index thread exit"; ENGINE_LOG_TRACE << "Background build index thread exit";
} }
Status DBImpl::DropAll() {
Stop();
return meta_ptr_->DropAll();
}
Status DBImpl::Size(uint64_t& result) {
return meta_ptr_->Size(result);
}
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz
cpp/src/db/DBImpl.h
浏览文件 @ 5986de3f
...@@ -39,6 +39,7 @@ class DBImpl : public DB { ...@@ -39,6 +39,7 @@ class DBImpl : public DB {
Status Start() override; Status Start() override;
Status Stop() override; Status Stop() override;
Status DropAll() override;
Status CreateTable(meta::TableSchema &table_schema) override; Status CreateTable(meta::TableSchema &table_schema) override;
...@@ -58,6 +59,12 @@ class DBImpl : public DB { ...@@ -58,6 +59,12 @@ class DBImpl : public DB {
Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override;
Status CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status DescribeIndex(const std::string& table_id, TableIndex& index) override;
Status DropIndex(const std::string& table_id) override;
Status Query(const std::string &table_id, Status Query(const std::string &table_id,
uint64_t k, uint64_t k,
uint64_t nq, uint64_t nq,
...@@ -82,16 +89,8 @@ class DBImpl : public DB { ...@@ -82,16 +89,8 @@ class DBImpl : public DB {
const meta::DatesT &dates, const meta::DatesT &dates,
QueryResults &results) override; QueryResults &results) override;
Status DropAll() override;
Status Size(uint64_t &result) override; Status Size(uint64_t &result) override;
Status CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status DescribeIndex(const std::string& table_id, TableIndex& index) override;
Status DropIndex(const std::string& table_id) override;
private: private:
Status QueryAsync(const std::string &table_id, Status QueryAsync(const std::string &table_id,
const meta::TableFilesSchema &files, const meta::TableFilesSchema &files,
......
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 5986de3f
...@@ -48,9 +48,7 @@ MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode) ...@@ -48,9 +48,7 @@ MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode)
} }
MySQLMetaImpl::~MySQLMetaImpl() { MySQLMetaImpl::~MySQLMetaImpl() {
if (mode_ != Options::MODE::READ_ONLY) {
CleanUp();
}
} }
Status MySQLMetaImpl::NextTableId(std::string &table_id) { Status MySQLMetaImpl::NextTableId(std::string &table_id) {
...@@ -2002,6 +2000,7 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { ...@@ -2002,6 +2000,7 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) {
Status MySQLMetaImpl::DropAll() { Status MySQLMetaImpl::DropAll() {
try { try {
ENGINE_LOG_DEBUG << "Drop all mysql meta";
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) { if (connectionPtr == nullptr) {
......
cpp/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ 5986de3f
...@@ -74,7 +74,7 @@ SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_) ...@@ -74,7 +74,7 @@ SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_)
} }
SqliteMetaImpl::~SqliteMetaImpl() { SqliteMetaImpl::~SqliteMetaImpl() {
CleanUp();
} }
Status SqliteMetaImpl::NextTableId(std::string &table_id) { Status SqliteMetaImpl::NextTableId(std::string &table_id) {
...@@ -1205,6 +1205,14 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { ...@@ -1205,6 +1205,14 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) {
} }
Status SqliteMetaImpl::DropAll() { Status SqliteMetaImpl::DropAll() {
ENGINE_LOG_DEBUG << "Drop all sqlite meta";
try {
ConnectorPtr->drop_table("Tables");
ConnectorPtr->drop_table("TableFiles");
} catch (std::exception &e) {
return HandleException("Encounter exception when drop all meta", e);
}
return Status::OK(); return Status::OK();
} }
......
cpp/unittest/db/mem_test.cpp
浏览文件 @ 5986de3f
...@@ -54,7 +54,7 @@ void BuildVectors(int64_t n, std::vector<float> &vectors) { ...@@ -54,7 +54,7 @@ void BuildVectors(int64_t n, std::vector<float> &vectors) {
} }
} }
TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
std::shared_ptr<engine::meta::SqliteMetaImpl> impl_ = engine::DBMetaImplFactory::Build(); std::shared_ptr<engine::meta::SqliteMetaImpl> impl_ = engine::DBMetaImplFactory::Build();
...@@ -102,7 +102,7 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { ...@@ -102,7 +102,7 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
std::shared_ptr<engine::meta::SqliteMetaImpl> impl_ = engine::DBMetaImplFactory::Build(); std::shared_ptr<engine::meta::SqliteMetaImpl> impl_ = engine::DBMetaImplFactory::Build();
auto options = engine::OptionsFactory::Build(); auto options = engine::OptionsFactory::Build();
...@@ -148,7 +148,7 @@ TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { ...@@ -148,7 +148,7 @@ TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::shared_ptr<engine::meta::SqliteMetaImpl> impl_ = engine::DBMetaImplFactory::Build(); std::shared_ptr<engine::meta::SqliteMetaImpl> impl_ = engine::DBMetaImplFactory::Build();
auto options = engine::OptionsFactory::Build(); auto options = engine::OptionsFactory::Build();
...@@ -212,19 +212,11 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { ...@@ -212,19 +212,11 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) {
status = mem_table.Serialize(); status = mem_table.Serialize();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = impl_->DropAll(); status = impl_->DropAll();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { TEST_F(MemManagerTest, SERIAL_INSERT_SEARCH_TEST) {
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
auto db_ = engine::DBFactory::Build(options);
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -268,18 +260,9 @@ TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { ...@@ -268,18 +260,9 @@ TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) {
ASSERT_EQ(results[0][0].first, pair.first); ASSERT_EQ(results[0][0].first, pair.first);
ASSERT_LT(results[0][0].second, 0.00001); ASSERT_LT(results[0][0].second, 0.00001);
} }
delete db_;
} }
TEST_F(NewMemManagerTest, INSERT_TEST) { TEST_F(MemManagerTest, INSERT_TEST) {
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
auto db_ = engine::DBFactory::Build(options);
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -303,18 +286,9 @@ TEST_F(NewMemManagerTest, INSERT_TEST) { ...@@ -303,18 +286,9 @@ TEST_F(NewMemManagerTest, INSERT_TEST) {
auto end_time = METRICS_NOW_TIME; auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time); auto total_time = METRICS_MICROSECONDS(start_time, end_time);
LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time; LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time;
delete db_;
} }
TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { TEST_F(MemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) {
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
auto db_ = engine::DBFactory::Build(options);
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -383,12 +357,9 @@ TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { ...@@ -383,12 +357,9 @@ TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) {
} }
search.join(); search.join();
delete db_;
}; };
TEST_F(DBTest, VECTOR_IDS_TEST) TEST_F(MemManagerTest, VECTOR_IDS_TEST) {
{
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -458,39 +429,3 @@ TEST_F(DBTest, VECTOR_IDS_TEST) ...@@ -458,39 +429,3 @@ TEST_F(DBTest, VECTOR_IDS_TEST)
ASSERT_EQ(vector_ids[i], i + nb); ASSERT_EQ(vector_ids[i], i + nb);
} }
} }
TEST_F(NewMemManagerTest, MEMMANAGER_TEST) {
int setenv_res = setenv("MILVUS_USE_OLD_MEM_MANAGER", "ON", 1);
ASSERT_TRUE(setenv_res == 0);
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
auto db_ = engine::DBFactory::Build(options);
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
auto start_time = METRICS_NOW_TIME;
int insert_loop = 20;
for (int i = 0; i < insert_loop; ++i) {
int64_t nb = 40960;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_TRUE(status.ok());
}
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time;
delete db_;
}
cpp/unittest/db/mysql_db_test.cpp
浏览文件 @ 5986de3f
...@@ -46,7 +46,7 @@ namespace { ...@@ -46,7 +46,7 @@ namespace {
} }
TEST_F(MySQLDBTest, DB_TEST) { TEST_F(MySqlDBTest, DB_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -131,7 +131,7 @@ TEST_F(MySQLDBTest, DB_TEST) { ...@@ -131,7 +131,7 @@ TEST_F(MySQLDBTest, DB_TEST) {
search.join(); search.join();
}; };
TEST_F(MySQLDBTest, SEARCH_TEST) { TEST_F(MySqlDBTest, SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -183,7 +183,7 @@ TEST_F(MySQLDBTest, SEARCH_TEST) { ...@@ -183,7 +183,7 @@ TEST_F(MySQLDBTest, SEARCH_TEST) {
ASSERT_STATS(stat); ASSERT_STATS(stat);
}; };
TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
...@@ -228,7 +228,7 @@ TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { ...@@ -228,7 +228,7 @@ TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) {
ASSERT_LE(size, 1 * engine::meta::G); ASSERT_LE(size, 1 * engine::meta::G);
}; };
TEST_F(MySQLDBTest, DELETE_TEST) { TEST_F(MySqlDBTest, DELETE_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema(); engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info); engine::Status stat = db_->CreateTable(table_info);
// std::cout << stat.ToString() << std::endl; // std::cout << stat.ToString() << std::endl;
......
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @ 5986de3f
...@@ -21,81 +21,59 @@ ...@@ -21,81 +21,59 @@
using namespace zilliz::milvus::engine; using namespace zilliz::milvus::engine;
TEST_F(MySQLTest, TABLE_TEST) { TEST_F(MySqlMetaTest, TABLE_TEST) {
DBMetaOptions options;
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_table"; auto table_id = "meta_test_table";
meta::TableSchema table; meta::TableSchema table;
table.table_id_ = table_id; table.table_id_ = table_id;
auto status = impl.CreateTable(table); auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
auto gid = table.id_; auto gid = table.id_;
table.id_ = -1; table.id_ = -1;
status = impl.DescribeTable(table); status = impl_->DescribeTable(table);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(table.id_, gid); ASSERT_EQ(table.id_, gid);
ASSERT_EQ(table.table_id_, table_id); ASSERT_EQ(table.table_id_, table_id);
table.table_id_ = "not_found"; table.table_id_ = "not_found";
status = impl.DescribeTable(table); status = impl_->DescribeTable(table);
ASSERT_TRUE(!status.ok()); ASSERT_TRUE(!status.ok());
table.table_id_ = table_id; table.table_id_ = table_id;
status = impl.CreateTable(table); status = impl_->CreateTable(table);
ASSERT_TRUE(status.IsAlreadyExist()); ASSERT_TRUE(status.IsAlreadyExist());
table.table_id_ = ""; table.table_id_ = "";
status = impl.CreateTable(table); status = impl_->CreateTable(table);
// ASSERT_TRUE(status.ok()); // ASSERT_TRUE(status.ok());
status = impl.DropAll(); status = impl_->DropAll();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(MySQLTest, TABLE_FILE_TEST) { TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
DBMetaOptions options;
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_table"; auto table_id = "meta_test_table";
meta::TableSchema table; meta::TableSchema table;
table.table_id_ = table_id; table.table_id_ = table_id;
table.dimension_ = 256; table.dimension_ = 256;
auto status = impl.CreateTable(table); auto status = impl_->CreateTable(table);
meta::TableFileSchema table_file; meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_; table_file.table_id_ = table.table_id_;
status = impl.CreateTableFile(table_file); status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
meta::DatesT dates; meta::DatesT dates;
dates.push_back(utils::GetDate()); dates.push_back(utils::GetDate());
status = impl.DropPartitionsByDates(table_file.table_id_, dates); status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
uint64_t cnt = 0; uint64_t cnt = 0;
status = impl.Count(table_id, cnt); status = impl_->Count(table_id, cnt);
// ASSERT_TRUE(status.ok()); // ASSERT_TRUE(status.ok());
// ASSERT_EQ(cnt, 0UL); // ASSERT_EQ(cnt, 0UL);
...@@ -104,7 +82,7 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) { ...@@ -104,7 +82,7 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) {
auto new_file_type = meta::TableFileSchema::INDEX; auto new_file_type = meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type; table_file.file_type_ = new_file_type;
status = impl.UpdateTableFile(table_file); status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, new_file_type); ASSERT_EQ(table_file.file_type_, new_file_type);
...@@ -112,42 +90,31 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) { ...@@ -112,42 +90,31 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) {
for (auto i=2; i < 10; ++i) { for (auto i=2; i < 10; ++i) {
dates.push_back(utils::GetDateWithDelta(-1*i)); dates.push_back(utils::GetDateWithDelta(-1*i));
} }
status = impl.DropPartitionsByDates(table_file.table_id_, dates); status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
table_file.date_ = utils::GetDateWithDelta(-2); table_file.date_ = utils::GetDateWithDelta(-2);
status = impl.UpdateTableFile(table_file); status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2)); ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
dates.clear(); dates.clear();
dates.push_back(table_file.date_); dates.push_back(table_file.date_);
status = impl.DropPartitionsByDates(table_file.table_id_, dates); status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_}; std::vector<size_t> ids = {table_file.id_};
meta::TableFilesSchema files; meta::TableFilesSchema files;
status = impl.GetTableFiles(table_file.table_id_, ids, files); status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), 1UL); ASSERT_EQ(files.size(), 1UL);
ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE); ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE);
// status = impl.NextTableId(table_id);
status = impl.DropAll();
ASSERT_TRUE(status.ok());
} }
TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0)); srand(time(0));
DBMetaOptions options; DBMetaOptions options = GetOptions().meta;
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
int days_num = rand() % 100; int days_num = rand() % 100;
std::stringstream ss; std::stringstream ss;
...@@ -211,14 +178,8 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { ...@@ -211,14 +178,8 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
DBMetaOptions options; DBMetaOptions options = GetOptions().meta;
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
options.archive_conf = ArchiveConf("delete", "disk:11"); options.archive_conf = ArchiveConf("delete", "disk:11");
int mode = Options::MODE::SINGLE; int mode = Options::MODE::SINGLE;
...@@ -269,23 +230,12 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { ...@@ -269,23 +230,12 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
TEST_F(MySQLTest, TABLE_FILES_TEST) { TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
DBMetaOptions options;
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
int mode = Options::MODE::SINGLE;
auto impl = meta::MySQLMetaImpl(options, mode);
auto table_id = "meta_test_group"; auto table_id = "meta_test_group";
meta::TableSchema table; meta::TableSchema table;
table.table_id_ = table_id; table.table_id_ = table_id;
auto status = impl.CreateTable(table); auto status = impl_->CreateTable(table);
int new_files_cnt = 4; int new_files_cnt = 4;
int raw_files_cnt = 5; int raw_files_cnt = 5;
...@@ -296,66 +246,66 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ...@@ -296,66 +246,66 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
table_file.table_id_ = table.table_id_; table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_files_cnt; ++i) { for (auto i=0; i<new_files_cnt; ++i) {
status = impl.CreateTableFile(table_file); status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW; table_file.file_type_ = meta::TableFileSchema::NEW;
status = impl.UpdateTableFile(table_file); status = impl_->UpdateTableFile(table_file);
} }
for (auto i=0; i<raw_files_cnt; ++i) { for (auto i=0; i<raw_files_cnt; ++i) {
status = impl.CreateTableFile(table_file); status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::RAW; table_file.file_type_ = meta::TableFileSchema::RAW;
status = impl.UpdateTableFile(table_file); status = impl_->UpdateTableFile(table_file);
} }
for (auto i=0; i<to_index_files_cnt; ++i) { for (auto i=0; i<to_index_files_cnt; ++i) {
status = impl.CreateTableFile(table_file); status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::TO_INDEX; table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
status = impl.UpdateTableFile(table_file); status = impl_->UpdateTableFile(table_file);
} }
for (auto i=0; i<index_files_cnt; ++i) { for (auto i=0; i<index_files_cnt; ++i) {
status = impl.CreateTableFile(table_file); status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::INDEX; table_file.file_type_ = meta::TableFileSchema::INDEX;
status = impl.UpdateTableFile(table_file); status = impl_->UpdateTableFile(table_file);
} }
meta::TableFilesSchema files; meta::TableFilesSchema files;
status = impl.FilesToIndex(files); status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt); ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files; meta::DatePartionedTableFilesSchema dated_files;
status = impl.FilesToMerge(table.table_id_, dated_files); status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl.FilesToIndex(files); status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt); ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_}; meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids; std::vector<size_t> ids;
status = impl.FilesToSearch(table_id, ids, dates, dated_files); status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt); to_index_files_cnt+raw_files_cnt+index_files_cnt);
status = impl.FilesToSearch(table_id, ids, meta::DatesT(), dated_files); status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt); to_index_files_cnt+raw_files_cnt+index_files_cnt);
status = impl.FilesToSearch(table_id, ids, meta::DatesT(), dated_files); status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt); to_index_files_cnt+raw_files_cnt+index_files_cnt);
ids.push_back(size_t(9999999999)); ids.push_back(size_t(9999999999));
status = impl.FilesToSearch(table_id, ids, dates, dated_files); status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(),0); ASSERT_EQ(dated_files[table_file.date_].size(),0);
status = impl.DropAll(); status = impl_->DropAll();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
} }
cpp/unittest/db/utils.cpp
浏览文件 @ 5986de3f
...@@ -42,8 +42,8 @@ void ASSERT_STATS(engine::Status& stat) { ...@@ -42,8 +42,8 @@ void ASSERT_STATS(engine::Status& stat) {
} }
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void DBTest::InitLog() { void BaseTest::InitLog() {
el::Configurations defaultConf; el::Configurations defaultConf;
defaultConf.setToDefault(); defaultConf.setToDefault();
defaultConf.set(el::Level::Debug, defaultConf.set(el::Level::Debug,
...@@ -51,13 +51,14 @@ void DBTest::InitLog() { ...@@ -51,13 +51,14 @@ void DBTest::InitLog() {
el::Loggers::reconfigureLogger("default", defaultConf); el::Loggers::reconfigureLogger("default", defaultConf);
} }
engine::Options DBTest::GetOptions() { engine::Options BaseTest::GetOptions() {
auto options = engine::OptionsFactory::Build(); auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test"; options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/"; options.meta.backend_uri = "sqlite://:@:/";
return options; return options;
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void DBTest::SetUp() { void DBTest::SetUp() {
InitLog(); InitLog();
...@@ -82,6 +83,7 @@ void DBTest::SetUp() { ...@@ -82,6 +83,7 @@ void DBTest::SetUp() {
} }
void DBTest::TearDown() { void DBTest::TearDown() {
db_->Stop();
db_->DropAll(); db_->DropAll();
delete db_; delete db_;
...@@ -91,6 +93,7 @@ void DBTest::TearDown() { ...@@ -91,6 +93,7 @@ void DBTest::TearDown() {
boost::filesystem::remove_all("/tmp/milvus_test"); boost::filesystem::remove_all("/tmp/milvus_test");
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::Options DBTest2::GetOptions() { engine::Options DBTest2::GetOptions() {
auto options = engine::OptionsFactory::Build(); auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test"; options.meta.path = "/tmp/milvus_test";
...@@ -99,6 +102,7 @@ engine::Options DBTest2::GetOptions() { ...@@ -99,6 +102,7 @@ engine::Options DBTest2::GetOptions() {
return options; return options;
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MetaTest::SetUp() { void MetaTest::SetUp() {
InitLog(); InitLog();
impl_ = engine::DBMetaImplFactory::Build(); impl_ = engine::DBMetaImplFactory::Build();
...@@ -108,21 +112,8 @@ void MetaTest::TearDown() { ...@@ -108,21 +112,8 @@ void MetaTest::TearDown() {
impl_->DropAll(); impl_->DropAll();
} }
zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// std::string path = "/tmp/milvus_test"; engine::Options MySqlDBTest::GetOptions() {
// engine::DBMetaOptions options = engine::DBMetaOptionsFactory::Build(path);
zilliz::milvus::engine::DBMetaOptions options;
options.path = "/tmp/milvus_test";
options.backend_uri = DBTestEnvironment::getURI();
if(options.backend_uri.empty()) {
options.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/";
}
return options;
}
zilliz::milvus::engine::Options MySQLDBTest::GetOptions() {
auto options = engine::OptionsFactory::Build(); auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test"; options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = DBTestEnvironment::getURI(); options.meta.backend_uri = DBTestEnvironment::getURI();
...@@ -134,33 +125,32 @@ zilliz::milvus::engine::Options MySQLDBTest::GetOptions() { ...@@ -134,33 +125,32 @@ zilliz::milvus::engine::Options MySQLDBTest::GetOptions() {
return options; return options;
} }
void NewMemManagerTest::InitLog() { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
el::Configurations defaultConf; void MySqlMetaTest::SetUp() {
defaultConf.setToDefault();
defaultConf.set(el::Level::Debug,
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::Loggers::reconfigureLogger("default", defaultConf);
}
void NewMemManagerTest::SetUp() {
InitLog(); InitLog();
auto res_mgr = engine::ResMgrInst::GetInstance(); engine::DBMetaOptions options = GetOptions().meta;
res_mgr->Clear(); int mode = engine::Options::MODE::SINGLE;
res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false)); impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options, mode);
res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, true)); }
auto default_conn = engine::Connection("IO", 500.0); void MySqlMetaTest::TearDown() {
res_mgr->Connect("disk", "cpu", default_conn); impl_->DropAll();
res_mgr->Start();
engine::SchedInst::GetInstance()->Start();
} }
void NewMemManagerTest::TearDown() { zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() {
engine::ResMgrInst::GetInstance()->Stop(); auto options = engine::OptionsFactory::Build();
engine::SchedInst::GetInstance()->Stop(); options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = DBTestEnvironment::getURI();
if(options.meta.backend_uri.empty()) {
options.meta.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/";
}
return options;
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) { int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
if (argc > 1) { if (argc > 1) {
......
cpp/unittest/db/utils.h
浏览文件 @ 5986de3f
...@@ -34,44 +34,29 @@ ...@@ -34,44 +34,29 @@
void ASSERT_STATS(zilliz::milvus::engine::Status &stat); void ASSERT_STATS(zilliz::milvus::engine::Status &stat);
//class TestEnv : public ::testing::Environment { class BaseTest : public ::testing::Test {
//public: protected:
// void InitLog();
// static std::string getURI() { virtual zilliz::milvus::engine::Options GetOptions();
// if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) { };
// return uri;
// } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else { class DBTest : public BaseTest {
// return "";
// }
// }
//
// void SetUp() override {
// getURI();
// }
//
//};
//
//::testing::Environment* const test_env =
// ::testing::AddGlobalTestEnvironment(new TestEnv);
class DBTest : public ::testing::Test {
protected: protected:
zilliz::milvus::engine::DB *db_; zilliz::milvus::engine::DB *db_;
void InitLog();
virtual void SetUp() override; virtual void SetUp() override;
virtual void TearDown() override; virtual void TearDown() override;
virtual zilliz::milvus::engine::Options GetOptions();
}; };
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DBTest2 : public DBTest { class DBTest2 : public DBTest {
protected: protected:
virtual zilliz::milvus::engine::Options GetOptions() override; virtual zilliz::milvus::engine::Options GetOptions() override;
}; };
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MetaTest : public DBTest { class MetaTest : public BaseTest {
protected: protected:
std::shared_ptr<zilliz::milvus::engine::meta::SqliteMetaImpl> impl_; std::shared_ptr<zilliz::milvus::engine::meta::SqliteMetaImpl> impl_;
...@@ -79,19 +64,23 @@ class MetaTest : public DBTest { ...@@ -79,19 +64,23 @@ class MetaTest : public DBTest {
virtual void TearDown() override; virtual void TearDown() override;
}; };
class MySQLTest : public ::testing::Test { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
protected: class MySqlDBTest : public DBTest {
// std::shared_ptr<zilliz::milvus::engine::meta::MySQLMetaImpl> impl_; protected:
zilliz::milvus::engine::DBMetaOptions getDBMetaOptions(); zilliz::milvus::engine::Options GetOptions();
}; };
class MySQLDBTest : public DBTest { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MySqlMetaTest : public BaseTest {
protected: protected:
std::shared_ptr<zilliz::milvus::engine::meta::MySQLMetaImpl> impl_;
virtual void SetUp() override;
virtual void TearDown() override;
zilliz::milvus::engine::Options GetOptions(); zilliz::milvus::engine::Options GetOptions();
}; };
class NewMemManagerTest : public ::testing::Test {
void InitLog(); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void SetUp() override; class MemManagerTest : public DBTest {
void TearDown() override;
}; };
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或