Skip to content

M
milvus

milvus / milvus
12 个月 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 c9320225 编写于 作者: S starlord
浏览文件
操作

format unittest code 


Former-commit-id: f5979db268dc71e79ef556617fb8992dda04a422
上级 aac8065c
展开全部 隐藏空白更改
内联 并排
Showing with 1344 addition and 1267 deletion
cpp/CMakeLists.txt
浏览文件 @ c9320225
......@@ -189,7 +189,7 @@ add_custom_target(lint
--exclude_globs
${LINT_EXCLUSIONS_FILE}
--source_dir
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}
${MILVUS_LINT_QUIET})
#
......
cpp/build-support/lint_exclusions.txt
浏览文件 @ c9320225
......@@ -3,6 +3,10 @@
*cmake_build*
*src/thirdparty*
*src/core/thirdparty*
*thirdparty*
*src/grpc*
*easylogging++*
*SqliteMetaImpl.cpp
\ No newline at end of file
*SqliteMetaImpl.cpp
*src/core*
*src/wrapper*
*unittest/wrapper*
\ No newline at end of file
cpp/build.sh
浏览文件 @ c9320225
......@@ -99,21 +99,26 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
# cpplint check
make lint
if [ $? -ne 0 ]; then
echo "ERROR! cpplint check not pass"
exit 1
fi
# clang-format check
make check-clang-format
if [ $? -ne 0 ]; then
echo "ERROR! clang-format check failed"
exit 1
fi
# clang-tidy check
make check-clang-tidy
if [ $? -ne 0 ]; then
echo "ERROR! clang-tidy check failed"
echo "ERROR! cpplint check failed"
exit 1
fi
echo "cpplint check pass!"
# # clang-format check
# make check-clang-format
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-format check failed"
# exit 1
# fi
# echo "clang-format check pass!"
#
# # clang-tidy check
# make check-clang-tidy
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-tidy check failed"
# exit 1
# fi
# echo "clang-tidy check pass!"
else
# compile and build
make -j 4 || exit 1
......
cpp/src/db/meta/MetaFactory.cpp
浏览文件 @ c9320225
......@@ -40,9 +40,8 @@ MetaFactory::BuildOption(const std::string &path) {
if (p == "") {
srand(time(nullptr));
std::stringstream ss;
uint32_t rd = 0;
rand_r(&rd);
ss << "/tmp/" << rd;
uint32_t seed = 1;
ss << "/tmp/" << rand_r(&seed);
p = ss.str();
}
......
cpp/unittest/CMakeLists.txt
浏览文件 @ c9320225
......@@ -118,6 +118,8 @@ set(unittest_libs
cublas
)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
message(STATUS "CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}")
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
......
cpp/unittest/db/db_tests.cpp
浏览文件 @ c9320225
此差异已折叠。
cpp/unittest/db/engine_test.cpp
浏览文件 @ c9320225
......@@ -21,67 +21,66 @@
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "utils.h"
#include "db/utils.h"
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
}
TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::INVALID,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::INVALID,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr == nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::FAISS_IDMAP,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IDMAP,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::FAISS_IVFFLAT,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IVFFLAT,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::FAISS_IVFSQ8,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IVFSQ8,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::NSG_MIX,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::NSG_MIX,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
......@@ -90,27 +89,26 @@ TEST_F(EngineTest, FACTORY_TEST) {
TEST_F(EngineTest, ENGINE_IMPL_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
dimension,
file_path,
engine::EngineType::FAISS_IVFFLAT,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IVFFLAT,
ms::engine::MetricType::IP,
1024);
std::vector<float> data;
std::vector<long> ids;
std::vector<int64_t> ids;
const int row_count = 10000;
data.reserve(row_count*dimension);
ids.reserve(row_count);
for(long i = 0; i < row_count; i++) {
for (int64_t i = 0; i < row_count; i++) {
ids.push_back(i);
for(uint16_t k = 0; k < dimension; k++) {
for (uint16_t k = 0; k < dimension; k++) {
data.push_back(i*dimension + k);
}
}
auto status = engine_ptr->AddWithIds((long)ids.size(), data.data(), ids.data());
auto status = engine_ptr->AddWithIds((int64_t)ids.size(), data.data(), ids.data());
ASSERT_TRUE(status.ok());
ASSERT_EQ(engine_ptr->Dimension(), dimension);
......@@ -127,5 +125,4 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
//
// auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", engine::EngineType::FAISS_IVFSQ8);
// //ASSERT_TRUE(status.ok());
}
cpp/unittest/db/mem_test.cpp
浏览文件 @ c9320225
......@@ -25,7 +25,7 @@
#include "db/engine/EngineFactory.h"
#include "db/meta/MetaConsts.h"
#include "metrics/Metrics.h"
#include "utils.h"
#include "db/utils.h"
#include <boost/filesystem.hpp>
#include <thread>
......@@ -35,30 +35,32 @@
#include <random>
#include <chrono>
using namespace zilliz::milvus;
namespace {
static std::string TABLE_NAME = "test_group";
namespace ms = zilliz::milvus;
static constexpr int64_t TABLE_DIM = 256;
std::string GenTableName() {
std::string
GetTableName() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
TABLE_NAME = std::to_string(micros);
return TABLE_NAME;
now.time_since_epoch()).count();
static std::string table_name = std::to_string(micros);
return table_name;
}
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
ms::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = GenTableName();
table_info.engine_type_ = (int) engine::EngineType::FAISS_IDMAP;
table_info.table_id_ = GetTableName();
table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void BuildVectors(int64_t n, std::vector<float> &vectors) {
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
......@@ -67,15 +69,15 @@ void BuildVectors(int64_t n, std::vector<float> &vectors) {
data[TABLE_DIM * i + j] = drand48();
}
}
}
} // namespace
TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
engine::meta::TableSchema table_schema = BuildTableSchema();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
engine::meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = TABLE_NAME;
ms::engine::meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = GetTableName();
status = impl_->CreateTableFile(table_file_schema);
ASSERT_TRUE(status.ok());
......@@ -83,17 +85,17 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
std::vector<float> vectors;
BuildVectors(n, vectors);
engine::VectorSource source(n, vectors.data());
ms::engine::VectorSource source(n, vectors.data());
size_t num_vectors_added;
engine::ExecutionEnginePtr execution_engine_ =
engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(engine::EngineType) table_file_schema.engine_type_,
(engine::MetricType)table_file_schema.metric_type_,
table_schema.nlist_);
engine::IDNumbers vector_ids;
ms::engine::ExecutionEnginePtr execution_engine_ =
ms::engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(ms::engine::EngineType) table_file_schema.engine_type_,
(ms::engine::MetricType) table_file_schema.metric_type_,
table_schema.nlist_);
ms::engine::IDNumbers vector_ids;
status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids);
ASSERT_TRUE(status.ok());
vector_ids = source.GetVectorIds();
......@@ -113,19 +115,19 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
auto options = GetOptions();
engine::meta::TableSchema table_schema = BuildTableSchema();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
engine::MemTableFile mem_table_file(TABLE_NAME, impl_, options);
ms::engine::MemTableFile mem_table_file(GetTableName(), impl_, options);
int64_t n_100 = 100;
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
engine::VectorSourcePtr source = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
ms::engine::VectorSourcePtr source = std::make_shared<ms::engine::VectorSource>(n_100, vectors_100.data());
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
status = mem_table_file.Add(source, vector_ids);
ASSERT_TRUE(status.ok());
......@@ -137,11 +139,11 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem);
int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem;
int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
engine::VectorSourcePtr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
ms::engine::VectorSourcePtr source_128M = std::make_shared<ms::engine::VectorSource>(n_max, vectors_128M.data());
vector_ids.clear();
status = mem_table_file.Add(source_128M, vector_ids);
......@@ -154,7 +156,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
TEST_F(MemManagerTest, MEM_TABLE_TEST) {
auto options = GetOptions();
engine::meta::TableSchema table_schema = BuildTableSchema();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
......@@ -162,27 +164,27 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
engine::VectorSourcePtr source_100 = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
ms::engine::VectorSourcePtr source_100 = std::make_shared<ms::engine::VectorSource>(n_100, vectors_100.data());
engine::MemTable mem_table(TABLE_NAME, impl_, options);
ms::engine::MemTable mem_table(GetTableName(), impl_, options);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
status = mem_table.Add(source_100, vector_ids);
ASSERT_TRUE(status.ok());
vector_ids = source_100->GetVectorIds();
ASSERT_EQ(vector_ids.size(), 100);
engine::MemTableFilePtr mem_table_file;
ms::engine::MemTableFilePtr mem_table_file;
mem_table.GetCurrentMemTableFile(mem_table_file);
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem);
int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem;
int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
engine::VectorSourcePtr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
ms::engine::VectorSourcePtr source_128M = std::make_shared<ms::engine::VectorSource>(n_max, vectors_128M.data());
status = mem_table.Add(source_128M, vector_ids);
ASSERT_TRUE(status.ok());
......@@ -198,7 +200,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_1G;
BuildVectors(n_1G, vectors_1G);
engine::VectorSourcePtr source_1G = std::make_shared<engine::VectorSource>(n_1G, vectors_1G.data());
ms::engine::VectorSourcePtr source_1G = std::make_shared<ms::engine::VectorSource>(n_1G, vectors_1G.data());
vector_ids.clear();
status = mem_table.Add(source_1G, vector_ids);
......@@ -207,7 +209,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
vector_ids = source_1G->GetVectorIds();
ASSERT_EQ(vector_ids.size(), n_1G);
int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM);
int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / ms::engine::MAX_TABLE_FILE_MEM);
ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount);
status = mem_table.Serialize();
......@@ -215,11 +217,11 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
}
TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
......@@ -228,12 +230,12 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
for(int64_t i = 0; i < nb; i++) {
ms::engine::IDNumbers vector_ids;
for (int64_t i = 0; i < nb; i++) {
vector_ids.push_back(i);
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk
......@@ -256,19 +258,19 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
int topk = 10, nprobe = 10;
for (auto &pair : search_vectors) {
auto &search = pair.second;
engine::QueryResults results;
stat = db_->Query(TABLE_NAME, topk, 1, nprobe, search.data(), results);
ms::engine::QueryResults results;
stat = db_->Query(GetTableName(), topk, 1, nprobe, search.data(), results);
ASSERT_EQ(results[0][0].first, pair.first);
ASSERT_LT(results[0][0].second, 1e-4);
}
}
TEST_F(MemManagerTest2, INSERT_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
......@@ -280,8 +282,8 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
int64_t nb = 40960;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ms::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
}
auto end_time = METRICS_NOW_TIME;
......@@ -290,17 +292,17 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
}
TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int64_t nb = 40960;
std::vector<float> xb;
......@@ -311,7 +313,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
BuildVectors(qb, qxb);
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
......@@ -326,8 +328,8 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
prev_count = count;
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / engine::meta::M << " M";
stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
......@@ -349,29 +351,28 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
for (auto i = 0; i < loop; ++i) {
if (i == 0) {
db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
db_->InsertVectors(GetTableName(), qb, qxb.data(), target_ids);
ASSERT_EQ(target_ids.size(), qb);
} else {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
}
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
search.join();
};
}
TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
int64_t nb = 100000;
std::vector<float> xb;
......@@ -382,7 +383,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
vector_ids[i] = i;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], 0);
ASSERT_TRUE(stat.ok());
......@@ -394,7 +395,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
for (auto i = 0; i < nb; i++) {
vector_ids[i] = i + nb;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb);
ASSERT_TRUE(stat.ok());
......@@ -406,15 +407,15 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
for (auto i = 0; i < nb; i++) {
vector_ids[i] = i + nb / 2;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb/2);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb / 2);
ASSERT_TRUE(stat.ok());
nb = 65536; //128M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
nb = 100;
......@@ -425,8 +426,9 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
for (auto i = 0; i < nb; i++) {
vector_ids[i] = i + nb;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
for (auto i = 0; i < nb; i++) {
ASSERT_EQ(vector_ids[i], i + nb);
}
}
cpp/unittest/db/meta_tests.cpp
浏览文件 @ c9320225
......@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
......@@ -25,13 +25,16 @@
#include <stdlib.h>
#include <time.h>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine;
namespace {
namespace ms = zilliz::milvus;
} // namespace
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
......@@ -49,7 +52,7 @@ TEST_F(MetaTest, TABLE_TEST) {
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
table.table_id_ = "";
status = impl_->CreateTable(table);
......@@ -59,16 +62,16 @@ TEST_F(MetaTest, TABLE_TEST) {
TEST_F(MetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW);
uint64_t cnt = 0;
status = impl_->Count(table_id, cnt);
......@@ -77,30 +80,30 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
auto file_id = table_file.file_id_;
auto new_file_type = meta::TableFileSchema::INDEX;
auto new_file_type = ms::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, new_file_type);
meta::DatesT dates;
dates.push_back(utils::GetDate());
ms::engine::meta::DatesT dates;
dates.push_back(ms::engine::utils::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(utils::GetDateWithDelta(-1*i));
for (auto i = 2; i < 10; ++i) {
dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i));
}
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date_ = utils::GetDateWithDelta(-2);
table_file.date_ = ms::engine::utils::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date_);
......@@ -108,7 +111,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_};
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), 0UL);
......@@ -116,33 +119,34 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0));
DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
int days_num = rand() % 100;
unsigned int seed = 1;
int days_num = rand_r(&seed) % 100;
std::stringstream ss;
ss << "days:" << days_num;
options.archive_conf_ = ArchiveConf("delete", ss.str());
options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str());
meta::SqliteMetaImpl impl(options);
ms::engine::meta::SqliteMetaImpl impl(options);
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
int64_t ts = ms::engine::utils::GetMicroSecTimeStamp();
std::vector<int> days;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
table_file.created_on_ = ts - day*meta::D_SEC*meta::US_PS - 10000;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
......@@ -152,13 +156,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
i++;
}
......@@ -167,28 +171,28 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
}
TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
options.archive_conf_ = ArchiveConf("delete", "disk:11");
options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11");
meta::SqliteMetaImpl impl(options);
ms::engine::meta::SqliteMetaImpl impl(options);
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
auto each_size = 2UL;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * meta::G;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::meta::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
......@@ -197,13 +201,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
++i;
}
......@@ -214,7 +218,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
......@@ -226,51 +230,51 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
uint64_t to_index_files_cnt = 6;
uint64_t index_files_cnt = 7;
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_merge_files_cnt; ++i) {
for (auto i = 0; i < new_merge_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_index_files_cnt; ++i) {
for (auto i = 0; i < new_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<backup_files_cnt; ++i) {
for (auto i = 0; i < backup_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::BACKUP;
table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_files_cnt; ++i) {
for (auto i = 0; i < new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
for (auto i = 0; i < raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::RAW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
for (auto i = 0; i < to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
for (auto i = 0; i < index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
......@@ -278,36 +282,36 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
uint64_t total_row_count = 0;
status = impl_->Count(table_id, total_row_count);
ASSERT_TRUE(status.ok());
ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt);
ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt);
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
ms::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_};
ms::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),0);
ASSERT_EQ(dated_files[table_file.date_].size(), 0);
std::vector<int> file_types;
std::vector<std::string> file_ids;
......@@ -316,26 +320,26 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
meta::TableFileSchema::NEW,
meta::TableFileSchema::NEW_MERGE,
meta::TableFileSchema::NEW_INDEX,
meta::TableFileSchema::TO_INDEX,
meta::TableFileSchema::INDEX,
meta::TableFileSchema::RAW,
meta::TableFileSchema::BACKUP,
ms::engine::meta::TableFileSchema::NEW,
ms::engine::meta::TableFileSchema::NEW_MERGE,
ms::engine::meta::TableFileSchema::NEW_INDEX,
ms::engine::meta::TableFileSchema::TO_INDEX,
ms::engine::meta::TableFileSchema::INDEX,
ms::engine::meta::TableFileSchema::RAW,
ms::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);
ASSERT_TRUE(status.ok());
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
status = impl_->CleanUp();
ASSERT_TRUE(status.ok());
......@@ -350,11 +354,11 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
TEST_F(MetaTest, INDEX_TEST) {
auto table_id = "index_test";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
TableIndex index;
ms::engine::TableIndex index;
index.metric_type_ = 2;
index.nlist_ = 1234;
index.engine_type_ = 3;
......@@ -365,12 +369,12 @@ TEST_F(MetaTest, INDEX_TEST) {
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
engine::meta::TableSchema table_info;
ms::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
ASSERT_EQ(table_info.flag_, flag);
TableIndex index_out;
ms::engine::TableIndex index_out;
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_EQ(index_out.metric_type_, index.metric_type_);
ASSERT_EQ(index_out.nlist_, index.nlist_);
......@@ -385,4 +389,4 @@ TEST_F(MetaTest, INDEX_TEST) {
status = impl_->UpdateTableFilesToIndex(table_id);
ASSERT_TRUE(status.ok());
}
\ No newline at end of file
}
cpp/unittest/db/misc_test.cpp
浏览文件 @ c9320225
......@@ -27,43 +27,47 @@
#include <boost/filesystem.hpp>
#include <vector>
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
} // namespace
TEST(DBMiscTest, EXCEPTION_TEST) {
Exception ex1(100, "error");
ms::Exception ex1(100, "error");
std::string what = ex1.what();
ASSERT_EQ(what, "error");
ASSERT_EQ(ex1.code(), 100);
InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT);
ms::InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), ms::SERVER_INVALID_ARGUMENT);
}
TEST(DBMiscTest, OPTIONS_TEST) {
try {
engine::ArchiveConf archive("$$##");
} catch (std::exception& ex) {
ms::engine::ArchiveConf archive("$$##");
} catch (std::exception &ex) {
ASSERT_TRUE(true);
}
{
engine::ArchiveConf archive("delete", "no");
ms::engine::ArchiveConf archive("delete", "no");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
engine::ArchiveConf archive("delete", "1:2");
ms::engine::ArchiveConf archive("delete", "1:2");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
engine::ArchiveConf archive("delete", "1:2:3");
ms::engine::ArchiveConf archive("delete", "1:2:3");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
engine::ArchiveConf archive("delete");
engine::ArchiveConf::CriteriaT criterial = {
ms::engine::ArchiveConf archive("delete");
ms::engine::ArchiveConf::CriteriaT criterial = {
{"disk", 1024},
{"days", 100}
};
......@@ -76,29 +80,29 @@ TEST(DBMiscTest, OPTIONS_TEST) {
}
TEST(DBMiscTest, META_TEST) {
engine::DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
engine::meta::SqliteMetaImpl impl(options);
ms::engine::meta::SqliteMetaImpl impl(options);
time_t tt;
time( &tt );
time(&tt);
int delta = 10;
engine::meta::DateT dt = engine::utils::GetDate(tt, delta);
ms::engine::meta::DateT dt = ms::engine::utils::GetDate(tt, delta);
ASSERT_GT(dt, 0);
}
TEST(DBMiscTest, UTILS_TEST) {
engine::DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test/main";
options.slave_paths_.push_back("/tmp/milvus_test/slave_1");
options.slave_paths_.push_back("/tmp/milvus_test/slave_2");
const std::string TABLE_NAME = "test_tbl";
auto status = engine::utils::CreateTablePath(options, TABLE_NAME);
auto status = ms::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for(auto& path : options.slave_paths_) {
ASSERT_TRUE(boost::filesystem::exists(path));
for (auto &path : options.slave_paths_) {
ASSERT_TRUE(boost::filesystem::exists(path));
}
// options.slave_paths.push_back("/");
......@@ -109,18 +113,18 @@ TEST(DBMiscTest, UTILS_TEST) {
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
engine::meta::TableFileSchema file;
ms::engine::meta::TableFileSchema file;
file.id_ = 50;
file.table_id_ = TABLE_NAME;
file.file_type_ = 3;
file.date_ = 155000;
status = engine::utils::GetTableFilePath(options, file);
status = ms::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(file.location_.empty());
status = engine::utils::DeleteTablePath(options, TABLE_NAME);
status = ms::engine::utils::DeleteTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
status = engine::utils::DeleteTableFilePath(options, file);
status = ms::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
}
\ No newline at end of file
}
cpp/unittest/db/mysql_db_test.cpp
浏览文件 @ c9320225
......@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/meta/MetaConsts.h"
......@@ -26,48 +26,49 @@
#include <thread>
#include <random>
using namespace zilliz::milvus;
namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
return table_info;
}
namespace ms = zilliz::milvus;
void BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n*TABLE_DIM);
float* data = vectors.data();
for(int i = 0; i < n; i++) {
for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
static const char *TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
ms::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
} // namespace
TEST_F(MySqlDBTest, DB_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int64_t nb = 50;
std::vector<float> xb;
......@@ -81,7 +82,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
ASSERT_EQ(target_ids.size(), qb);
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(5));
......@@ -90,22 +91,22 @@ TEST_F(MySqlDBTest, DB_TEST) {
uint64_t count = 0;
uint64_t prev_count = 0;
for (auto j=0; j<10; ++j) {
for (auto j = 0; j < 10; ++j) {
ss.str("");
db_->Size(count);
prev_count = count;
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
for (auto k=0; k<qb; ++k) {
for (auto k = 0; k < qb; ++k) {
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
bool exists = false;
for (auto& result : results[k]) {
for (auto &result : results[k]) {
if (result.first == target_ids[k]) {
exists = true;
}
......@@ -127,7 +128,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
for (auto i = 0; i < loop; ++i) {
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
......@@ -139,13 +140,13 @@ TEST_F(MySqlDBTest, DB_TEST) {
}
search.join();
};
}
TEST_F(MySqlDBTest, SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
......@@ -155,68 +156,68 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
size_t nb = VECTOR_COUNT;
size_t nq = 10;
size_t k = 5;
std::vector<float> xb(nb*TABLE_DIM);
std::vector<float> xq(nq*TABLE_DIM);
std::vector<long> ids(nb);
std::vector<float> xb(nb * TABLE_DIM);
std::vector<float> xq(nq * TABLE_DIM);
std::vector<int64_t> ids(nb);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
for (size_t i = 0; i < nb*TABLE_DIM; i++) {
for (size_t i = 0; i < nb * TABLE_DIM; i++) {
xb[i] = dis_xt(gen);
if (i < nb){
if (i < nb) {
ids[i] = i;
}
}
for (size_t i = 0; i < nq*TABLE_DIM; i++) {
for (size_t i = 0; i < nq * TABLE_DIM; i++) {
xq[i] = dis_xt(gen);
}
// result data
//std::vector<long> nns_gt(k*nq);
std::vector<long> nns(k*nq); // nns = nearst neg search
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids);
if (j == 200){ sleep(1);}
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
ASSERT_TRUE(stat.ok());
}
sleep(2); // wait until build index finish
engine::QueryResults results;
ms::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_TRUE(stat.ok());
};
}
TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
std::vector<engine::meta::TableSchema> table_schema_array;
std::vector<ms::engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for(auto& schema : table_schema_array) {
if(schema.table_id_ == TABLE_NAME) {
for (auto &schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;
}
}
ASSERT_TRUE(bfound);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
uint64_t size;
db_->Size(size);
......@@ -226,7 +227,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
BuildVectors(nb, xb);
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
......@@ -235,15 +236,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * engine::meta::G);
};
ASSERT_LE(size, 1 * ms::engine::meta::G);
}
TEST_F(MySqlDBTest, DELETE_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
// std::cout << stat.ToString() << std::endl;
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
......@@ -252,7 +253,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
db_->HasTable(TABLE_NAME, has_table);
ASSERT_TRUE(has_table);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
uint64_t size;
db_->Size(size);
......@@ -262,7 +263,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
BuildVectors(nb, xb);
int loop = 20;
for (auto i=0; i<loop; ++i) {
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
......@@ -276,4 +277,5 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
//
// db_->HasTable(TABLE_NAME, has_table);
// ASSERT_FALSE(has_table);
};
}
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @ c9320225
......@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
......@@ -27,13 +27,17 @@
#include <gtest/gtest.h>
#include <mysql++/mysql++.h>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine;
namespace {
namespace ms = zilliz::milvus;
}
TEST_F(MySqlMetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
......@@ -51,7 +55,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
table.table_id_ = "";
status = impl_->CreateTable(table);
......@@ -64,20 +68,19 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW);
meta::DatesT dates;
dates.push_back(utils::GetDate());
ms::engine::meta::DatesT dates;
dates.push_back(ms::engine::utils::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
......@@ -88,7 +91,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto file_id = table_file.file_id_;
auto new_file_type = meta::TableFileSchema::INDEX;
auto new_file_type = ms::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
......@@ -96,17 +99,17 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
ASSERT_EQ(table_file.file_type_, new_file_type);
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(utils::GetDateWithDelta(-1*i));
for (auto i = 2; i < 10; ++i) {
dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i));
}
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date_ = utils::GetDateWithDelta(-2);
table_file.date_ = ms::engine::utils::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date_);
......@@ -114,41 +117,42 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_};
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_EQ(files.size(), 0UL);
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0));
DBMetaOptions options = GetOptions().meta_;
ms::engine::DBMetaOptions options = GetOptions().meta_;
int days_num = rand() % 100;
unsigned int seed = 1;
int days_num = rand_r(&seed) % 100;
std::stringstream ss;
ss << "days:" << days_num;
options.archive_conf_ = ArchiveConf("delete", ss.str());
int mode = DBOptions::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str());
int mode = ms::engine::DBOptions::MODE::SINGLE;
ms::engine::meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
int64_t ts = ms::engine::utils::GetMicroSecTimeStamp();
std::vector<int> days;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
table_file.created_on_ = ts - day*meta::D_SEC*meta::US_PS - 10000;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
......@@ -158,19 +162,19 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
i++;
}
std::vector<int> file_types = {
(int) meta::TableFileSchema::NEW,
(int) ms::engine::meta::TableFileSchema::NEW,
};
std::vector<std::string> file_ids;
status = impl.FilesByType(table_id, file_types, file_ids);
......@@ -184,32 +188,32 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
DBMetaOptions options = GetOptions().meta_;
ms::engine::DBMetaOptions options = GetOptions().meta_;
options.archive_conf_ = ArchiveConf("delete", "disk:11");
int mode = DBOptions::MODE::SINGLE;
auto impl = meta::MySQLMetaImpl(options, mode);
options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11");
int mode = ms::engine::DBOptions::MODE::SINGLE;
auto impl = ms::engine::meta::MySQLMetaImpl(options, mode);
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableSchema table_schema;
ms::engine::meta::TableSchema table_schema;
table_schema.table_id_ = "";
status = impl.CreateTable(table_schema);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
auto each_size = 2UL;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * meta::G;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::meta::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
......@@ -218,13 +222,13 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
++i;
}
......@@ -236,7 +240,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
......@@ -248,51 +252,51 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
uint64_t to_index_files_cnt = 6;
uint64_t index_files_cnt = 7;
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_merge_files_cnt; ++i) {
for (auto i = 0; i < new_merge_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_index_files_cnt; ++i) {
for (auto i = 0; i < new_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<backup_files_cnt; ++i) {
for (auto i = 0; i < backup_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::BACKUP;
table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_files_cnt; ++i) {
for (auto i = 0; i < new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
for (auto i = 0; i < raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::RAW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
for (auto i = 0; i < to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
for (auto i = 0; i < index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
......@@ -300,36 +304,36 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
uint64_t total_row_count = 0;
status = impl_->Count(table_id, total_row_count);
ASSERT_TRUE(status.ok());
ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt);
ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt);
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
ms::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_};
ms::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),0);
ASSERT_EQ(dated_files[table_file.date_].size(), 0);
std::vector<int> file_types;
std::vector<std::string> file_ids;
......@@ -338,19 +342,19 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
meta::TableFileSchema::NEW,
meta::TableFileSchema::NEW_MERGE,
meta::TableFileSchema::NEW_INDEX,
meta::TableFileSchema::TO_INDEX,
meta::TableFileSchema::INDEX,
meta::TableFileSchema::RAW,
meta::TableFileSchema::BACKUP,
ms::engine::meta::TableFileSchema::NEW,
ms::engine::meta::TableFileSchema::NEW_MERGE,
ms::engine::meta::TableFileSchema::NEW_INDEX,
ms::engine::meta::TableFileSchema::TO_INDEX,
ms::engine::meta::TableFileSchema::INDEX,
ms::engine::meta::TableFileSchema::RAW,
ms::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);
......@@ -366,11 +370,11 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
TEST_F(MySqlMetaTest, INDEX_TEST) {
auto table_id = "index_test";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
TableIndex index;
ms::engine::TableIndex index;
index.metric_type_ = 2;
index.nlist_ = 1234;
index.engine_type_ = 3;
......@@ -381,12 +385,12 @@ TEST_F(MySqlMetaTest, INDEX_TEST) {
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
engine::meta::TableSchema table_info;
ms::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
ASSERT_EQ(table_info.flag_, flag);
TableIndex index_out;
ms::engine::TableIndex index_out;
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_EQ(index_out.metric_type_, index.metric_type_);
ASSERT_EQ(index_out.nlist_, index.nlist_);
......
cpp/unittest/db/search_test.cpp
浏览文件 @ c9320225
......@@ -22,53 +22,55 @@
#include "scheduler/task/SearchTask.h"
#include "utils/TimeRecorder.h"
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
static constexpr uint64_t NQ = 15;
static constexpr uint64_t TOP_K = 64;
void BuildResult(uint64_t nq,
uint64_t topk,
bool ascending,
std::vector<long> &output_ids,
std::vector<float> &output_distence) {
void
BuildResult(uint64_t nq,
uint64_t topk,
bool ascending,
std::vector<int64_t> &output_ids,
std::vector<float> &output_distence) {
output_ids.clear();
output_ids.resize(nq*topk);
output_ids.resize(nq * topk);
output_distence.clear();
output_distence.resize(nq*topk);
output_distence.resize(nq * topk);
for(uint64_t i = 0; i < nq; i++) {
for(uint64_t j = 0; j < topk; j++) {
output_ids[i * topk + j] = (long)(drand48()*100000);
for (uint64_t i = 0; i < nq; i++) {
for (uint64_t j = 0; j < topk; j++) {
output_ids[i * topk + j] = (int64_t) (drand48() * 100000);
output_distence[i * topk + j] = ascending ? (j + drand48()) : ((topk - j) + drand48());
}
}
}
void CheckResult(const scheduler::Id2DistanceMap& src_1,
const scheduler::Id2DistanceMap& src_2,
const scheduler::Id2DistanceMap& target,
bool ascending) {
for(uint64_t i = 0; i < target.size() - 1; i++) {
if(ascending) {
void
CheckResult(const ms::scheduler::Id2DistanceMap &src_1,
const ms::scheduler::Id2DistanceMap &src_2,
const ms::scheduler::Id2DistanceMap &target,
bool ascending) {
for (uint64_t i = 0; i < target.size() - 1; i++) {
if (ascending) {
ASSERT_LE(target[i].second, target[i + 1].second);
} else {
ASSERT_GE(target[i].second, target[i + 1].second);
}
}
using ID2DistMap = std::map<long, float>;
using ID2DistMap = std::map<int64_t, float>;
ID2DistMap src_map_1, src_map_2;
for(const auto& pair : src_1) {
for (const auto &pair : src_1) {
src_map_1.insert(pair);
}
for(const auto& pair : src_2) {
for (const auto &pair : src_2) {
src_map_2.insert(pair);
}
for(const auto& pair : target) {
for (const auto &pair : target) {
ASSERT_TRUE(src_map_1.find(pair.first) != src_map_1.end() || src_map_2.find(pair.first) != src_map_2.end());
float dist = src_map_1.find(pair.first) != src_map_1.end() ? src_map_1[pair.first] : src_map_2[pair.first];
......@@ -76,40 +78,42 @@ void CheckResult(const scheduler::Id2DistanceMap& src_1,
}
}
void CheckCluster(const std::vector<long>& target_ids,
const std::vector<float>& target_distence,
const scheduler::ResultSet& src_result,
int64_t nq,
int64_t topk) {
void
CheckCluster(const std::vector<int64_t> &target_ids,
const std::vector<float> &target_distence,
const ms::scheduler::ResultSet &src_result,
int64_t nq,
int64_t topk) {
ASSERT_EQ(src_result.size(), nq);
for(int64_t i = 0; i < nq; i++) {
auto& res = src_result[i];
for (int64_t i = 0; i < nq; i++) {
auto &res = src_result[i];
ASSERT_EQ(res.size(), topk);
if(res.empty()) {
if (res.empty()) {
continue;
}
ASSERT_EQ(res[0].first, target_ids[i*topk]);
ASSERT_EQ(res[topk - 1].first, target_ids[i*topk + topk - 1]);
ASSERT_EQ(res[0].first, target_ids[i * topk]);
ASSERT_EQ(res[topk - 1].first, target_ids[i * topk + topk - 1]);
}
}
void CheckTopkResult(const scheduler::ResultSet& src_result,
bool ascending,
int64_t nq,
int64_t topk) {
void
CheckTopkResult(const ms::scheduler::ResultSet &src_result,
bool ascending,
int64_t nq,
int64_t topk) {
ASSERT_EQ(src_result.size(), nq);
for(int64_t i = 0; i < nq; i++) {
auto& res = src_result[i];
for (int64_t i = 0; i < nq; i++) {
auto &res = src_result[i];
ASSERT_EQ(res.size(), topk);
if(res.empty()) {
if (res.empty()) {
continue;
}
for(int64_t k = 0; k < topk - 1; k++) {
if(ascending) {
for (int64_t k = 0; k < topk - 1; k++) {
if (ascending) {
ASSERT_LE(res[k].second, res[k + 1].second);
} else {
ASSERT_GE(res[k].second, res[k + 1].second);
......@@ -118,87 +122,87 @@ void CheckTopkResult(const scheduler::ResultSet& src_result,
}
}
}
} // namespace
TEST(DBSearchTest, TOPK_TEST) {
bool ascending = true;
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
scheduler::ResultSet src_result;
auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
ms::scheduler::ResultSet src_result;
auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(src_result.empty());
BuildResult(NQ, TOP_K, ascending, target_ids, target_distence);
status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(src_result.size(), NQ);
scheduler::ResultSet target_result;
status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result);
ms::scheduler::ResultSet target_result;
status = ms::scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result);
status = ms::scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result);
ASSERT_FALSE(status.ok());
status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(src_result.empty());
ASSERT_EQ(target_result.size(), NQ);
std::vector<long> src_ids;
std::vector<int64_t> src_ids;
std::vector<float> src_distence;
uint64_t wrong_topk = TOP_K - 10;
BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence);
status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result);
status = ms::scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result);
ASSERT_TRUE(status.ok());
status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
for(uint64_t i = 0; i < NQ; i++) {
for (uint64_t i = 0; i < NQ; i++) {
ASSERT_EQ(target_result[i].size(), TOP_K);
}
wrong_topk = TOP_K + 10;
BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence);
status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
for(uint64_t i = 0; i < NQ; i++) {
for (uint64_t i = 0; i < NQ; i++) {
ASSERT_EQ(target_result[i].size(), TOP_K);
}
}
TEST(DBSearchTest, MERGE_TEST) {
bool ascending = true;
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
std::vector<long> src_ids;
std::vector<int64_t> src_ids;
std::vector<float> src_distence;
scheduler::ResultSet src_result, target_result;
ms::scheduler::ResultSet src_result, target_result;
uint64_t src_count = 5, target_count = 8;
BuildResult(1, src_count, ascending, src_ids, src_distence);
BuildResult(1, target_count, ascending, target_ids, target_distence);
auto status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result);
auto status = ms::scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result);
ASSERT_TRUE(status.ok());
status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result);
status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result);
ASSERT_TRUE(status.ok());
{
scheduler::Id2DistanceMap src = src_result[0];
scheduler::Id2DistanceMap target = target_result[0];
status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ms::scheduler::Id2DistanceMap src = src_result[0];
ms::scheduler::Id2DistanceMap target = target_result[0];
status = ms::scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), 10);
CheckResult(src_result[0], target_result[0], target, ascending);
}
{
scheduler::Id2DistanceMap src = src_result[0];
scheduler::Id2DistanceMap target;
status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ms::scheduler::Id2DistanceMap src = src_result[0];
ms::scheduler::Id2DistanceMap target;
status = ms::scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), src_count);
ASSERT_TRUE(src.empty());
......@@ -206,18 +210,18 @@ TEST(DBSearchTest, MERGE_TEST) {
}
{
scheduler::Id2DistanceMap src = src_result[0];
scheduler::Id2DistanceMap target = target_result[0];
status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ms::scheduler::Id2DistanceMap src = src_result[0];
ms::scheduler::Id2DistanceMap target = target_result[0];
status = ms::scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), src_count + target_count);
CheckResult(src_result[0], target_result[0], target, ascending);
}
{
scheduler::Id2DistanceMap target = src_result[0];
scheduler::Id2DistanceMap src = target_result[0];
status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ms::scheduler::Id2DistanceMap target = src_result[0];
ms::scheduler::Id2DistanceMap src = target_result[0];
status = ms::scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), src_count + target_count);
CheckResult(src_result[0], target_result[0], target, ascending);
......@@ -226,17 +230,17 @@ TEST(DBSearchTest, MERGE_TEST) {
TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) {
bool ascending = true;
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
scheduler::ResultSet src_result;
ms::scheduler::ResultSet src_result;
auto DoCluster = [&](int64_t nq, int64_t topk) {
TimeRecorder rc("DoCluster");
ms::TimeRecorder rc("DoCluster");
src_result.clear();
BuildResult(nq, topk, ascending, target_ids, target_distence);
rc.RecordSection("build id/dietance map");
auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(src_result.size(), nq);
......@@ -257,29 +261,33 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) {
}
TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
scheduler::ResultSet src_result;
ms::scheduler::ResultSet src_result;
std::vector<long> insufficient_ids;
std::vector<int64_t> insufficient_ids;
std::vector<float> insufficient_distence;
scheduler::ResultSet insufficient_result;
ms::scheduler::ResultSet insufficient_result;
auto DoTopk = [&](int64_t nq, int64_t topk,int64_t insufficient_topk, bool ascending) {
auto DoTopk = [&](int64_t nq, int64_t topk, int64_t insufficient_topk, bool ascending) {
src_result.clear();
insufficient_result.clear();
TimeRecorder rc("DoCluster");
ms::TimeRecorder rc("DoCluster");
BuildResult(nq, topk, ascending, target_ids, target_distence);
auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
rc.RecordSection("cluster result");
BuildResult(nq, insufficient_topk, ascending, insufficient_ids, insufficient_distence);
status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result);
status = ms::scheduler::XSearchTask::ClusterResult(target_ids,
target_distence,
nq,
insufficient_topk,
insufficient_result);
rc.RecordSection("cluster result");
scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result);
ms::scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result);
ASSERT_TRUE(status.ok());
rc.RecordSection("topk");
......@@ -292,4 +300,4 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
// DoTopk(9987, 12, 10, false);
// DoTopk(77777, 1000, 1, false);
// DoTopk(5432, 8899, 8899, true);
}
\ No newline at end of file
}
cpp/unittest/db/utils.cpp
浏览文件 @ c9320225
......@@ -18,9 +18,11 @@
#include <iostream>
#include <thread>
#include <memory>
#include <string>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "db/utils.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "db/DBFactory.h"
......@@ -29,83 +31,98 @@
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;
namespace {
static std::string uri;
namespace ms = zilliz::milvus;
class DBTestEnvironment : public ::testing::Environment {
public:
// explicit DBTestEnvironment(std::string uri) : uri_(uri) {}
public:
explicit DBTestEnvironment(const std::string& uri)
: uri_(uri) {
}
static std::string getURI() {
return uri;
std::string getURI() const {
return uri_;
}
void SetUp() override {
getURI();
}
private:
std::string uri_;
};
DBTestEnvironment* test_env = nullptr;
} // namespace
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void BaseTest::InitLog() {
void
BaseTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.set(el::Level::Debug,
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::Loggers::reconfigureLogger("default", defaultConf);
}
void BaseTest::SetUp() {
void
BaseTest::SetUp() {
InitLog();
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024*1024*200, 1024*1024*300, 2);
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024 * 1024 * 200, 1024 * 1024 * 300, 2);
}
void BaseTest::TearDown() {
void
BaseTest::TearDown() {
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->ClearCache();
zilliz::milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache();
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
engine::DBOptions BaseTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
BaseTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void DBTest::SetUp() {
void
DBTest::SetUp() {
BaseTest::SetUp();
auto res_mgr = scheduler::ResMgrInst::GetInstance();
auto res_mgr = ms::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false));
res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
auto default_conn = scheduler::Connection("IO", 500.0);
auto PCIE = scheduler::Connection("IO", 11000.0);
auto default_conn = ms::scheduler::Connection("IO", 500.0);
auto PCIE = ms::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Connect("cpu", "gtx1660", PCIE);
res_mgr->Start();
scheduler::SchedInst::GetInstance()->Start();
ms::scheduler::SchedInst::GetInstance()->Start();
scheduler::JobMgrInst::GetInstance()->Start();
ms::scheduler::JobMgrInst::GetInstance()->Start();
auto options = GetOptions();
db_ = engine::DBFactory::Build(options);
db_ = ms::engine::DBFactory::Build(options);
}
void DBTest::TearDown() {
void
DBTest::TearDown() {
db_->Stop();
db_->DropAll();
scheduler::JobMgrInst::GetInstance()->Stop();
scheduler::SchedInst::GetInstance()->Stop();
scheduler::ResMgrInst::GetInstance()->Stop();
ms::scheduler::JobMgrInst::GetInstance()->Stop();
ms::scheduler::SchedInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Stop();
BaseTest::TearDown();
......@@ -114,23 +131,26 @@ void DBTest::TearDown() {
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::DBOptions DBTest2::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
DBTest2::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.archive_conf_ = engine::ArchiveConf("delete", "disk:1");
options.meta_.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:1");
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MetaTest::SetUp() {
void
MetaTest::SetUp() {
BaseTest::SetUp();
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::SqliteMetaImpl>(options.meta_);
impl_ = std::make_shared<ms::engine::meta::SqliteMetaImpl>(options.meta_);
}
void MetaTest::TearDown() {
void
MetaTest::TearDown() {
impl_->DropAll();
BaseTest::TearDown();
......@@ -140,27 +160,26 @@ void MetaTest::TearDown() {
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::DBOptions MySqlDBTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
MySqlDBTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = DBTestEnvironment::getURI();
if(options.meta_.backend_uri_.empty()) {
options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/";
}
options.meta_.backend_uri_ = test_env->getURI();
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MySqlMetaTest::SetUp() {
void
MySqlMetaTest::SetUp() {
BaseTest::SetUp();
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options.meta_, options.mode_);
impl_ = std::make_shared<ms::engine::meta::MySQLMetaImpl>(options.meta_, options.mode_);
}
void MySqlMetaTest::TearDown() {
void
MySqlMetaTest::TearDown() {
impl_->DropAll();
BaseTest::TearDown();
......@@ -169,29 +188,26 @@ void MySqlMetaTest::TearDown() {
boost::filesystem::remove_all(options.meta_.path_);
}
engine::DBOptions MySqlMetaTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
MySqlMetaTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = DBTestEnvironment::getURI();
if(options.meta_.backend_uri_.empty()) {
options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/";
}
options.meta_.backend_uri_ = test_env->getURI();
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
int
main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
std::string uri;
if (argc > 1) {
uri = argv[1];
}
// if(uri.empty()) {
// uri = "mysql://root:Fantast1c@192.168.1.194:3306/";
// }
// std::cout << uri << std::endl;
::testing::AddGlobalTestEnvironment(new DBTestEnvironment);
test_env = new DBTestEnvironment(uri);
::testing::AddGlobalTestEnvironment(test_env);
return RUN_ALL_TESTS();
}
cpp/unittest/db/utils.h
浏览文件 @ c9320225
......@@ -20,7 +20,7 @@
#include <gtest/gtest.h>
#include <chrono>
//#include <src/db/MySQLMetaImpl.h>
#include <memory>
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
......@@ -28,7 +28,6 @@
#include "scheduler/SchedInst.h"
#include "scheduler/ResourceFactory.h"
#define TIMING
#ifdef TIMING
......@@ -36,8 +35,7 @@
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \
std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now()-start \
).count() << " ms ";
std::chrono::high_resolution_clock::now()-start).count() << " ms ";
#else
#define INIT_TIMER
#define START_TIMER
......@@ -45,11 +43,11 @@
#endif
class BaseTest : public ::testing::Test {
protected:
protected:
void InitLog();
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
virtual zilliz::milvus::engine::DBOptions GetOptions();
};
......@@ -58,14 +56,14 @@ class DBTest : public BaseTest {
protected:
zilliz::milvus::engine::DBPtr db_;
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DBTest2 : public DBTest {
protected:
virtual zilliz::milvus::engine::DBOptions GetOptions() override;
zilliz::milvus::engine::DBOptions GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -77,14 +75,14 @@ class MetaTest : public BaseTest {
protected:
std::shared_ptr<zilliz::milvus::engine::meta::SqliteMetaImpl> impl_;
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MySqlDBTest : public DBTest {
protected:
zilliz::milvus::engine::DBOptions GetOptions();
protected:
zilliz::milvus::engine::DBOptions GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -92,12 +90,11 @@ class MySqlMetaTest : public BaseTest {
protected:
std::shared_ptr<zilliz::milvus::engine::meta::MySQLMetaImpl> impl_;
virtual void SetUp() override;
virtual void TearDown() override;
zilliz::milvus::engine::DBOptions GetOptions();
void SetUp() override;
void TearDown() override;
zilliz::milvus::engine::DBOptions GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MemManagerTest : public MetaTest {
};
......
cpp/unittest/main.cpp
浏览文件 @ c9320225
......@@ -19,13 +19,11 @@
#include <gmock/gmock.h>
#include "utils/easylogging++.h"
#include "utils/CommonUtil.h"
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;
int main(int argc, char **argv) {
int
main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
cpp/unittest/metrics/metricbase_test.cpp
浏览文件 @ c9320225
......@@ -21,12 +21,16 @@
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::milvus;
namespace {
TEST(MetricbaseTest, METRICBASE_TEST){
server::MetricsBase instance = server::MetricsBase::GetInstance();
namespace ms = zilliz::milvus;
} // namespace
TEST(MetricbaseTest, METRICBASE_TEST) {
ms::server::MetricsBase instance = ms::server::MetricsBase::GetInstance();
instance.Init();
server::SystemInfo::GetInstance().Init();
ms::server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
......@@ -60,10 +64,10 @@ TEST(MetricbaseTest, METRICBASE_TEST){
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1,1,1);
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
instance.ConnectionGaugeIncrement();
instance.ConnectionGaugeDecrement();
instance.KeepingAliveCounterIncrement();
instance.OctetsSet();
}
\ No newline at end of file
}
cpp/unittest/metrics/metrics_test.cpp
浏览文件 @ c9320225
......@@ -15,75 +15,73 @@
// specific language governing permissions and limitations
// under the License.
#include <chrono>
#include <chrono>
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <gtest/gtest.h>
//#include "prometheus/registry.h"
//#include "prometheus/exposer.h"
#include <cache/CpuCacheMgr.h>
#include <src/server/Config.h>
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "metrics/Metrics.h"
#include "utils.h"
#include "metrics/utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
namespace {
using namespace zilliz::milvus;
namespace ms = zilliz::milvus;
} // namespace
TEST_F(MetricTest, METRIC_TEST) {
server::Config::GetInstance().SetMetricConfigCollector("zabbix");
server::Metrics::GetInstance();
server::Config::GetInstance().SetMetricConfigCollector("prometheus");
server::Metrics::GetInstance();
ms::server::Config::GetInstance().SetMetricConfigCollector("zabbix");
ms::server::Metrics::GetInstance();
ms::server::Config::GetInstance().SetMetricConfigCollector("prometheus");
ms::server::Metrics::GetInstance();
server::SystemInfo::GetInstance().Init();
ms::server::SystemInfo::GetInstance().Init();
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
server::Metrics::GetInstance().Init();
ms::server::Metrics::GetInstance().Init();
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL*1024*1024*1024);
std::cout<<zilliz::milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity()<<std::endl;
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
std::cout << zilliz::milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl;
static const char* group_name = "test_group";
static const char *group_name = "test_group";
static const int group_dim = 256;
engine::meta::TableSchema group_info;
ms::engine::meta::TableSchema group_info;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
auto stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
ms::engine::meta::TableSchema group_info_get;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int d = 256;
int nb = 50;
float *xb = new float[d * nb];
for(int i = 0; i < nb; i++) {
for(int j = 0; j < d; j++) xb[d * i + j] = drand48();
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) xb[d * i + j] = drand48();
xb[d * i] += i / 2000.;
}
int qb = 5;
float *qxb = new float[d * qb];
for(int i = 0; i < qb; i++) {
for(int j = 0; j < d; j++) qxb[d * i + j] = drand48();
for (int i = 0; i < qb; i++) {
for (int j = 0; j < d; j++) qxb[d * i + j] = drand48();
qxb[d * i] += i / 2000.;
}
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
......@@ -92,23 +90,23 @@ TEST_F(MetricTest, METRIC_TEST) {
uint64_t count = 0;
uint64_t prev_count = 0;
for (auto j=0; j<10; ++j) {
for (auto j = 0; j < 10; ++j) {
ss.str("");
db_->Size(count);
prev_count = count;
START_TIMER;
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/(1024*1024) << " M";
ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024)
<< " M";
for (auto k=0; k<qb; ++k) {
for (auto k = 0; k < qb; ++k) {
// ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
// for (auto result : results[k]) {
// ss << result.first << " ";
// }
}
ASSERT_TRUE(count >= prev_count);
std::this_thread::sleep_for(std::chrono::seconds(1));
......@@ -117,8 +115,8 @@ TEST_F(MetricTest, METRIC_TEST) {
int loop = 10000;
for (auto i=0; i<loop; ++i) {
if (i==40) {
for (auto i = 0; i < loop; ++i) {
if (i == 40) {
db_->InsertVectors(group_name, qb, qxb, target_ids);
ASSERT_EQ(target_ids.size(), qb);
} else {
......@@ -129,37 +127,37 @@ TEST_F(MetricTest, METRIC_TEST) {
search.join();
delete [] xb;
delete [] qxb;
};
delete[] xb;
delete[] qxb;
}
TEST_F(MetricTest, COLLECTOR_METRICS_TEST){
auto status = Status::OK();
server::CollectInsertMetrics insert_metrics0(0, status);
status = Status(DB_ERROR, "error");
server::CollectInsertMetrics insert_metrics1(0, status);
TEST_F(MetricTest, COLLECTOR_METRICS_TEST) {
auto status = ms::Status::OK();
ms::server::CollectInsertMetrics insert_metrics0(0, status);
status = ms::Status(ms::DB_ERROR, "error");
ms::server::CollectInsertMetrics insert_metrics1(0, status);
server::CollectQueryMetrics query_metrics(10);
ms::server::CollectQueryMetrics query_metrics(10);
server::CollectMergeFilesMetrics merge_metrics();
ms::server::CollectMergeFilesMetrics merge_metrics();
server::CollectBuildIndexMetrics build_index_metrics();
ms::server::CollectBuildIndexMetrics build_index_metrics();
server::CollectExecutionEngineMetrics execution_metrics(10);
ms::server::CollectExecutionEngineMetrics execution_metrics(10);
server::CollectSerializeMetrics serialize_metrics(10);
ms::server::CollectSerializeMetrics serialize_metrics(10);
server::CollectAddMetrics add_metrics(10, 128);
ms::server::CollectAddMetrics add_metrics(10, 128);
server::CollectDurationMetrics duration_metrics_raw(engine::meta::TableFileSchema::RAW);
server::CollectDurationMetrics duration_metrics_index(engine::meta::TableFileSchema::TO_INDEX);
server::CollectDurationMetrics duration_metrics_delete(engine::meta::TableFileSchema::TO_DELETE);
ms::server::CollectDurationMetrics duration_metrics_raw(ms::engine::meta::TableFileSchema::RAW);
ms::server::CollectDurationMetrics duration_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX);
ms::server::CollectDurationMetrics duration_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE);
server::CollectSearchTaskMetrics search_metrics_raw(engine::meta::TableFileSchema::RAW);
server::CollectSearchTaskMetrics search_metrics_index(engine::meta::TableFileSchema::TO_INDEX);
server::CollectSearchTaskMetrics search_metrics_delete(engine::meta::TableFileSchema::TO_DELETE);
ms::server::CollectSearchTaskMetrics search_metrics_raw(ms::engine::meta::TableFileSchema::RAW);
ms::server::CollectSearchTaskMetrics search_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX);
ms::server::CollectSearchTaskMetrics search_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE);
server::MetricCollector metric_collector();
ms::server::MetricCollector metric_collector();
}
cpp/unittest/metrics/prometheus_test.cpp
浏览文件 @ c9320225
......@@ -22,15 +22,19 @@
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::milvus;
namespace {
TEST(PrometheusTest, PROMETHEUS_TEST){
server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
namespace ms = zilliz::milvus;
server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance();
} // namespace
TEST(PrometheusTest, PROMETHEUS_TEST) {
ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
ms::server::PrometheusMetrics instance = ms::server::PrometheusMetrics::GetInstance();
instance.Init();
instance.SetStartup(true);
server::SystemInfo::GetInstance().Init();
ms::server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
......@@ -64,7 +68,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST){
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1,1,1);
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
instance.QueryIndexTypePerSecondSet("IDMap", 1.0);
instance.ConnectionGaugeIncrement();
......@@ -76,10 +80,9 @@ TEST(PrometheusTest, PROMETHEUS_TEST){
instance.GPUTemperature();
instance.CPUTemperature();
server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
instance.Init();
instance.CPUCoreUsagePercentSet();
instance.GPUTemperature();
instance.CPUTemperature();
}
\ No newline at end of file
}
cpp/unittest/metrics/utils.cpp
浏览文件 @ c9320225
......@@ -18,32 +18,38 @@
#include <iostream>
#include <thread>
#include <string>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "metrics/utils.h"
#include "db/DBFactory.h"
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;
namespace {
static std::string uri;
namespace ms = zilliz::milvus;
class DBTestEnvironment : public ::testing::Environment {
public:
public:
explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {}
// explicit DBTestEnvironment(std::string uri) : uri_(uri) {}
static std::string getURI() {
return uri;
std::string getURI() const {
return uri_;
}
void SetUp() override {
getURI();
}
private:
std::string uri_;
};
DBTestEnvironment* test_env = nullptr;
} // namespace
void MetricTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
......@@ -52,8 +58,8 @@ void MetricTest::InitLog() {
el::Loggers::reconfigureLogger("default", defaultConf);
}
engine::DBOptions MetricTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions MetricTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
......@@ -62,7 +68,7 @@ engine::DBOptions MetricTest::GetOptions() {
void MetricTest::SetUp() {
InitLog();
auto options = GetOptions();
db_ = engine::DBFactory::Build(options);
db_ = ms::engine::DBFactory::Build(options);
}
void MetricTest::TearDown() {
......@@ -72,10 +78,12 @@ void MetricTest::TearDown() {
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
std::string uri;
if (argc > 1) {
uri = argv[1];
}
// std::cout << uri << std::endl;
::testing::AddGlobalTestEnvironment(new DBTestEnvironment);
test_env = new DBTestEnvironment(uri);
::testing::AddGlobalTestEnvironment(test_env);
return RUN_ALL_TESTS();
}
cpp/unittest/metrics/utils.h
浏览文件 @ c9320225
......@@ -26,7 +26,6 @@
#include "db/meta/SqliteMetaImpl.h"
#include "db/meta/MySQLMetaImpl.h"
#define TIMING
#ifdef TIMING
......@@ -34,15 +33,15 @@
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \
std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now()-start \
).count() << " ms ";
std::chrono::high_resolution_clock::now()-start).count() << " ms ";
#else
#define INIT_TIMER
#define START_TIMER
#define STOP_TIMER(name)
#endif
void ASSERT_STATS(zilliz::milvus::Status& stat);
void
ASSERT_STATS(zilliz::milvus::Status &stat);
//class TestEnv : public ::testing::Environment {
//public:
......@@ -66,11 +65,11 @@ void ASSERT_STATS(zilliz::milvus::Status& stat);
// ::testing::AddGlobalTestEnvironment(new TestEnv);
class MetricTest : public ::testing::Test {
protected:
protected:
zilliz::milvus::engine::DBPtr db_;
void InitLog();
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
virtual zilliz::milvus::engine::DBOptions GetOptions();
};
\ No newline at end of file
};
cpp/unittest/scheduler/algorithm_test.cpp
浏览文件 @ c9320225
......@@ -101,11 +101,8 @@ TEST_F(AlgorithmTest, SHORTESTPATH_TEST) {
std::cout << sp[sp.size() - 1] << std::endl;
sp.pop_back();
}
}
}
}
}
\ No newline at end of file
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/event_test.cpp
浏览文件 @ c9320225
......@@ -60,7 +60,7 @@ TEST(EventTest, TASKTABLE_UPDATED_EVENT) {
std::cout << *EventPtr(event);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/node_test.cpp
浏览文件 @ c9320225
......@@ -19,37 +19,40 @@
#include "scheduler/resource/Node.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
class NodeTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
node1_ = std::make_shared<Node>();
node2_ = std::make_shared<Node>();
node3_ = std::make_shared<Node>();
isolated_node1_ = std::make_shared<Node>();
isolated_node2_ = std::make_shared<Node>();
node1_ = std::make_shared<ms::Node>();
node2_ = std::make_shared<ms::Node>();
node3_ = std::make_shared<ms::Node>();
isolated_node1_ = std::make_shared<ms::Node>();
isolated_node2_ = std::make_shared<ms::Node>();
auto pcie = Connection("PCIe", 11.0);
auto pcie = ms::Connection("PCIe", 11.0);
node1_->AddNeighbour(node2_, pcie);
node1_->AddNeighbour(node3_, pcie);
node2_->AddNeighbour(node1_, pcie);
}
NodePtr node1_;
NodePtr node2_;
NodePtr node3_;
NodePtr isolated_node1_;
NodePtr isolated_node2_;
ms::NodePtr node1_;
ms::NodePtr node2_;
ms::NodePtr node3_;
ms::NodePtr isolated_node1_;
ms::NodePtr isolated_node2_;
};
TEST_F(NodeTest, ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
auto pcie = ms::Connection("PCIe", 11.0);
isolated_node1_->AddNeighbour(isolated_node2_, pcie);
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 1);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
......@@ -58,7 +61,7 @@ TEST_F(NodeTest, ADD_NEIGHBOUR) {
TEST_F(NodeTest, REPEAT_ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
auto pcie = ms::Connection("PCIe", 11.0);
isolated_node1_->AddNeighbour(isolated_node2_, pcie);
isolated_node1_->AddNeighbour(isolated_node2_, pcie);
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 1);
......@@ -97,3 +100,4 @@ TEST_F(NodeTest, DUMP) {
std::cout << node2_->Dump();
ASSERT_FALSE(node2_->Dump().empty());
}
cpp/unittest/scheduler/normal_test.cpp
浏览文件 @ c9320225
......@@ -25,35 +25,37 @@
#include "utils/Log.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
TEST(NormalTest, INST_TEST) {
// ResourceMgr only compose resources, provide unified event
auto res_mgr = ResMgrInst::GetInstance();
auto res_mgr = ms::ResMgrInst::GetInstance();
res_mgr->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true));
auto IO = Connection("IO", 500.0);
auto IO = ms::Connection("IO", 500.0);
res_mgr->Connect("disk", "cpu", IO);
auto scheduler = SchedInst::GetInstance();
auto scheduler = ms::SchedInst::GetInstance();
res_mgr->Start();
scheduler->Start();
const uint64_t NUM_TASK = 1000;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
std::vector<std::shared_ptr<ms::TestTask>> tasks;
ms::TableFileSchemaPtr dummy = nullptr;
auto disks = res_mgr->GetDiskResources();
ASSERT_FALSE(disks.empty());
if (auto observe = disks[0].lock()) {
for (uint64_t i = 0; i < NUM_TASK; ++i) {
auto task = std::make_shared<TestTask>(dummy);
task->label() = std::make_shared<DefaultLabel>();
auto task = std::make_shared<ms::TestTask>(dummy);
task->label() = std::make_shared<ms::DefaultLabel>();
tasks.push_back(task);
observe->task_table().Put(task);
}
......@@ -67,5 +69,4 @@ TEST(NormalTest, INST_TEST) {
scheduler->Stop();
res_mgr->Stop();
}
cpp/unittest/scheduler/resource_factory_test.cpp
浏览文件 @ c9320225
......@@ -19,15 +19,18 @@
#include "scheduler/ResourceFactory.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
TEST(ResourceFactoryTest, CREATE) {
auto disk = ResourceFactory::Create("ssd", "DISK", 0);
auto cpu = ResourceFactory::Create("cpu", "CPU", 0);
auto gpu = ResourceFactory::Create("gpu", "GPU", 0);
auto disk = ms::ResourceFactory::Create("ssd", "DISK", 0);
auto cpu = ms::ResourceFactory::Create("cpu", "CPU", 0);
auto gpu = ms::ResourceFactory::Create("gpu", "GPU", 0);
ASSERT_TRUE(std::dynamic_pointer_cast<DiskResource>(disk));
ASSERT_TRUE(std::dynamic_pointer_cast<CpuResource>(cpu));
ASSERT_TRUE(std::dynamic_pointer_cast<GpuResource>(gpu));
ASSERT_TRUE(std::dynamic_pointer_cast<ms::DiskResource>(disk));
ASSERT_TRUE(std::dynamic_pointer_cast<ms::CpuResource>(cpu));
ASSERT_TRUE(std::dynamic_pointer_cast<ms::GpuResource>(gpu));
}
cpp/unittest/scheduler/resource_mgr_test.cpp
浏览文件 @ c9320225
......@@ -24,15 +24,13 @@
#include "scheduler/ResourceMgr.h"
#include <gtest/gtest.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
/************ ResourceMgrBaseTest ************/
class ResourceMgrBaseTest : public testing::Test {
protected:
protected:
void
SetUp() override {
empty_mgr_ = std::make_shared<ResourceMgr>();
......@@ -77,7 +75,6 @@ TEST_F(ResourceMgrBaseTest, CONNECT) {
ASSERT_TRUE(empty_mgr_->Connect("resource1", "resource2", io));
}
TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
......@@ -87,7 +84,6 @@ TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
ASSERT_FALSE(empty_mgr_->Connect("xx", "yy", io));
}
TEST_F(ResourceMgrBaseTest, CLEAR) {
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Clear();
......@@ -163,7 +159,7 @@ TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) {
/************ ResourceMgrAdvanceTest ************/
class ResourceMgrAdvanceTest : public testing::Test {
protected:
protected:
void
SetUp() override {
mgr1_ = std::make_shared<ResourceMgr>();
......@@ -193,7 +189,6 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
ASSERT_TRUE(flag);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/resource_test.cpp
浏览文件 @ c9320225
......@@ -26,20 +26,19 @@
#include "scheduler/ResourceFactory.h"
#include <gtest/gtest.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
/************ ResourceBaseTest ************/
class ResourceBaseTest : public testing::Test {
protected:
protected:
void
SetUp() override {
only_loader_ = std::make_shared<DiskResource>(name1, id1, true, false);
only_executor_ = std::make_shared<CpuResource>(name2, id2, false, true);
both_enable_ = std::make_shared<GpuResource>(name3, id3, true, true);
both_disable_ = std::make_shared<TestResource>(name4, id4, false, false);
only_loader_ = std::make_shared<DiskResource>(name1, id1, true, false);
only_executor_ = std::make_shared<CpuResource>(name2, id2, false, true);
both_enable_ = std::make_shared<GpuResource>(name3, id3, true, true);
both_disable_ = std::make_shared<TestResource>(name4, id4, false, false);
}
const std::string name1 = "only_loader_";
......@@ -104,7 +103,7 @@ TEST_F(ResourceBaseTest, DUMP) {
/************ ResourceAdvanceTest ************/
class ResourceAdvanceTest : public testing::Test {
protected:
protected:
void
SetUp() override {
disk_resource_ = ResourceFactory::Create("ssd", "DISK", 0);
......@@ -156,13 +155,17 @@ protected:
void
WaitLoader(uint64_t count) {
std::unique_lock<std::mutex> lock(load_mutex_);
cv_.wait(lock, [&] { return load_count_ == count; });
cv_.wait(lock, [&] {
return load_count_ == count;
});
}
void
WaitExecutor(uint64_t count) {
std::unique_lock<std::mutex> lock(exec_mutex_);
cv_.wait(lock, [&] { return exec_count_ == count; });
cv_.wait(lock, [&] {
return exec_count_ == count;
});
}
ResourcePtr disk_resource_;
......@@ -277,6 +280,6 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/schedinst_test.cpp
浏览文件 @ c9320225
......@@ -25,10 +25,8 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
class SchedInstTest : public testing::Test {
protected:
protected:
void
SetUp() override {
boost::filesystem::create_directory(TMP_DIR);
......@@ -83,6 +81,8 @@ TEST_F(SchedInstTest, SIMPLE_GPU) {
StartSchedulerService();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/scheduler_test.cpp
浏览文件 @ c9320225
......@@ -28,19 +28,18 @@
#include "utils/Error.h"
#include "wrapper/VecIndex.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class MockVecIndex : public engine::VecIndex {
public:
virtual Status BuildAll(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg,
const long &nt = 0,
const float *xt = nullptr) {
public:
virtual Status BuildAll(const int64_t &nb,
const float *xb,
const int64_t *ids,
const engine::Config &cfg,
const int64_t &nt = 0,
const float *xt = nullptr) {
}
engine::VecIndexPtr Clone() override {
......@@ -55,27 +54,23 @@ public:
return engine::IndexType::INVALID;
}
virtual Status Add(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg = engine::Config()) {
virtual Status Add(const int64_t &nb,
const float *xb,
const int64_t *ids,
const engine::Config &cfg = engine::Config()) {
}
virtual Status Search(const long &nq,
const float *xq,
float *dist,
long *ids,
const engine::Config &cfg = engine::Config()) {
virtual Status Search(const int64_t &nq,
const float *xq,
float *dist,
int64_t *ids,
const engine::Config &cfg = engine::Config()) {
}
engine::VecIndexPtr CopyToGpu(const int64_t &device_id, const engine::Config &cfg) override {
}
engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override {
}
virtual int64_t Dimension() {
......@@ -92,20 +87,18 @@ public:
}
virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) {
}
public:
public:
int64_t dimension_ = 512;
int64_t ntotal_ = 0;
};
class SchedulerTest : public testing::Test {
protected:
protected:
void
SetUp() override {
constexpr int64_t cache_cap = 1024*1024*1024;
constexpr int64_t cache_cap = 1024 * 1024 * 1024;
cache::GpuCacheMgr::GetInstance(0)->SetCapacity(cache_cap);
cache::GpuCacheMgr::GetInstance(1)->SetCapacity(cache_cap);
......@@ -170,7 +163,6 @@ TEST_F(SchedulerTest, ON_LOAD_COMPLETED) {
sleep(3);
ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
}
TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
......@@ -193,7 +185,7 @@ TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
}
class SchedulerTest2 : public testing::Test {
protected:
protected:
void
SetUp() override {
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false);
......@@ -243,7 +235,6 @@ protected:
std::shared_ptr<Scheduler> scheduler_;
};
TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
......@@ -260,6 +251,6 @@ TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
// ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/task_test.cpp
浏览文件 @ c9320225
......@@ -19,17 +19,16 @@
#include "scheduler/task/SearchTask.h"
#include <gtest/gtest.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
TEST(TaskTest, INVALID_INDEX) {
auto search_task = std::make_shared<XSearchTask>(nullptr);
search_task->Load(LoadType::TEST, 10);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
cpp/unittest/scheduler/tasktable_test.cpp
浏览文件 @ c9320225
......@@ -20,51 +20,53 @@
#include "scheduler/task/TestTask.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
/************ TaskTableBaseTest ************/
class TaskTableItemTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
std::vector<TaskTableItemState> states{
TaskTableItemState::INVALID,
TaskTableItemState::START,
TaskTableItemState::LOADING,
TaskTableItemState::LOADED,
TaskTableItemState::EXECUTING,
TaskTableItemState::EXECUTED,
TaskTableItemState::MOVING,
TaskTableItemState::MOVED};
std::vector<ms::TaskTableItemState> states{
ms::TaskTableItemState::INVALID,
ms::TaskTableItemState::START,
ms::TaskTableItemState::LOADING,
ms::TaskTableItemState::LOADED,
ms::TaskTableItemState::EXECUTING,
ms::TaskTableItemState::EXECUTED,
ms::TaskTableItemState::MOVING,
ms::TaskTableItemState::MOVED};
for (auto &state : states) {
auto item = std::make_shared<TaskTableItem>();
auto item = std::make_shared<ms::TaskTableItem>();
item->state = state;
items_.emplace_back(item);
}
}
TaskTableItem default_;
std::vector<TaskTableItemPtr> items_;
ms::TaskTableItem default_;
std::vector<ms::TaskTableItemPtr> items_;
};
TEST_F(TaskTableItemTest, CONSTRUCT) {
ASSERT_EQ(default_.id, 0);
ASSERT_EQ(default_.task, nullptr);
ASSERT_EQ(default_.state, TaskTableItemState::INVALID);
ASSERT_EQ(default_.state, ms::TaskTableItemState::INVALID);
}
TEST_F(TaskTableItemTest, DESTRUCT) {
auto p_item = new TaskTableItem();
auto p_item = new ms::TaskTableItem();
delete p_item;
}
TEST_F(TaskTableItemTest, IS_FINISH) {
for (auto &item : items_) {
if (item->state == TaskTableItemState::EXECUTED
|| item->state == TaskTableItemState::MOVED) {
if (item->state == ms::TaskTableItemState::EXECUTED
|| item->state == ms::TaskTableItemState::MOVED) {
ASSERT_TRUE(item->IsFinish());
} else {
ASSERT_FALSE(item->IsFinish());
......@@ -82,9 +84,9 @@ TEST_F(TaskTableItemTest, LOAD) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Load();
if (before_state == TaskTableItemState::START) {
if (before_state == ms::TaskTableItemState::START) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::LOADING);
ASSERT_EQ(item->state, ms::TaskTableItemState::LOADING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
......@@ -96,9 +98,9 @@ TEST_F(TaskTableItemTest, LOADED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Loaded();
if (before_state == TaskTableItemState::LOADING) {
if (before_state == ms::TaskTableItemState::LOADING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::LOADED);
ASSERT_EQ(item->state, ms::TaskTableItemState::LOADED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
......@@ -110,9 +112,9 @@ TEST_F(TaskTableItemTest, EXECUTE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Execute();
if (before_state == TaskTableItemState::LOADED) {
if (before_state == ms::TaskTableItemState::LOADED) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::EXECUTING);
ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
......@@ -120,14 +122,13 @@ TEST_F(TaskTableItemTest, EXECUTE) {
}
}
TEST_F(TaskTableItemTest, EXECUTED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Executed();
if (before_state == TaskTableItemState::EXECUTING) {
if (before_state == ms::TaskTableItemState::EXECUTING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::EXECUTED);
ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
......@@ -139,9 +140,9 @@ TEST_F(TaskTableItemTest, MOVE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Move();
if (before_state == TaskTableItemState::LOADED) {
if (before_state == ms::TaskTableItemState::LOADED) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::MOVING);
ASSERT_EQ(item->state, ms::TaskTableItemState::MOVING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
......@@ -153,9 +154,9 @@ TEST_F(TaskTableItemTest, MOVED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Moved();
if (before_state == TaskTableItemState::MOVING) {
if (before_state == ms::TaskTableItemState::MOVING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::MOVED);
ASSERT_EQ(item->state, ms::TaskTableItemState::MOVED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
......@@ -166,19 +167,19 @@ TEST_F(TaskTableItemTest, MOVED) {
/************ TaskTableBaseTest ************/
class TaskTableBaseTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
TableFileSchemaPtr dummy = nullptr;
ms::TableFileSchemaPtr dummy = nullptr;
invalid_task_ = nullptr;
task1_ = std::make_shared<TestTask>(dummy);
task2_ = std::make_shared<TestTask>(dummy);
task1_ = std::make_shared<ms::TestTask>(dummy);
task2_ = std::make_shared<ms::TestTask>(dummy);
}
TaskPtr invalid_task_;
TaskPtr task1_;
TaskPtr task2_;
TaskTable empty_table_;
ms::TaskPtr invalid_task_;
ms::TaskPtr task1_;
ms::TaskPtr task2_;
ms::TaskTable empty_table_;
};
TEST_F(TaskTableBaseTest, SUBSCRIBER) {
......@@ -191,7 +192,6 @@ TEST_F(TaskTableBaseTest, SUBSCRIBER) {
ASSERT_TRUE(flag);
}
TEST_F(TaskTableBaseTest, PUT_TASK) {
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
......@@ -203,14 +203,14 @@ TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) {
}
TEST_F(TaskTableBaseTest, PUT_BATCH) {
std::vector<TaskPtr> tasks{task1_, task2_};
std::vector<ms::TaskPtr> tasks{task1_, task2_};
empty_table_.Put(tasks);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
ASSERT_EQ(empty_table_.Get(1)->task, task2_);
}
TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) {
std::vector<TaskPtr> tasks{};
std::vector<ms::TaskPtr> tasks{};
empty_table_.Put(tasks);
}
......@@ -236,8 +236,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
auto indexes = empty_table_.PickToLoad(1);
ASSERT_EQ(indexes.size(), 1);
......@@ -249,8 +249,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
auto indexes = empty_table_.PickToLoad(3);
ASSERT_EQ(indexes.size(), 3);
......@@ -264,8 +264,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
// first pick, non-cache
auto indexes = empty_table_.PickToLoad(1);
......@@ -274,7 +274,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
// second pick, iterate from 2
// invalid state change
empty_table_[1]->state = TaskTableItemState::START;
empty_table_[1]->state = ms::TaskTableItemState::START;
indexes = empty_table_.PickToLoad(1);
ASSERT_EQ(indexes.size(), 1);
ASSERT_EQ(indexes[0], 2);
......@@ -285,9 +285,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[2]->state = TaskTableItemState::LOADED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
auto indexes = empty_table_.PickToExecute(1);
ASSERT_EQ(indexes.size(), 1);
......@@ -299,10 +299,10 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[2]->state = TaskTableItemState::LOADED;
empty_table_[3]->state = TaskTableItemState::LOADED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
empty_table_[3]->state = ms::TaskTableItemState::LOADED;
auto indexes = empty_table_.PickToExecute(3);
ASSERT_EQ(indexes.size(), 2);
......@@ -315,9 +315,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[2]->state = TaskTableItemState::LOADED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
// first pick, non-cache
auto indexes = empty_table_.PickToExecute(1);
......@@ -326,40 +326,39 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
// second pick, iterate from 2
// invalid state change
empty_table_[1]->state = TaskTableItemState::START;
empty_table_[1]->state = ms::TaskTableItemState::START;
indexes = empty_table_.PickToExecute(1);
ASSERT_EQ(indexes.size(), 1);
ASSERT_EQ(indexes[0], 2);
}
/************ TaskTableAdvanceTest ************/
class TaskTableAdvanceTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
TableFileSchemaPtr dummy = nullptr;
ms::TableFileSchemaPtr dummy = nullptr;
for (uint64_t i = 0; i < 8; ++i) {
auto task = std::make_shared<TestTask>(dummy);
auto task = std::make_shared<ms::TestTask>(dummy);
table1_.Put(task);
}
table1_.Get(0)->state = TaskTableItemState::INVALID;
table1_.Get(1)->state = TaskTableItemState::START;
table1_.Get(2)->state = TaskTableItemState::LOADING;
table1_.Get(3)->state = TaskTableItemState::LOADED;
table1_.Get(4)->state = TaskTableItemState::EXECUTING;
table1_.Get(5)->state = TaskTableItemState::EXECUTED;
table1_.Get(6)->state = TaskTableItemState::MOVING;
table1_.Get(7)->state = TaskTableItemState::MOVED;
table1_.Get(0)->state = ms::TaskTableItemState::INVALID;
table1_.Get(1)->state = ms::TaskTableItemState::START;
table1_.Get(2)->state = ms::TaskTableItemState::LOADING;
table1_.Get(3)->state = ms::TaskTableItemState::LOADED;
table1_.Get(4)->state = ms::TaskTableItemState::EXECUTING;
table1_.Get(5)->state = ms::TaskTableItemState::EXECUTED;
table1_.Get(6)->state = ms::TaskTableItemState::MOVING;
table1_.Get(7)->state = ms::TaskTableItemState::MOVED;
}
TaskTable table1_;
ms::TaskTable table1_;
};
TEST_F(TaskTableAdvanceTest, LOAD) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
......@@ -369,8 +368,8 @@ TEST_F(TaskTableAdvanceTest, LOAD) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::START) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::LOADING);
if (before_state[i] == ms::TaskTableItemState::START) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
......@@ -378,7 +377,7 @@ TEST_F(TaskTableAdvanceTest, LOAD) {
}
TEST_F(TaskTableAdvanceTest, LOADED) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
......@@ -388,8 +387,8 @@ TEST_F(TaskTableAdvanceTest, LOADED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::LOADING) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::LOADED);
if (before_state[i] == ms::TaskTableItemState::LOADING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
......@@ -397,7 +396,7 @@ TEST_F(TaskTableAdvanceTest, LOADED) {
}
TEST_F(TaskTableAdvanceTest, EXECUTE) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
......@@ -407,8 +406,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::EXECUTING);
if (before_state[i] == ms::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
......@@ -416,7 +415,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) {
}
TEST_F(TaskTableAdvanceTest, EXECUTED) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
......@@ -426,8 +425,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::EXECUTING) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::EXECUTED);
if (before_state[i] == ms::TaskTableItemState::EXECUTING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
......@@ -435,7 +434,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) {
}
TEST_F(TaskTableAdvanceTest, MOVE) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
......@@ -445,8 +444,8 @@ TEST_F(TaskTableAdvanceTest, MOVE) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::MOVING);
if (before_state[i] == ms::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
......@@ -454,7 +453,7 @@ TEST_F(TaskTableAdvanceTest, MOVE) {
}
TEST_F(TaskTableAdvanceTest, MOVED) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
......@@ -464,8 +463,8 @@ TEST_F(TaskTableAdvanceTest, MOVED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::MOVING) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::MOVED);
if (before_state[i] == ms::TaskTableItemState::MOVING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
......
cpp/unittest/server/cache_test.cpp
浏览文件 @ c9320225
......@@ -21,41 +21,40 @@
#include "utils/Error.h"
#include "wrapper/VecIndex.h"
using namespace zilliz::milvus;
namespace {
class InvalidCacheMgr : public cache::CacheMgr<cache::DataObjPtr> {
public:
namespace ms = zilliz::milvus;
class InvalidCacheMgr : public ms::cache::CacheMgr<ms::cache::DataObjPtr> {
public:
InvalidCacheMgr() {
}
};
class LessItemCacheMgr : public cache::CacheMgr<cache::DataObjPtr> {
public:
class LessItemCacheMgr : public ms::cache::CacheMgr<ms::cache::DataObjPtr> {
public:
LessItemCacheMgr() {
cache_ = std::make_shared<cache::Cache<cache::DataObjPtr>>(1UL << 12, 10);
cache_ = std::make_shared<ms::cache::Cache<ms::cache::DataObjPtr>>(1UL << 12, 10);
}
};
class MockVecIndex : public engine::VecIndex {
public:
class MockVecIndex : public ms::engine::VecIndex {
public:
MockVecIndex(int64_t dim, int64_t total)
: dimension_(dim),
ntotal_(total){
ntotal_(total) {
}
virtual Status BuildAll(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg,
const long &nt = 0,
const float *xt = nullptr) {
return Status();
virtual ms::Status BuildAll(const int64_t &nb,
const float *xb,
const int64_t *ids,
const ms::engine::Config &cfg,
const int64_t &nt = 0,
const float *xt = nullptr) {
return ms::Status();
}
engine::VecIndexPtr Clone() override {
ms::engine::VecIndexPtr Clone() override {
return zilliz::milvus::engine::VecIndexPtr();
}
......@@ -63,31 +62,31 @@ public:
return 0;
}
engine::IndexType GetType() override {
return engine::IndexType::INVALID;
ms::engine::IndexType GetType() override {
return ms::engine::IndexType::INVALID;
}
virtual Status Add(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg = engine::Config()) {
return Status();
virtual ms::Status Add(const int64_t &nb,
const float *xb,
const int64_t *ids,
const ms::engine::Config &cfg = ms::engine::Config()) {
return ms::Status();
}
virtual Status Search(const long &nq,
const float *xq,
float *dist,
long *ids,
const engine::Config &cfg = engine::Config()) {
return Status();
virtual ms::Status Search(const int64_t &nq,
const float *xq,
float *dist,
int64_t *ids,
const ms::engine::Config &cfg = ms::engine::Config()) {
return ms::Status();
}
engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const engine::Config &cfg) override {
ms::engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const ms::engine::Config &cfg) override {
return nullptr;
}
engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override {
ms::engine::VecIndexPtr CopyToCpu(const ms::engine::Config &cfg) override {
return nullptr;
}
......@@ -104,19 +103,19 @@ public:
return binset;
}
virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) {
return Status();
virtual ms::Status Load(const zilliz::knowhere::BinarySet &index_binary) {
return ms::Status();
}
public:
public:
int64_t dimension_ = 256;
int64_t ntotal_ = 0;
};
}
} // namespace
TEST(CacheTest, DUMMY_TEST) {
engine::Config cfg;
ms::engine::Config cfg;
MockVecIndex mock_index(256, 1000);
mock_index.Dimension();
mock_index.Count();
......@@ -134,9 +133,9 @@ TEST(CacheTest, DUMMY_TEST) {
}
TEST(CacheTest, CPU_CACHE_TEST) {
auto cpu_mgr = cache::CpuCacheMgr::GetInstance();
auto cpu_mgr = ms::cache::CpuCacheMgr::GetInstance();
const int64_t gbyte = 1024*1024*1024;
const int64_t gbyte = 1024 * 1024 * 1024;
int64_t g_num = 16;
int64_t cap = g_num * gbyte;
cpu_mgr->SetCapacity(cap);
......@@ -145,8 +144,8 @@ TEST(CacheTest, CPU_CACHE_TEST) {
uint64_t item_count = 20;
for (uint64_t i = 0; i < item_count; i++) {
//each vector is 1k byte, total size less than 1G
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
ASSERT_LT(cpu_mgr->ItemCount(), g_num);
......@@ -169,8 +168,8 @@ TEST(CacheTest, CPU_CACHE_TEST) {
cpu_mgr->SetCapacity(g_num * gbyte);
//each vector is 1k byte, total size less than 6G
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_6g", data_obj);
ASSERT_TRUE(cpu_mgr->ItemExists("index_6g"));
}
......@@ -179,12 +178,12 @@ TEST(CacheTest, CPU_CACHE_TEST) {
}
TEST(CacheTest, GPU_CACHE_TEST) {
auto gpu_mgr = cache::GpuCacheMgr::GetInstance(0);
auto gpu_mgr = ms::cache::GpuCacheMgr::GetInstance(0);
for(int i = 0; i < 20; i++) {
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
......@@ -196,15 +195,14 @@ TEST(CacheTest, GPU_CACHE_TEST) {
for (auto i = 0; i < 3; i++) {
// TODO: use gpu index to mock
//each vector is 1k byte, total size less than 2G
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
std::cout << data_obj->size() <<std::endl;
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
std::cout << data_obj->size() << std::endl;
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
gpu_mgr->ClearCache();
ASSERT_EQ(gpu_mgr->ItemCount(), 0);
}
TEST(CacheTest, INVALID_TEST) {
......@@ -214,7 +212,7 @@ TEST(CacheTest, INVALID_TEST) {
ASSERT_FALSE(mgr.ItemExists("test"));
ASSERT_EQ(mgr.GetItem("test"), nullptr);
mgr.InsertItem("test", cache::DataObjPtr());
mgr.InsertItem("test", ms::cache::DataObjPtr());
mgr.InsertItem("test", nullptr);
mgr.EraseItem("test");
mgr.PrintInfo();
......@@ -226,12 +224,12 @@ TEST(CacheTest, INVALID_TEST) {
{
LessItemCacheMgr mgr;
for(int i = 0; i < 20; i++) {
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
mgr.InsertItem("index_" + std::to_string(i), data_obj);
}
ASSERT_EQ(mgr.GetItem("index_0"), nullptr);
}
}
\ No newline at end of file
}
cpp/unittest/server/config_test.cpp
浏览文件 @ c9320225
......@@ -23,39 +23,39 @@
#include "utils/ValidationUtil.h"
#include "server/Config.h"
using namespace zilliz::milvus;
namespace {
static const char* CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
namespace ms = zilliz::milvus;
static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
static constexpr uint64_t KB = 1024;
static constexpr uint64_t MB = KB*1024;
static constexpr uint64_t GB = MB*1024;
static constexpr uint64_t MB = KB * 1024;
static constexpr uint64_t GB = MB * 1024;
}
} // namespace
TEST(ConfigTest, CONFIG_TEST) {
server::ConfigMgr* config_mgr = server::ConfigMgr::GetInstance();
ms::server::ConfigMgr *config_mgr = ms::server::ConfigMgr::GetInstance();
ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR);
ms::ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(LOG_FILE_PATH);
ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR);
ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_EQ(err, ms::SERVER_SUCCESS);
config_mgr->Print();
server::ConfigNode& root_config = config_mgr->GetRootNode();
server::ConfigNode& server_config = root_config.GetChild("server_config");
server::ConfigNode& db_config = root_config.GetChild("db_config");
server::ConfigNode& metric_config = root_config.GetChild("metric_config");
server::ConfigNode& cache_config = root_config.GetChild("cache_config");
server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
ms::server::ConfigNode &root_config = config_mgr->GetRootNode();
ms::server::ConfigNode &server_config = root_config.GetChild("server_config");
ms::server::ConfigNode &db_config = root_config.GetChild("db_config");
ms::server::ConfigNode &metric_config = root_config.GetChild("metric_config");
ms::server::ConfigNode &cache_config = root_config.GetChild("cache_config");
ms::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
auto valus = invalid_config.GetSequence("not_exist");
float ff = invalid_config.GetFloatValue("not_exist", 3.0);
ASSERT_EQ(ff, 3.0);
......@@ -63,16 +63,16 @@ TEST(ConfigTest, CONFIG_TEST) {
std::string address = server_config.GetValue("address");
ASSERT_TRUE(!address.empty());
int64_t port = server_config.GetInt64Value("port");
ASSERT_TRUE(port != 0);
ASSERT_NE(port, 0);
server_config.SetValue("test", "2.5");
double test = server_config.GetDoubleValue("test");
ASSERT_EQ(test, 2.5);
server::ConfigNode fake;
ms::server::ConfigNode fake;
server_config.AddChild("fake", fake);
fake = server_config.GetChild("fake");
server::ConfigNodeArr arr;
ms::server::ConfigNodeArr arr;
server_config.GetChildren(arr);
ASSERT_EQ(arr.size(), 1UL);
......@@ -89,7 +89,7 @@ TEST(ConfigTest, CONFIG_TEST) {
auto seq = server_config.GetSequence("seq");
ASSERT_EQ(seq.size(), 2UL);
server::ConfigNode combine;
ms::server::ConfigNode combine;
combine.Combine(server_config);
combine.PrintAll();
......@@ -102,8 +102,8 @@ TEST(ConfigTest, CONFIG_TEST) {
}
TEST(ConfigTest, SERVER_CONFIG_TEST) {
server::Config& config = server::Config::GetInstance();
Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
ms::server::Config &config = ms::server::Config::GetInstance();
ms::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_TRUE(s.ok());
s = config.ValidateConfig();
......@@ -113,4 +113,4 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok());
}
\ No newline at end of file
}
cpp/unittest/server/rpc_test.cpp
浏览文件 @ c9320225
......@@ -23,7 +23,7 @@
#include "server/grpc_impl/GrpcRequestHandler.h"
#include "server/grpc_impl/GrpcRequestScheduler.h"
#include "server/grpc_impl/GrpcRequestTask.h"
#include "version.h"
#include "../version.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
......@@ -34,10 +34,10 @@
#include "scheduler/ResourceFactory.h"
#include "utils/CommonUtil.h"
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
static const char *TABLE_NAME = "test_grpc";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t INDEX_FILE_SIZE = 1024;
......@@ -49,30 +49,29 @@ class RpcHandlerTest : public testing::Test {
protected:
void
SetUp() override {
auto res_mgr = scheduler::ResMgrInst::GetInstance();
auto res_mgr = ms::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
auto default_conn = scheduler::Connection("IO", 500.0);
auto PCIE = scheduler::Connection("IO", 11000.0);
auto default_conn = ms::scheduler::Connection("IO", 500.0);
auto PCIE = ms::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Connect("cpu", "gtx1660", PCIE);
res_mgr->Start();
scheduler::SchedInst::GetInstance()->Start();
scheduler::JobMgrInst::GetInstance()->Start();
ms::scheduler::SchedInst::GetInstance()->Start();
ms::scheduler::JobMgrInst::GetInstance()->Start();
engine::DBOptions opt;
ms::engine::DBOptions opt;
server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test");
server::Config::GetInstance().SetDBConfigSecondaryPath("");
server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
server::Config::GetInstance().SetCacheConfigCacheInsertData("");
server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
ms::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
ms::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test");
ms::server::Config::GetInstance().SetDBConfigSecondaryPath("");
ms::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
ms::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
ms::server::Config::GetInstance().SetCacheConfigCacheInsertData("");
ms::server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
// DBWrapper::GetInstance().GetInstance().StartService();
......@@ -82,11 +81,11 @@ class RpcHandlerTest : public testing::Test {
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
server::Config::GetInstance().SetResourceConfigMode("single");
server::DBWrapper::GetInstance().StartService();
ms::server::Config::GetInstance().SetResourceConfigMode("single");
ms::server::DBWrapper::GetInstance().StartService();
//initialize handler, create table
handler = std::make_shared<server::grpc::GrpcRequestHandler>();
handler = std::make_shared<ms::server::grpc::GrpcRequestHandler>();
::grpc::ServerContext context;
::milvus::grpc::TableSchema request;
::milvus::grpc::Status status;
......@@ -99,18 +98,20 @@ class RpcHandlerTest : public testing::Test {
void
TearDown() override {
server::DBWrapper::GetInstance().StopService();
scheduler::JobMgrInst::GetInstance()->Stop();
scheduler::ResMgrInst::GetInstance()->Stop();
scheduler::SchedInst::GetInstance()->Stop();
ms::server::DBWrapper::GetInstance().StopService();
ms::scheduler::JobMgrInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Stop();
ms::scheduler::SchedInst::GetInstance()->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
}
protected:
std::shared_ptr<server::grpc::GrpcRequestHandler> handler;
std::shared_ptr<ms::server::grpc::GrpcRequestHandler> handler;
};
void BuildVectors(int64_t from, int64_t to,
std::vector<std::vector<float >> &vector_record_array) {
void
BuildVectors(int64_t from, int64_t to,
std::vector<std::vector<float >> &vector_record_array) {
if (to <= from) {
return;
}
......@@ -127,20 +128,22 @@ void BuildVectors(int64_t from, int64_t to,
}
}
std::string CurrentTmDate(int64_t offset_day = 0) {
std::string
CurrentTmDate(int64_t offset_day = 0) {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tt = tt + 24 * SECONDS_EACH_HOUR * offset_day;
tm *t = gmtime(&tt);
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1)
+ "-" + std::to_string(t->tm_mday);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
return str;
}
}
} // namespace
TEST_F(RpcHandlerTest, HAS_TABLE_TEST) {
::grpc::ServerContext context;
......@@ -311,7 +314,6 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
::grpc::Status status = handler->DescribeTable(&context, &table_name, &table_schema);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
::milvus::grpc::InsertParam request;
std::vector<std::vector<float>> record_array;
BuildVectors(0, VECTOR_COUNT, record_array);
......@@ -419,31 +421,29 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) {
grpc_status = handler->DeleteByRange(&context, &request, &status);
request.mutable_range()->set_end_value(CurrentTmDate(-2));
grpc_status = handler->DeleteByRange(&context, &request, &status);
}
//////////////////////////////////////////////////////////////////////
namespace {
class DummyTask : public server::grpc::GrpcBaseTask {
public:
Status
class DummyTask : public ms::server::grpc::GrpcBaseTask {
public:
ms::Status
OnExecute() override {
return Status::OK();
return ms::Status::OK();
}
static server::grpc::BaseTaskPtr
static ms::server::grpc::BaseTaskPtr
Create(std::string &dummy) {
return std::shared_ptr<server::grpc::GrpcBaseTask>(new DummyTask(dummy));
return std::shared_ptr<ms::server::grpc::GrpcBaseTask>(new DummyTask(dummy));
}
public:
public:
explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) {
}
};
class RpcSchedulerTest : public testing::Test {
protected:
protected:
void
SetUp() override {
std::string dummy = "dql";
......@@ -453,22 +453,22 @@ protected:
std::shared_ptr<DummyTask> task_ptr;
};
}
} // namespace
TEST_F(RpcSchedulerTest, BASE_TASK_TEST){
TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
auto status = task_ptr->Execute();
ASSERT_TRUE(status.ok());
server::grpc::GrpcRequestScheduler::GetInstance().Start();
ms::server::grpc::GrpcRequestScheduler::GetInstance().Start();
::milvus::grpc::Status grpc_status;
std::string dummy = "dql";
server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy);
server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status);
ms::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status);
server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
task_ptr = nullptr;
server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
server::grpc::GrpcRequestScheduler::GetInstance().Stop();
ms::server::grpc::GrpcRequestScheduler::GetInstance().Stop();
}
cpp/unittest/server/util_test.cpp
浏览文件 @ c9320225
此差异已折叠。
cpp/unittest/wrapper/utils.cpp
浏览文件 @ c9320225
......@@ -20,10 +20,10 @@
#include "utils.h"
void DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
float *xb, float *xq, long *ids,
const int &k, long *gt_ids, float *gt_dis) {
void
DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
float *xb, float *xq, long *ids,
const int &k, long *gt_ids, float *gt_dis) {
for (auto i = 0; i < nb; ++i) {
for (auto j = 0; j < dim; ++j) {
//p_data[i * d + j] = float(base + i);
......@@ -42,15 +42,16 @@ void DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
index.search(nq, xq, k, gt_dis, gt_ids);
}
void DataGenBase::GenData(const int &dim,
const int &nb,
const int &nq,
std::vector<float> &xb,
std::vector<float> &xq,
std::vector<long> &ids,
const int &k,
std::vector<long> &gt_ids,
std::vector<float> &gt_dis) {
void
DataGenBase::GenData(const int &dim,
const int &nb,
const int &nq,
std::vector<float> &xb,
std::vector<float> &xq,
std::vector<long> &ids,
const int &k,
std::vector<long> &gt_ids,
std::vector<float> &gt_dis) {
xb.resize(nb * dim);
xq.resize(nq * dim);
ids.resize(nb);
......
cpp/unittest/wrapper/utils.h
浏览文件 @ c9320225
......@@ -24,12 +24,10 @@
#include <cstdio>
#include <fstream>
class DataGenBase;
using DataGenPtr = std::shared_ptr<DataGenBase>;
class DataGenBase {
public:
virtual void GenData(const int &dim, const int &nb, const int &nq, float *xb, float *xq, long *ids,
......
cpp/unittest/wrapper/wrapper_test.cpp
浏览文件 @ c9320225
......@@ -18,7 +18,7 @@
#include "utils/easylogging++.h"
#include "src/wrapper/VecIndex.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "utils.h"
#include "wrapper/utils.h"
#include <gtest/gtest.h>
......
cpp/version.h.macro
浏览文件 @ c9320225
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#define MILVUS_VERSION "@MILVUS_VERSION@"
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或