提交 789c6f9e 编写于 作者: Y Yu Kun

Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0


Former-commit-id: c5e7ad8a644fb1bec6ee528e5c5723e3f14f0b04
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
mode: single # milvus deployment type: single, cluster, read_only
time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
......@@ -18,6 +17,7 @@ db_config:
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
metric_config:
is_startup: off # if monitoring start: on, off
......@@ -33,8 +33,6 @@ cache_config:
insert_cache_immediately: false # insert data will be load into cache immediately for hot query
gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
gpu_ids: # gpu id
- 0
engine_config:
use_blas_threshold: 20
......
......@@ -17,36 +17,14 @@ std::mutex GpuCacheMgr::mutex_;
std::unordered_map<uint64_t, GpuCacheMgrPtr> GpuCacheMgr::instance_;
namespace {
constexpr int64_t unit = 1024 * 1024 * 1024;
std::vector<uint64_t> load() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
auto conf_gpu_ids = config.GetSequence(server::CONFIG_GPU_IDS);
std::vector<uint64_t > gpu_ids;
for (auto gpu_id : conf_gpu_ids) {
gpu_ids.push_back(std::atoi(gpu_id.c_str()));
}
return gpu_ids;
}
}
bool GpuCacheMgr::GpuIdInConfig(uint64_t gpu_id) {
static std::vector<uint64_t > ids = load();
for (auto id : ids) {
if (gpu_id == id) return true;
}
return false;
constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
}
GpuCacheMgr::GpuCacheMgr() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 2);
cap *= unit;
cap *= G_BYTE;
cache_ = std::make_shared<Cache>(cap, 1UL<<32);
double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85);
......@@ -58,6 +36,19 @@ GpuCacheMgr::GpuCacheMgr() {
}
}
CacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
if (instance_.find(gpu_id) == instance_.end()) {
instance_.insert(std::pair<uint64_t, GpuCacheMgrPtr>(gpu_id, std::make_shared<GpuCacheMgr>()));
}
return instance_[gpu_id].get();
} else {
std::lock_guard<std::mutex> lock(mutex_);
return instance_[gpu_id].get();
}
}
void GpuCacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) {
//TODO: copy data to gpu
if (cache_ == nullptr) {
......
......@@ -19,21 +19,7 @@ class GpuCacheMgr : public CacheMgr {
public:
GpuCacheMgr();
static bool GpuIdInConfig(uint64_t gpu_id);
static CacheMgr* GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
if (instance_.find(gpu_id) == instance_.end()) {
if (GpuIdInConfig(gpu_id)) {
instance_.insert(std::pair<uint64_t, GpuCacheMgrPtr>(gpu_id, std::make_shared<GpuCacheMgr>()));
} else {
return nullptr;
}
}
}
return instance_[gpu_id].get();
}
static CacheMgr* GetInstance(uint64_t gpu_id);
void InsertItem(const std::string& key, const DataObjPtr& data) override;
......
......@@ -168,22 +168,13 @@ void PrometheusMetrics::CPUTemperature() {
}
void PrometheusMetrics::GpuCacheUsageGaugeSet() {
if(!startup_) return;
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
auto conf_gpu_ids = config.GetSequence(server::CONFIG_GPU_IDS);
std::vector<uint64_t > gpu_ids;
for (auto gpu_id : conf_gpu_ids) {
gpu_ids.push_back(std::atoi(gpu_id.c_str()));
}
for(auto i = 0; i < gpu_ids.size(); ++i) {
uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity();
prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}});
gpu_cache.Set(cache_usage * 100 / cache_capacity);
}
// std::vector<uint64_t > gpu_ids = {0};
// for(auto i = 0; i < gpu_ids.size(); ++i) {
// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
// uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity();
// prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}});
// gpu_cache.Set(cache_usage * 100 / cache_capacity);
// }
}
}
......
......@@ -113,18 +113,6 @@ ServerConfig::CheckServerConfig() {
}
}
std::string gpu_index_str = server_config.GetValue(CONFIG_GPU_INDEX, "0");
if (ValidationUtil::ValidateStringIsNumber(gpu_index_str) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl;
okay = false;
} else {
int32_t gpu_index = std::stol(gpu_index_str);
if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl;
okay = false;
}
}
std::string mode = server_config.GetValue(CONFIG_CLUSTER_MODE, "single");
if (mode != "single" && mode != "cluster" && mode != "read_only") {
std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl;
......@@ -214,6 +202,18 @@ ServerConfig::CheckDBConfig() {
}
}
std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, "0");
if (ValidationUtil::ValidateStringIsNumber(gpu_index_str) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl;
okay = false;
} else {
int32_t gpu_index = std::stol(gpu_index_str);
if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl;
okay = false;
}
}
return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT);
}
......@@ -313,7 +313,7 @@ ServerConfig::CheckCacheConfig() {
else {
uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str);
gpu_cache_capacity *= GB;
int gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);
size_t gpu_memory;
if (ValidationUtil::GetGpuMemory(gpu_index, gpu_memory) != SERVER_SUCCESS) {
std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl;
......@@ -340,19 +340,6 @@ ServerConfig::CheckCacheConfig() {
okay = false;
}
auto conf_gpu_ids = cache_config.GetSequence(server::CONFIG_GPU_IDS);
for (std::string &gpu_id : conf_gpu_ids) {
if (ValidationUtil::ValidateStringIsNumber(gpu_id) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_id " << gpu_id << " is not a number" << std::endl;
okay = false;
}
else if (ValidationUtil::ValidateGpuIndex(std::stol(gpu_id)) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_id " << gpu_id << " is invalid" << std::endl;
okay = false;
}
}
return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT);
}
......@@ -483,7 +470,7 @@ ServerConfig::CheckResourceConfig() {
}
}
else if (type == "GPU") {
int build_index_gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
int build_index_gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);
if (device_id == build_index_gpu_index) {
resource_valid_flag = true;
}
......
......@@ -18,7 +18,6 @@ static const char* CONFIG_SERVER = "server_config";
static const char* CONFIG_SERVER_ADDRESS = "address";
static const char* CONFIG_SERVER_PORT = "port";
static const char* CONFIG_CLUSTER_MODE = "mode";
static const char* CONFIG_GPU_INDEX = "gpu_index";
static const char* CONFIG_TIME_ZONE = "time_zone";
static const char* CONFIG_DB = "db_config";
......@@ -29,6 +28,7 @@ static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char* CONFIG_DB_PARALLEL_REDUCE = "parallel_reduce";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_LOG = "log_config";
......@@ -37,7 +37,6 @@ static const char* CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity";
static const char* CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity";
static const char* CACHE_FREE_PERCENT = "cpu_cache_free_percent";
static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately";
static const char* CONFIG_GPU_IDS = "gpu_ids";
static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent";
static const char* CONFIG_METRIC = "metric_config";
......
......@@ -25,8 +25,8 @@ namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 10000;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;
......
......@@ -105,9 +105,6 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
ASSERT_EQ(engine_ptr->Dimension(), dimension);
ASSERT_EQ(engine_ptr->Count(), ids.size());
// server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
// config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
//
// status = engine_ptr->CopyToGpu(0);
// //ASSERT_TRUE(status.ok());
//
......
......@@ -24,8 +24,6 @@ namespace {
static std::string TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 10000;
std::string GenTableName() {
auto now = std::chrono::system_clock::now();
......@@ -212,7 +210,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::map<int64_t, std::vector<float>> search_vectors;
{
engine::IDNumbers vector_ids;
int64_t nb = 1024000;
int64_t nb = 100000;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
......@@ -224,7 +222,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::mt19937 gen(rd());
std::uniform_int_distribution<int64_t> dis(0, nb - 1);
int64_t num_query = 20;
int64_t num_query = 10;
for (int64_t i = 0; i < num_query; ++i) {
int64_t index = dis(gen);
std::vector<float> search;
......
......@@ -128,13 +128,13 @@ TEST(DBMiscTest, UTILS_TEST) {
ASSERT_TRUE(boost::filesystem::exists(path));
}
options.slave_paths.push_back("/");
status = engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_FALSE(status.ok());
options.path = "/";
status = engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_FALSE(status.ok());
// options.slave_paths.push_back("/");
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
//
// options.path = "/";
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
engine::meta::TableFileSchema file;
file.id_ = 50;
......
......@@ -22,8 +22,8 @@ namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 10000;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
......
......@@ -288,7 +288,7 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
DoTopk(5, 10, 4, false);
DoTopk(20005, 998, 123, true);
DoTopk(9987, 12, 10, false);
DoTopk(77777, 1000, 1, false);
DoTopk(5432, 8899, 8899, true);
// DoTopk(9987, 12, 10, false);
// DoTopk(77777, 1000, 1, false);
// DoTopk(5432, 8899, 8899, true);
}
\ No newline at end of file
......@@ -66,9 +66,6 @@ engine::Options BaseTest::GetOptions() {
void DBTest::SetUp() {
BaseTest::SetUp();
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
auto res_mgr = engine::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false));
......
......@@ -142,7 +142,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
)
);
TEST_P(KnowhereWrapperTest, base_test) {
TEST_P(KnowhereWrapperTest, BASE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
......@@ -154,7 +154,7 @@ TEST_P(KnowhereWrapperTest, base_test) {
AssertResult(res_ids, res_dis);
}
TEST_P(KnowhereWrapperTest, to_gpu_test) {
TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
......@@ -174,7 +174,7 @@ TEST_P(KnowhereWrapperTest, to_gpu_test) {
}
{
std::string file_location = "/tmp/test_gpu_file";
std::string file_location = "/tmp/knowhere_gpu_file";
write_index(index_, file_location);
auto new_index = read_index(file_location);
......@@ -186,11 +186,11 @@ TEST_P(KnowhereWrapperTest, to_gpu_test) {
}
}
TEST_P(KnowhereWrapperTest, to_cpu_test) {
TEST_P(KnowhereWrapperTest, TO_CPU_TEST) {
// dev
}
TEST_P(KnowhereWrapperTest, serialize) {
TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
......@@ -215,7 +215,7 @@ TEST_P(KnowhereWrapperTest, serialize) {
}
{
std::string file_location = "/tmp/whatever";
std::string file_location = "/tmp/knowhere";
write_index(index_, file_location);
auto new_index = read_index(file_location);
EXPECT_EQ(new_index->GetType(), ConvertToCpuIndexType(index_type));
......
......@@ -24,7 +24,7 @@
using namespace zilliz::milvus;
TEST_F(MetricTest, Metric_Tes) {
TEST_F(MetricTest, METRIC_TEST) {
server::ConfigNode &configNode = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_METRIC);
configNode.SetValue(server::CONFIG_METRIC_COLLECTOR, "zabbix");
server::Metrics::GetInstance();
......@@ -122,7 +122,7 @@ TEST_F(MetricTest, Metric_Tes) {
delete [] qxb;
};
TEST_F(MetricTest, Collector_Metrics_Test){
TEST_F(MetricTest, COLLECTOR_METRICS_TEST){
engine::Status status = engine::Status::OK();
server::CollectInsertMetrics insert_metrics0(0, status);
status = engine::Status(DB_ERROR, "error");
......
......@@ -53,7 +53,7 @@ class AlgorithmTest : public testing::Test {
ResourceMgrPtr res_mgr_;
};
TEST_F(AlgorithmTest, ShortestPath_test) {
TEST_F(AlgorithmTest, SHORTESTPATH_TEST) {
std::vector<std::string> sp;
uint64_t cost;
cost = ShortestPath(disk_.lock(), gpu_0_.lock(), res_mgr_, sp);
......
......@@ -15,7 +15,7 @@ namespace zilliz {
namespace milvus {
namespace engine {
TEST(EventTest, start_up_event) {
TEST(EventTest, START_UP_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<StartUpEvent>(res);
ASSERT_FALSE(event->Dump().empty());
......@@ -23,7 +23,7 @@ TEST(EventTest, start_up_event) {
std::cout << *EventPtr(event);
}
TEST(EventTest, load_completed_event) {
TEST(EventTest, LOAD_COMPLETED_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<LoadCompletedEvent>(res, nullptr);
ASSERT_FALSE(event->Dump().empty());
......@@ -31,7 +31,7 @@ TEST(EventTest, load_completed_event) {
std::cout << *EventPtr(event);
}
TEST(EventTest, finish_task_event) {
TEST(EventTest, FINISH_TASK_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<FinishTaskEvent>(res, nullptr);
ASSERT_FALSE(event->Dump().empty());
......@@ -40,7 +40,7 @@ TEST(EventTest, finish_task_event) {
}
TEST(EventTest, tasktable_updated_event) {
TEST(EventTest, TASKTABLE_UPDATED_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<TaskTableUpdatedEvent>(res);
ASSERT_FALSE(event->Dump().empty());
......
......@@ -28,7 +28,7 @@ protected:
NodePtr isolated_node2_;
};
TEST_F(NodeTest, add_neighbour) {
TEST_F(NodeTest, ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
......@@ -37,7 +37,7 @@ TEST_F(NodeTest, add_neighbour) {
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
}
TEST_F(NodeTest, repeat_add_neighbour) {
TEST_F(NodeTest, REPEAT_ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
......@@ -47,7 +47,7 @@ TEST_F(NodeTest, repeat_add_neighbour) {
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
}
TEST_F(NodeTest, get_neighbours) {
TEST_F(NodeTest, GET_NEIGHBOURS) {
{
bool n2 = false, n3 = false;
auto node1_neighbours = node1_->GetNeighbours();
......@@ -72,7 +72,7 @@ TEST_F(NodeTest, get_neighbours) {
}
}
TEST_F(NodeTest, dump) {
TEST_F(NodeTest, DUMP) {
std::cout << node1_->Dump();
ASSERT_FALSE(node1_->Dump().empty());
......
......@@ -11,7 +11,7 @@
using namespace zilliz::milvus::engine;
TEST(normal_test, inst_test) {
TEST(NormalTest, INST_TEST) {
// ResourceMgr only compose resources, provide unified event
auto res_mgr = ResMgrInst::GetInstance();
......
......@@ -4,7 +4,7 @@
using namespace zilliz::milvus::engine;
TEST(resource_factory_test, create) {
TEST(ResourceFactoryTest, CREATE) {
auto disk = ResourceFactory::Create("ssd", "DISK", 0);
auto cpu = ResourceFactory::Create("cpu", "CPU", 0);
auto gpu = ResourceFactory::Create("gpu", "GPU", 0);
......
......@@ -44,19 +44,19 @@ protected:
ResourcePtr gpu_res;
};
TEST_F(ResourceMgrBaseTest, add) {
TEST_F(ResourceMgrBaseTest, ADD) {
auto resource = std::make_shared<TestResource>("test", 0, true, true);
auto ret = empty_mgr_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
}
TEST_F(ResourceMgrBaseTest, add_disk) {
TEST_F(ResourceMgrBaseTest, ADD_DISK) {
auto resource = std::make_shared<DiskResource>("disk", 0, true, true);
auto ret = empty_mgr_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
}
TEST_F(ResourceMgrBaseTest, connect) {
TEST_F(ResourceMgrBaseTest, CONNECT) {
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
empty_mgr_->Add(resource1);
......@@ -66,7 +66,7 @@ TEST_F(ResourceMgrBaseTest, connect) {
}
TEST_F(ResourceMgrBaseTest, invalid_connect) {
TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
empty_mgr_->Add(resource1);
......@@ -76,19 +76,19 @@ TEST_F(ResourceMgrBaseTest, invalid_connect) {
}
TEST_F(ResourceMgrBaseTest, clear) {
TEST_F(ResourceMgrBaseTest, CLEAR) {
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Clear();
ASSERT_EQ(mgr1_->GetNumOfResource(), 0);
}
TEST_F(ResourceMgrBaseTest, get_disk_resources) {
TEST_F(ResourceMgrBaseTest, GET_DISK_RESOURCES) {
auto disks = mgr1_->GetDiskResources();
ASSERT_EQ(disks.size(), 1);
ASSERT_EQ(disks[0].lock(), disk_res);
}
TEST_F(ResourceMgrBaseTest, get_all_resources) {
TEST_F(ResourceMgrBaseTest, GET_ALL_RESOURCES) {
bool disk = false, cpu = false, gpu = false;
auto resources = mgr1_->GetAllResources();
ASSERT_EQ(resources.size(), 3);
......@@ -103,13 +103,13 @@ TEST_F(ResourceMgrBaseTest, get_all_resources) {
ASSERT_TRUE(gpu);
}
TEST_F(ResourceMgrBaseTest, get_compute_resources) {
TEST_F(ResourceMgrBaseTest, GET_COMPUTE_RESOURCES) {
auto compute_resources = mgr1_->GetComputeResources();
ASSERT_EQ(compute_resources.size(), 1);
ASSERT_EQ(compute_resources[0], gpu_res);
}
TEST_F(ResourceMgrBaseTest, get_resource_by_type_and_deviceid) {
TEST_F(ResourceMgrBaseTest, GET_RESOURCE_BY_TYPE_AND_DEVICEID) {
auto cpu = mgr1_->GetResource(ResourceType::CPU, 1);
ASSERT_EQ(cpu, cpu_res);
......@@ -117,7 +117,7 @@ TEST_F(ResourceMgrBaseTest, get_resource_by_type_and_deviceid) {
ASSERT_EQ(invalid, nullptr);
}
TEST_F(ResourceMgrBaseTest, get_resource_by_name) {
TEST_F(ResourceMgrBaseTest, GET_RESOURCE_BY_NAME) {
auto disk = mgr1_->GetResource("disk");
ASSERT_EQ(disk, disk_res);
......@@ -125,26 +125,26 @@ TEST_F(ResourceMgrBaseTest, get_resource_by_name) {
ASSERT_EQ(invalid, nullptr);
}
TEST_F(ResourceMgrBaseTest, get_num_of_resource) {
TEST_F(ResourceMgrBaseTest, GET_NUM_OF_RESOURCE) {
ASSERT_EQ(empty_mgr_->GetNumOfResource(), 0);
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
}
TEST_F(ResourceMgrBaseTest, get_num_of_compute_resource) {
TEST_F(ResourceMgrBaseTest, GET_NUM_OF_COMPUTE_RESOURCE) {
ASSERT_EQ(empty_mgr_->GetNumOfComputeResource(), 0);
ASSERT_EQ(mgr1_->GetNumOfComputeResource(), 1);
}
TEST_F(ResourceMgrBaseTest, get_num_of_gpu_resource) {
TEST_F(ResourceMgrBaseTest, GET_NUM_OF_GPU_RESOURCE) {
ASSERT_EQ(empty_mgr_->GetNumGpuResource(), 0);
ASSERT_EQ(mgr1_->GetNumGpuResource(), 1);
}
TEST_F(ResourceMgrBaseTest, dump) {
TEST_F(ResourceMgrBaseTest, DUMP) {
ASSERT_FALSE(mgr1_->Dump().empty());
}
TEST_F(ResourceMgrBaseTest, dump_tasktables) {
TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) {
ASSERT_FALSE(mgr1_->DumpTaskTables().empty());
}
......@@ -169,7 +169,7 @@ class ResourceMgrAdvanceTest : public testing::Test {
ResourcePtr disk_res;
};
TEST_F(ResourceMgrAdvanceTest, register_subscriber) {
TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
bool flag = false;
auto callback = [&](EventPtr event) {
flag = true;
......
......@@ -46,42 +46,42 @@ protected:
ResourcePtr both_disable_ = nullptr;
};
TEST_F(ResourceBaseTest, name) {
TEST_F(ResourceBaseTest, NAME) {
ASSERT_EQ(only_loader_->name(), name1);
ASSERT_EQ(only_executor_->name(), name2);
ASSERT_EQ(both_enable_->name(), name3);
ASSERT_EQ(both_disable_->name(), name4);
}
TEST_F(ResourceBaseTest, type) {
TEST_F(ResourceBaseTest, TYPE) {
ASSERT_EQ(only_loader_->type(), ResourceType::DISK);
ASSERT_EQ(only_executor_->type(), ResourceType::CPU);
ASSERT_EQ(both_enable_->type(), ResourceType::GPU);
ASSERT_EQ(both_disable_->type(), ResourceType::TEST);
}
TEST_F(ResourceBaseTest, device_id) {
TEST_F(ResourceBaseTest, DEVICE_ID) {
ASSERT_EQ(only_loader_->device_id(), id1);
ASSERT_EQ(only_executor_->device_id(), id2);
ASSERT_EQ(both_enable_->device_id(), id3);
ASSERT_EQ(both_disable_->device_id(), id4);
}
TEST_F(ResourceBaseTest, has_loader) {
TEST_F(ResourceBaseTest, HAS_LOADER) {
ASSERT_TRUE(only_loader_->HasLoader());
ASSERT_FALSE(only_executor_->HasLoader());
ASSERT_TRUE(both_enable_->HasLoader());
ASSERT_FALSE(both_disable_->HasLoader());
}
TEST_F(ResourceBaseTest, has_executor) {
TEST_F(ResourceBaseTest, HAS_EXECUTOR) {
ASSERT_FALSE(only_loader_->HasExecutor());
ASSERT_TRUE(only_executor_->HasExecutor());
ASSERT_TRUE(both_enable_->HasExecutor());
ASSERT_FALSE(both_disable_->HasExecutor());
}
TEST_F(ResourceBaseTest, dump) {
TEST_F(ResourceBaseTest, DUMP) {
ASSERT_FALSE(only_loader_->Dump().empty());
ASSERT_FALSE(only_executor_->Dump().empty());
ASSERT_FALSE(both_enable_->Dump().empty());
......@@ -165,7 +165,7 @@ protected:
std::condition_variable cv_;
};
TEST_F(ResourceAdvanceTest, disk_resource_test) {
TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
......@@ -190,7 +190,7 @@ TEST_F(ResourceAdvanceTest, disk_resource_test) {
}
}
TEST_F(ResourceAdvanceTest, cpu_resource_test) {
TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
......@@ -215,7 +215,7 @@ TEST_F(ResourceAdvanceTest, cpu_resource_test) {
}
}
TEST_F(ResourceAdvanceTest, gpu_resource_test) {
TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
......@@ -240,7 +240,7 @@ TEST_F(ResourceAdvanceTest, gpu_resource_test) {
}
}
TEST_F(ResourceAdvanceTest, test_resource_test) {
TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
......
......@@ -68,7 +68,7 @@ protected:
const std::string CONFIG_FILE = "/tmp/milvus_sched_test/config.yaml";
};
TEST_F(SchedInstTest, simple_gpu) {
TEST_F(SchedInstTest, SIMPLE_GPU) {
StartSchedulerService();
}
......
......@@ -94,10 +94,6 @@ class SchedulerTest : public testing::Test {
protected:
void
SetUp() override {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
config.AddSequenceItem(server::CONFIG_GPU_IDS, "1");
ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false);
ResourcePtr gpu_0 = ResourceFactory::Create("gpu0", "GPU", 0);
ResourcePtr gpu_1 = ResourceFactory::Create("gpu1", "GPU", 1);
......@@ -142,7 +138,7 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) {
cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj);
}
TEST_F(SchedulerTest, OnLoadCompleted) {
TEST_F(SchedulerTest, ON_LOAD_COMPLETED) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = std::make_shared<meta::TableFileSchema>();
......@@ -162,7 +158,7 @@ TEST_F(SchedulerTest, OnLoadCompleted) {
}
TEST_F(SchedulerTest, PushTaskToNeighbourRandomlyTest) {
TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy1 = std::make_shared<meta::TableFileSchema>();
......@@ -233,7 +229,7 @@ protected:
};
TEST_F(SchedulerTest2, SpecifiedResourceTest) {
TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = std::make_shared<meta::TableFileSchema>();
......
......@@ -13,7 +13,7 @@ namespace milvus {
namespace engine {
TEST(TaskTest, invalid_index) {
TEST(TaskTest, INVALID_INDEX) {
auto search_task = std::make_shared<XSearchTask>(nullptr);
search_task->Load(LoadType::TEST, 10);
}
......
......@@ -32,18 +32,18 @@ protected:
std::vector<TaskTableItemPtr> items_;
};
TEST_F(TaskTableItemTest, construct) {
TEST_F(TaskTableItemTest, CONSTRUCT) {
ASSERT_EQ(default_.id, 0);
ASSERT_EQ(default_.task, nullptr);
ASSERT_EQ(default_.state, TaskTableItemState::INVALID);
}
TEST_F(TaskTableItemTest, destruct) {
TEST_F(TaskTableItemTest, DESTRUCT) {
auto p_item = new TaskTableItem();
delete p_item;
}
TEST_F(TaskTableItemTest, is_finish) {
TEST_F(TaskTableItemTest, IS_FINISH) {
for (auto &item : items_) {
if (item->state == TaskTableItemState::EXECUTED
|| item->state == TaskTableItemState::MOVED) {
......@@ -54,13 +54,13 @@ TEST_F(TaskTableItemTest, is_finish) {
}
}
TEST_F(TaskTableItemTest, dump) {
TEST_F(TaskTableItemTest, DUMP) {
for (auto &item : items_) {
ASSERT_FALSE(item->Dump().empty());
}
}
TEST_F(TaskTableItemTest, load) {
TEST_F(TaskTableItemTest, LOAD) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Load();
......@@ -74,7 +74,7 @@ TEST_F(TaskTableItemTest, load) {
}
}
TEST_F(TaskTableItemTest, loaded) {
TEST_F(TaskTableItemTest, LOADED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Loaded();
......@@ -88,7 +88,7 @@ TEST_F(TaskTableItemTest, loaded) {
}
}
TEST_F(TaskTableItemTest, execute) {
TEST_F(TaskTableItemTest, EXECUTE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Execute();
......@@ -103,7 +103,7 @@ TEST_F(TaskTableItemTest, execute) {
}
TEST_F(TaskTableItemTest, executed) {
TEST_F(TaskTableItemTest, EXECUTED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Executed();
......@@ -117,7 +117,7 @@ TEST_F(TaskTableItemTest, executed) {
}
}
TEST_F(TaskTableItemTest, move) {
TEST_F(TaskTableItemTest, MOVE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Move();
......@@ -131,7 +131,7 @@ TEST_F(TaskTableItemTest, move) {
}
}
TEST_F(TaskTableItemTest, moved) {
TEST_F(TaskTableItemTest, MOVED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Moved();
......@@ -163,7 +163,7 @@ protected:
TaskTable empty_table_;
};
TEST_F(TaskTableBaseTest, subscriber) {
TEST_F(TaskTableBaseTest, SUBSCRIBER) {
bool flag = false;
auto callback = [&]() {
flag = true;
......@@ -174,46 +174,46 @@ TEST_F(TaskTableBaseTest, subscriber) {
}
TEST_F(TaskTableBaseTest, put_task) {
TEST_F(TaskTableBaseTest, PUT_TASK) {
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
}
TEST_F(TaskTableBaseTest, put_invalid_test) {
TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) {
empty_table_.Put(invalid_task_);
ASSERT_EQ(empty_table_.Get(0)->task, invalid_task_);
}
TEST_F(TaskTableBaseTest, put_batch) {
TEST_F(TaskTableBaseTest, PUT_BATCH) {
std::vector<TaskPtr> tasks{task1_, task2_};
empty_table_.Put(tasks);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
ASSERT_EQ(empty_table_.Get(1)->task, task2_);
}
TEST_F(TaskTableBaseTest, put_empty_batch) {
TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) {
std::vector<TaskPtr> tasks{};
empty_table_.Put(tasks);
}
TEST_F(TaskTableBaseTest, empty) {
TEST_F(TaskTableBaseTest, EMPTY) {
ASSERT_TRUE(empty_table_.Empty());
empty_table_.Put(task1_);
ASSERT_FALSE(empty_table_.Empty());
}
TEST_F(TaskTableBaseTest, size) {
TEST_F(TaskTableBaseTest, SIZE) {
ASSERT_EQ(empty_table_.Size(), 0);
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Size(), 1);
}
TEST_F(TaskTableBaseTest, operator_) {
TEST_F(TaskTableBaseTest, OPERATOR) {
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Get(0), empty_table_[0]);
}
TEST_F(TaskTableBaseTest, pick_to_load) {
TEST_F(TaskTableBaseTest, PICK_TO_LOAD) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
......@@ -226,7 +226,7 @@ TEST_F(TaskTableBaseTest, pick_to_load) {
ASSERT_EQ(indexes[0], 2);
}
TEST_F(TaskTableBaseTest, pick_to_load_limit) {
TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
......@@ -241,7 +241,7 @@ TEST_F(TaskTableBaseTest, pick_to_load_limit) {
ASSERT_EQ(indexes[2], 4);
}
TEST_F(TaskTableBaseTest, pick_to_load_cache) {
TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
......@@ -262,7 +262,7 @@ TEST_F(TaskTableBaseTest, pick_to_load_cache) {
ASSERT_EQ(indexes[0], 2);
}
TEST_F(TaskTableBaseTest, pick_to_execute) {
TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
......@@ -276,7 +276,7 @@ TEST_F(TaskTableBaseTest, pick_to_execute) {
ASSERT_EQ(indexes[0], 2);
}
TEST_F(TaskTableBaseTest, pick_to_execute_limit) {
TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
......@@ -292,7 +292,7 @@ TEST_F(TaskTableBaseTest, pick_to_execute_limit) {
ASSERT_EQ(indexes[1], 3);
}
TEST_F(TaskTableBaseTest, pick_to_execute_cache) {
TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
......@@ -340,7 +340,7 @@ protected:
TaskTable table1_;
};
TEST_F(TaskTableAdvanceTest, load) {
TEST_F(TaskTableAdvanceTest, LOAD) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
......@@ -359,7 +359,7 @@ TEST_F(TaskTableAdvanceTest, load) {
}
}
TEST_F(TaskTableAdvanceTest, loaded) {
TEST_F(TaskTableAdvanceTest, LOADED) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
......@@ -378,7 +378,7 @@ TEST_F(TaskTableAdvanceTest, loaded) {
}
}
TEST_F(TaskTableAdvanceTest, execute) {
TEST_F(TaskTableAdvanceTest, EXECUTE) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
......@@ -397,7 +397,7 @@ TEST_F(TaskTableAdvanceTest, execute) {
}
}
TEST_F(TaskTableAdvanceTest, executed) {
TEST_F(TaskTableAdvanceTest, EXECUTED) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
......@@ -416,7 +416,7 @@ TEST_F(TaskTableAdvanceTest, executed) {
}
}
TEST_F(TaskTableAdvanceTest, move) {
TEST_F(TaskTableAdvanceTest, MOVE) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
......@@ -435,7 +435,7 @@ TEST_F(TaskTableAdvanceTest, move) {
}
}
TEST_F(TaskTableAdvanceTest, moved) {
TEST_F(TaskTableAdvanceTest, MOVED) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
......
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
mode: single # milvus deployment type: single, cluster, read_only
time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
db_config:
db_path: /tmp/milvus # milvus data storage path
......@@ -17,6 +17,7 @@ db_config:
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
metric_config:
is_startup: off # if monitoring start: on, off
......@@ -32,8 +33,6 @@ cache_config:
insert_cache_immediately: false # insert data will be load into cache immediately for hot query
gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
gpu_ids: # gpu id
- 0
engine_config:
use_blas_threshold: 20
......
......@@ -166,9 +166,6 @@ TEST(CacheTest, CPU_CACHE_TEST) {
}
TEST(CacheTest, GPU_CACHE_TEST) {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance(0);
const int dim = 256;
......
......@@ -144,8 +144,4 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size));
err = config.ValidateConfig();
ASSERT_NE(err, SERVER_SUCCESS);
server_config.SetValue(server::CONFIG_GPU_INDEX, "9999");
err = config.ValidateConfig();
ASSERT_NE(err, SERVER_SUCCESS);
}
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册