Skip to content

M
milvus

milvus / milvus
12 个月 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 5ffaa4fd 编写于 作者: S starlord
浏览文件
操作

excluse some files 


Former-commit-id: 9348aa652665488621be0dac17a26ad9198d2db1
上级 a8f6e5dc 64e3cc01
展开全部 隐藏空白更改
内联 并排
Showing with 1487 addition and 918 deletion
cpp/CHANGELOG.md
浏览文件 @ 5ffaa4fd
...@@ -18,7 +18,8 @@ Please mark all change in change log and use the ticket from JIRA. ...@@ -18,7 +18,8 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-562 - Add JobMgr and TaskCreator in Scheduler
- MS-566 - Refactor cmake - MS-566 - Refactor cmake
- MS-555 - Remove old scheduler - MS-555 - Remove old scheduler
- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data - MS-574 - Milvus configuration refactor
- MS-578 - Make sure milvus5.0 don't crack 0.3.1 data
- MS-585 - Update namespace in scheduler - MS-585 - Update namespace in scheduler
## New Feature ## New Feature
......
cpp/CMakeLists.txt
浏览文件 @ 5ffaa4fd
...@@ -44,7 +44,6 @@ set(MILVUS_VERSION "${GIT_BRANCH_NAME}") ...@@ -44,7 +44,6 @@ set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
find_package(ClangTools) find_package(ClangTools)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
if(CMAKE_BUILD_TYPE STREQUAL "Release") if(CMAKE_BUILD_TYPE STREQUAL "Release")
...@@ -57,6 +56,9 @@ message(STATUS "Build type = ${BUILD_TYPE}") ...@@ -57,6 +56,9 @@ message(STATUS "Build type = ${BUILD_TYPE}")
project(milvus VERSION "${MILVUS_VERSION}") project(milvus VERSION "${MILVUS_VERSION}")
project(milvus_engine LANGUAGES CUDA CXX) project(milvus_engine LANGUAGES CUDA CXX)
unset(CMAKE_EXPORT_COMPILE_COMMANDS CACHE)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}") set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}") set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}") set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
......
cpp/conf/server_config.template
浏览文件 @ 5ffaa4fd
server_config: server_config:
address: 0.0.0.0 # milvus server ip address (IPv4) address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 port: 19530 # port range: 1025 ~ 65534
mode: single # milvus deployment type: single, cluster, read_only mode: single # deployment type: single, cluster, read_only
time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time time_zone: UTC+8
db_config: db_config:
db_path: @MILVUS_DB_PATH@ # milvus data storage path path: @MILVUS_DB_PATH@ # milvus database path
db_slave_path: # secondry data storage path, split by semicolon slave_path: # secondary database path, split by semicolon
# URI format: dialect://username:password@host:port/database # URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters # All parts except dialect are optional, but you MUST include the delimiters
# Currently dialect supports mysql or sqlite # Currently dialect supports mysql or sqlite
db_backend_url: sqlite://:@:/ backend_url: sqlite://:@:/
archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. buffer_size: 4 # GB, maximum insert buffer size allowed
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB build_index_gpu: 0 # gpu id used for building index
build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
metric_config: metric_config:
is_startup: off # if monitoring start: on, off auto_bootup: off # whether enable monitoring when bootup
collector: prometheus # metrics collector: prometheus collector: prometheus # prometheus
prometheus_config: # following are prometheus configure prometheus_config:
port: 8080 # the port prometheus use to fetch metrics port: 8080 # port prometheus used to fetch metrics
push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address
push_gateway_port: 9091 # push method configure: push gateway port
cache_config: cache_config:
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory cpu_mem_capacity: 16 # GB, CPU memory size used for cache
cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered
insert_cache_immediately: false # insert data will be load into cache immediately for hot query cache_insert_data: false # whether load data into cache when insert
engine_config: engine_config:
use_blas_threshold: 20 blas_threshold: 20
resource_config: resource_config:
mode: simple mode: simple
resources: pool:
# - cpu - cpu
- gpu0 - gpu0
cpp/src/cache/CpuCacheMgr.cpp
浏览文件 @ 5ffaa4fd
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include "CpuCacheMgr.h" #include "CpuCacheMgr.h"
#include "server/ServerConfig.h" #include "server/Config.h"
#include "utils/Log.h" #include "utils/Log.h"
namespace zilliz { namespace zilliz {
...@@ -29,17 +29,27 @@ namespace { ...@@ -29,17 +29,27 @@ namespace {
} }
CpuCacheMgr::CpuCacheMgr() { CpuCacheMgr::CpuCacheMgr() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); server::Config& config = server::Config::GetInstance();
int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); Status s;
cap *= unit;
int32_t cpu_mem_cap;
s = config.GetCacheConfigCpuMemCapacity(cpu_mem_cap);
if (!s.ok()) {
SERVER_LOG_ERROR << s.message();
}
int64_t cap = cpu_mem_cap * unit;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32); cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); float cpu_mem_threshold;
if(free_percent > 0.0 && free_percent <= 1.0) { s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold);
cache_->set_freemem_percent(free_percent); if (!s.ok()) {
SERVER_LOG_ERROR << s.message();
}
if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) {
cache_->set_freemem_percent(cpu_mem_threshold);
} else { } else {
SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent << SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold
", defaultly set to " << cache_->freemem_percent(); << ", by default set to " << cache_->freemem_percent();
} }
} }
......
cpp/src/cache/GpuCacheMgr.cpp
浏览文件 @ 5ffaa4fd
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
#include <sstream> #include <sstream>
#include "utils/Log.h" #include "utils/Log.h"
#include "GpuCacheMgr.h" #include "GpuCacheMgr.h"
#include "server/ServerConfig.h" #include "server/Config.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
...@@ -33,18 +33,27 @@ namespace { ...@@ -33,18 +33,27 @@ namespace {
} }
GpuCacheMgr::GpuCacheMgr() { GpuCacheMgr::GpuCacheMgr() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); server::Config& config = server::Config::GetInstance();
Status s;
int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0); int32_t gpu_mem_cap;
cap *= G_BYTE; s = config.GetCacheConfigGpuMemCapacity(gpu_mem_cap);
if (!s.ok()) {
SERVER_LOG_ERROR << s.message();
}
int32_t cap = gpu_mem_cap * G_BYTE;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32); cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85); float gpu_mem_threshold;
if (free_percent > 0.0 && free_percent <= 1.0) { s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold);
cache_->set_freemem_percent(free_percent); if (!s.ok()) {
SERVER_LOG_ERROR << s.message();
}
if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) {
cache_->set_freemem_percent(gpu_mem_threshold);
} else { } else {
SERVER_LOG_ERROR << "Invalid gpu_cache_free_percent: " << free_percent << SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold
", defaultly set to " << cache_->freemem_percent(); << ", by default set to " << cache_->freemem_percent();
} }
} }
......
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @ 5ffaa4fd
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include "src/wrapper/vec_index.h" #include "src/wrapper/vec_index.h"
#include "src/wrapper/vec_impl.h" #include "src/wrapper/vec_impl.h"
#include "knowhere/common/Exception.h" #include "knowhere/common/Exception.h"
#include "server/Config.h"
#include <stdexcept> #include <stdexcept>
...@@ -326,9 +327,9 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { ...@@ -326,9 +327,9 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
// TODO(linxj): remove. // TODO(linxj): remove.
Status ExecutionEngineImpl::Init() { Status ExecutionEngineImpl::Init() {
using namespace zilliz::milvus::server; using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance(); server::Config &config = server::Config::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_DB); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); if (!s.ok()) return s;
return Status::OK(); return Status::OK();
} }
......
cpp/src/db/engine/ExecutionEngineImpl.h
浏览文件 @ 5ffaa4fd
...@@ -101,7 +101,7 @@ protected: ...@@ -101,7 +101,7 @@ protected:
std::string location_; std::string location_;
int32_t nlist_ = 0; int32_t nlist_ = 0;
int64_t gpu_num_ = 0; int32_t gpu_num_ = 0;
}; };
......
cpp/src/main.cpp
浏览文件 @ 5ffaa4fd
...@@ -110,7 +110,7 @@ main(int argc, char *argv[]) { ...@@ -110,7 +110,7 @@ main(int argc, char *argv[]) {
signal(SIGUSR2, server::SignalUtil::HandleSignal); signal(SIGUSR2, server::SignalUtil::HandleSignal);
signal(SIGTERM, server::SignalUtil::HandleSignal); signal(SIGTERM, server::SignalUtil::HandleSignal);
server::Server &server = server::Server::Instance(); server::Server &server = server::Server::GetInstance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
server.Start(); server.Start();
......
cpp/src/metrics/MetricBase.h
浏览文件 @ 5ffaa4fd
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
#pragma once #pragma once
#include "utils/Error.h" #include "utils/Error.h"
#include "server/ServerConfig.h"
#include "SystemInfo.h" #include "SystemInfo.h"
namespace zilliz { namespace zilliz {
...@@ -83,11 +82,6 @@ class MetricsBase{ ...@@ -83,11 +82,6 @@ class MetricsBase{
virtual void CPUTemperature() {}; virtual void CPUTemperature() {};
}; };
} }
} }
} }
\ No newline at end of file
cpp/src/metrics/Metrics.cpp
浏览文件 @ 5ffaa4fd
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
// under the License. // under the License.
#include "Metrics.h" #include "Metrics.h"
#include "server/Config.h"
#include "PrometheusMetrics.h" #include "PrometheusMetrics.h"
...@@ -31,8 +32,10 @@ Metrics::GetInstance() { ...@@ -31,8 +32,10 @@ Metrics::GetInstance() {
MetricsBase & MetricsBase &
Metrics::CreateMetricsCollector() { Metrics::CreateMetricsCollector() {
ConfigNode &config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); Config &config = Config::GetInstance();
std::string collector_type_str = config.GetValue(CONFIG_METRIC_COLLECTOR); std::string collector_type_str;
config.GetMetricConfigCollector(collector_type_str);
if (collector_type_str == "prometheus") { if (collector_type_str == "prometheus") {
return PrometheusMetrics::GetInstance(); return PrometheusMetrics::GetInstance();
......
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @ 5ffaa4fd
...@@ -16,8 +16,9 @@ ...@@ -16,8 +16,9 @@
// under the License. // under the License.
#include <cache/GpuCacheMgr.h> #include "cache/GpuCacheMgr.h"
#include "PrometheusMetrics.h" #include "PrometheusMetrics.h"
#include "server/Config.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "SystemInfo.h" #include "SystemInfo.h"
...@@ -26,15 +27,19 @@ namespace zilliz { ...@@ -26,15 +27,19 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
ErrorCode ErrorCode
PrometheusMetrics::Init() { PrometheusMetrics::Init() {
try { try {
ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); Config &config = Config::GetInstance();
startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "on"; Status s = config.GetMetricConfigAutoBootup(startup_);
if(!startup_) return SERVER_SUCCESS; if (!s.ok()) return s.code();
if (!startup_) return SERVER_SUCCESS;
// Following should be read from config file. // Following should be read from config file.
const std::string bind_address = configNode.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT); std::string bind_address;
const std::string uri = std::string("/metrics"); s = config.GetMetricConfigPrometheusPort(bind_address);
if (!s.ok()) return s.code();
const std::string uri = std::string("/tmp/metrics");
const std::size_t num_threads = 2; const std::size_t num_threads = 2;
// Init Exposer // Init Exposer
......
cpp/src/metrics/PrometheusMetrics.h
浏览文件 @ 5ffaa4fd
...@@ -17,15 +17,13 @@ ...@@ -17,15 +17,13 @@
#pragma once #pragma once
#include "utils/Error.h"
#include <memory> #include <memory>
#include <vector> #include <vector>
#include <prometheus/registry.h> #include <prometheus/registry.h>
#include <prometheus/exposer.h> #include <prometheus/exposer.h>
#include <iostream> #include <iostream>
#include "server/ServerConfig.h"
#include "utils/Error.h"
#include "MetricBase.h" #include "MetricBase.h"
...@@ -38,10 +36,6 @@ namespace zilliz { ...@@ -38,10 +36,6 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
class PrometheusMetrics: public MetricsBase { class PrometheusMetrics: public MetricsBase {
public: public:
...@@ -107,11 +101,6 @@ class PrometheusMetrics: public MetricsBase { ...@@ -107,11 +101,6 @@ class PrometheusMetrics: public MetricsBase {
void GPUTemperature() override; void GPUTemperature() override;
void CPUTemperature() override; void CPUTemperature() override;
std::shared_ptr<prometheus::Exposer> &exposer_ptr() {return exposer_ptr_; } std::shared_ptr<prometheus::Exposer> &exposer_ptr() {return exposer_ptr_; }
// prometheus::Exposer& exposer() { return exposer_;} // prometheus::Exposer& exposer() { return exposer_;}
std::shared_ptr<prometheus::Registry> &registry_ptr() {return registry_; } std::shared_ptr<prometheus::Registry> &registry_ptr() {return registry_; }
...@@ -125,8 +114,6 @@ class PrometheusMetrics: public MetricsBase { ...@@ -125,8 +114,6 @@ class PrometheusMetrics: public MetricsBase {
// .Register(*registry_); // .Register(*registry_);
// prometheus::Counter &connection_total_ = connect_request_.Add({}); // prometheus::Counter &connection_total_ = connect_request_.Add({});
////all from DBImpl.cpp ////all from DBImpl.cpp
using BucketBoundaries = std::vector<double>; using BucketBoundaries = std::vector<double>;
//record add_group request //record add_group request
...@@ -472,10 +459,8 @@ class PrometheusMetrics: public MetricsBase { ...@@ -472,10 +459,8 @@ class PrometheusMetrics: public MetricsBase {
.Name("CPU_temperature") .Name("CPU_temperature")
.Help("CPU temperature") .Help("CPU temperature")
.Register(*registry_); .Register(*registry_);
}; };
} }
} }
} }
......
cpp/src/scheduler/SchedInst.cpp
浏览文件 @ 5ffaa4fd
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include "SchedInst.h" #include "SchedInst.h"
#include "server/ServerConfig.h" #include "server/Config.h"
#include "ResourceFactory.h" #include "ResourceFactory.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "Utils.h" #include "Utils.h"
...@@ -38,13 +38,15 @@ std::mutex JobMgrInst::mutex_; ...@@ -38,13 +38,15 @@ std::mutex JobMgrInst::mutex_;
void void
load_simple_config() { load_simple_config() {
server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); server::Config &config = server::Config::GetInstance();
auto mode = config.GetValue("mode", "simple"); std::string mode;
config.GetResourceConfigMode(mode);
std::vector<std::string> pool;
config.GetResourceConfigPool(pool);
auto resources = config.GetSequence("resources");
bool cpu = false; bool cpu = false;
std::set<uint64_t> gpu_ids; std::set<uint64_t> gpu_ids;
for (auto &resource : resources) { for (auto &resource : pool) {
if (resource == "cpu") { if (resource == "cpu") {
cpu = true; cpu = true;
break; break;
...@@ -82,7 +84,7 @@ load_simple_config() { ...@@ -82,7 +84,7 @@ load_simple_config() {
void void
load_advance_config() { load_advance_config() {
// try { // try {
// server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); // server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE);
// //
// if (config.GetChildren().empty()) throw "resource_config null exception"; // if (config.GetChildren().empty()) throw "resource_config null exception";
// //
......
cpp/src/server/Config.cpp 0 → 100644
浏览文件 @ 5ffaa4fd
此差异已折叠。
cpp/src/server/Config.h 0 → 100644
浏览文件 @ 5ffaa4fd
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <mutex>
#include <unordered_map>
#include "yaml-cpp/yaml.h"
#include "utils/Status.h"
#include "config/ConfigNode.h"
namespace zilliz {
namespace milvus {
namespace server {
/* server config */
static const char* CONFIG_SERVER = "server_config";
static const char* CONFIG_SERVER_ADDRESS = "address";
static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1";
static const char* CONFIG_SERVER_PORT = "port";
static const char* CONFIG_SERVER_PORT_DEFAULT = "19530";
static const char* CONFIG_SERVER_MODE = "mode";
static const char* CONFIG_SERVER_MODE_DEFAULT = "single";
static const char* CONFIG_SERVER_TIME_ZONE = "time_zone";
static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8";
/* db config */
static const char* CONFIG_DB = "db_config";
static const char* CONFIG_DB_PATH = "path";
static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus";
static const char* CONFIG_DB_SLAVE_PATH = "slave_path";
static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = "";
static const char* CONFIG_DB_BACKEND_URL = "backend_url";
static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size";
static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
/* cache config */
static const char* CONFIG_CACHE = "cache_config";
static const char* CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity";
static const char* CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16";
static const char* CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity";
static const char* CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0";
static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold";
static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false";
/* metric config */
static const char* CONFIG_METRIC = "metric_config";
static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup";
static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false";
static const char* CONFIG_METRIC_COLLECTOR = "collector";
static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus";
static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config";
static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port";
static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080";
/* engine config */
static const char* CONFIG_ENGINE = "engine_config";
static const char* CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold";
static const char* CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
/* resource config */
static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_POOL = "pool";
class Config {
public:
static Config& GetInstance();
Status LoadConfigFile(const std::string& filename);
void PrintAll();
private:
ConfigNode& GetConfigNode(const std::string& name);
Status GetConfigValueInMem(const std::string& parent_key,
const std::string& child_key,
std::string& value);
void SetConfigValueInMem(const std::string& parent_key,
const std::string& child_key,
const std::string& value);
void PrintConfigSection(const std::string& config_node_name);
///////////////////////////////////////////////////////////////////////////
/* server config */
Status CheckServerConfigAddress(const std::string& value);
Status CheckServerConfigPort(const std::string& value);
Status CheckServerConfigMode(const std::string& value);
Status CheckServerConfigTimeZone(const std::string& value);
/* db config */
Status CheckDBConfigPath(const std::string& value);
Status CheckDBConfigSlavePath(const std::string& value);
Status CheckDBConfigBackendUrl(const std::string& value);
Status CheckDBConfigArchiveDiskThreshold(const std::string& value);
Status CheckDBConfigArchiveDaysThreshold(const std::string& value);
Status CheckDBConfigBufferSize(const std::string& value);
Status CheckDBConfigBuildIndexGPU(const std::string& value);
/* metric config */
Status CheckMetricConfigAutoBootup(const std::string& value);
Status CheckMetricConfigCollector(const std::string& value);
Status CheckMetricConfigPrometheusPort(const std::string& value);
/* cache config */
Status CheckCacheConfigCpuMemCapacity(const std::string& value);
Status CheckCacheConfigCpuMemThreshold(const std::string& value);
Status CheckCacheConfigGpuMemCapacity(const std::string& value);
Status CheckCacheConfigGpuMemThreshold(const std::string& value);
Status CheckCacheConfigCacheInsertData(const std::string& value);
/* engine config */
Status CheckEngineConfigBlasThreshold(const std::string& value);
Status CheckEngineConfigOmpThreadNum(const std::string& value);
/* resource config */
Status CheckResourceConfigMode(const std::string& value);
Status CheckResourceConfigPool(const std::vector<std::string>& value);
///////////////////////////////////////////////////////////////////////////
/* server config */
std::string GetServerConfigStrAddress();
std::string GetServerConfigStrPort();
std::string GetServerConfigStrMode();
std::string GetServerConfigStrTimeZone();
/* db config */
std::string GetDBConfigStrPath();
std::string GetDBConfigStrSlavePath();
std::string GetDBConfigStrBackendUrl();
std::string GetDBConfigStrArchiveDiskThreshold();
std::string GetDBConfigStrArchiveDaysThreshold();
std::string GetDBConfigStrBufferSize();
std::string GetDBConfigStrBuildIndexGPU();
/* metric config */
std::string GetMetricConfigStrAutoBootup();
std::string GetMetricConfigStrCollector();
std::string GetMetricConfigStrPrometheusPort();
/* cache config */
std::string GetCacheConfigStrCpuMemCapacity();
std::string GetCacheConfigStrCpuMemThreshold();
std::string GetCacheConfigStrGpuMemCapacity();
std::string GetCacheConfigStrGpuMemThreshold();
std::string GetCacheConfigStrCacheInsertData();
/* engine config */
std::string GetEngineConfigStrBlasThreshold();
std::string GetEngineConfigStrOmpThreadNum();
/* resource config */
std::string GetResourceConfigStrMode();
public:
/* server config */
Status GetServerConfigAddress(std::string& value);
Status GetServerConfigPort(std::string& value);
Status GetServerConfigMode(std::string& value);
Status GetServerConfigTimeZone(std::string& value);
/* db config */
Status GetDBConfigPath(std::string& value);
Status GetDBConfigSlavePath(std::string& value);
Status GetDBConfigBackendUrl(std::string& value);
Status GetDBConfigArchiveDiskThreshold(int32_t& value);
Status GetDBConfigArchiveDaysThreshold(int32_t& value);
Status GetDBConfigBufferSize(int32_t& value);
Status GetDBConfigBuildIndexGPU(int32_t& value);
/* metric config */
Status GetMetricConfigAutoBootup(bool& value);
Status GetMetricConfigCollector(std::string& value);
Status GetMetricConfigPrometheusPort(std::string& value);
/* cache config */
Status GetCacheConfigCpuMemCapacity(int32_t& value);
Status GetCacheConfigCpuMemThreshold(float& value);
Status GetCacheConfigGpuMemCapacity(int32_t& value);
Status GetCacheConfigGpuMemThreshold(float& value);
Status GetCacheConfigCacheInsertData(bool& value);
/* engine config */
Status GetEngineConfigBlasThreshold(int32_t& value);
Status GetEngineConfigOmpThreadNum(int32_t& value);
/* resource config */
Status GetResourceConfigMode(std::string& value);
Status GetResourceConfigPool(std::vector<std::string>& value);
public:
/* server config */
Status SetServerConfigAddress(const std::string& value);
Status SetServerConfigPort(const std::string& value);
Status SetServerConfigMode(const std::string& value);
Status SetServerConfigTimeZone(const std::string& value);
/* db config */
Status SetDBConfigPath(const std::string& value);
Status SetDBConfigSlavePath(const std::string& value);
Status SetDBConfigBackendUrl(const std::string& value);
Status SetDBConfigArchiveDiskThreshold(const std::string& value);
Status SetDBConfigArchiveDaysThreshold(const std::string& value);
Status SetDBConfigBufferSize(const std::string& value);
Status SetDBConfigBuildIndexGPU(const std::string& value);
/* metric config */
Status SetMetricConfigAutoBootup(const std::string& value);
Status SetMetricConfigCollector(const std::string& value);
Status SetMetricConfigPrometheusPort(const std::string& value);
/* cache config */
Status SetCacheConfigCpuMemCapacity(const std::string& value);
Status SetCacheConfigCpuMemThreshold(const std::string& value);
Status SetCacheConfigGpuMemCapacity(const std::string& value);
Status SetCacheConfigGpuMemThreshold(const std::string& value);
Status SetCacheConfigCacheInsertData(const std::string& value);
/* engine config */
Status SetEngineConfigBlasThreshold(const std::string& value);
Status SetEngineConfigOmpThreadNum(const std::string& value);
/* resource config */
Status SetResourceConfigMode(const std::string& value);
private:
std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;
std::mutex mutex_;
};
}
}
}
cpp/src/server/DBWrapper.cpp
浏览文件 @ 5ffaa4fd
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include "DBWrapper.h" #include "DBWrapper.h"
#include "ServerConfig.h" #include "Config.h"
#include "db/DBFactory.h" #include "db/DBFactory.h"
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
#include "utils/Log.h" #include "utils/Log.h"
...@@ -35,22 +35,34 @@ DBWrapper::DBWrapper() { ...@@ -35,22 +35,34 @@ DBWrapper::DBWrapper() {
} }
Status DBWrapper::StartService() { Status DBWrapper::StartService() {
Config& config = Config::GetInstance();
Status s;
//db config //db config
engine::DBOptions opt; engine::DBOptions opt;
ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
opt.meta_.backend_uri_ = db_config.GetValue(CONFIG_DB_URL);
std::string db_path = db_config.GetValue(CONFIG_DB_PATH);
opt.meta_.path_ = db_path + "/db";
std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_);
if (!s.ok()) return s;
std::string path;
s = config.GetDBConfigPath(path);
if (!s.ok()) return s;
opt.meta_.path_ = path + "/db";
std::string db_slave_path;
s = config.GetDBConfigSlavePath(db_slave_path);
if (!s.ok()) return s;
StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_);
// cache config // cache config
ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_);
opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false); if (!s.ok()) return s;
std::string mode;
s = config.GetServerConfigMode(mode);
if (!s.ok()) return s;
ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER);
std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single");
if (mode == "single") { if (mode == "single") {
opt.mode_ = engine::DBOptions::MODE::SINGLE; opt.mode_ = engine::DBOptions::MODE::SINGLE;
} }
...@@ -66,9 +78,10 @@ Status DBWrapper::StartService() { ...@@ -66,9 +78,10 @@ Status DBWrapper::StartService() {
} }
// engine config // engine config
ConfigNode& engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); int32_t omp_thread;
int32_t omp_thread = engine_config.GetInt32Value(CONFIG_OMP_THREAD_NUM, 0); s = config.GetEngineConfigOmpThreadNum(omp_thread);
if(omp_thread > 0) { if (!s.ok()) return s;
if (omp_thread > 0) {
omp_set_num_threads(omp_thread); omp_set_num_threads(omp_thread);
SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread;
} else { } else {
...@@ -79,16 +92,24 @@ Status DBWrapper::StartService() { ...@@ -79,16 +92,24 @@ Status DBWrapper::StartService() {
} }
} }
faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20);//init faiss global variable //init faiss global variable
int32_t blas_threshold;
s = config.GetEngineConfigBlasThreshold(blas_threshold);
if (!s.ok()) return s;
faiss::distance_compute_blas_threshold = blas_threshold;
//set archive config //set archive config
engine::ArchiveConf::CriteriaT criterial; engine::ArchiveConf::CriteriaT criterial;
int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); int32_t disk, days;
int64_t days = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); s = config.GetDBConfigArchiveDiskThreshold(disk);
if(disk > 0) { if (!s.ok()) return s;
if (disk > 0) {
criterial[engine::ARCHIVE_CONF_DISK] = disk; criterial[engine::ARCHIVE_CONF_DISK] = disk;
} }
if(days > 0) {
s = config.GetDBConfigArchiveDaysThreshold(days);
if (!s.ok()) return s;
if (days > 0) {
criterial[engine::ARCHIVE_CONF_DAYS] = days; criterial[engine::ARCHIVE_CONF_DAYS] = days;
} }
opt.meta_.archive_conf_.SetCriterias(criterial); opt.meta_.archive_conf_.SetCriterias(criterial);
......
cpp/src/server/Server.cpp
浏览文件 @ 5ffaa4fd
...@@ -16,14 +16,6 @@ ...@@ -16,14 +16,6 @@
// under the License. // under the License.
#include <thread> #include <thread>
#include "Server.h"
#include "server/grpc_impl/GrpcServer.h"
#include "utils/Log.h"
#include "utils/LogUtil.h"
#include "utils/SignalUtil.h"
#include "utils/TimeRecorder.h"
#include "metrics/Metrics.h"
#include <fcntl.h> #include <fcntl.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
...@@ -31,10 +23,17 @@ ...@@ -31,10 +23,17 @@
//#include <numaif.h> //#include <numaif.h>
#include <unistd.h> #include <unistd.h>
#include <string.h> #include <string.h>
#include <src/scheduler/SchedInst.h>
#include "src/wrapper/KnowhereResource.h"
#include "Server.h"
#include "server/grpc_impl/GrpcServer.h"
#include "server/Config.h"
#include "utils/Log.h"
#include "utils/LogUtil.h"
#include "utils/SignalUtil.h"
#include "utils/TimeRecorder.h"
#include "metrics/Metrics.h" #include "metrics/Metrics.h"
#include "scheduler/SchedInst.h"
#include "wrapper/KnowhereResource.h"
#include "DBWrapper.h" #include "DBWrapper.h"
...@@ -43,18 +42,11 @@ namespace milvus { ...@@ -43,18 +42,11 @@ namespace milvus {
namespace server { namespace server {
Server & Server &
Server::Instance() { Server::GetInstance() {
static Server server; static Server server;
return server; return server;
} }
Server::Server() {
}
Server::~Server() {
}
void void
Server::Init(int64_t daemonized, Server::Init(int64_t daemonized,
const std::string &pid_filename, const std::string &pid_filename,
...@@ -168,10 +160,14 @@ Server::Start() { ...@@ -168,10 +160,14 @@ Server::Start() {
} }
/* log path is defined in Config file, so InitLog must be called after LoadConfig */ /* log path is defined in Config file, so InitLog must be called after LoadConfig */
ServerConfig &config = ServerConfig::GetInstance(); Config &config = Config::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER); std::string time_zone;
Status s = config.GetServerConfigTimeZone(time_zone);
if (!s.ok()) {
std::cerr << "Fail to get server config timezone" << std::endl;
return;
}
std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8");
if (time_zone.length() == 3) { if (time_zone.length() == 3) {
time_zone = "CUT"; time_zone = "CUT";
} else { } else {
...@@ -239,13 +235,12 @@ Server::Stop() { ...@@ -239,13 +235,12 @@ Server::Stop() {
ErrorCode ErrorCode
Server::LoadConfig() { Server::LoadConfig() {
ServerConfig::GetInstance().LoadConfigFile(config_filename_); Config& config = Config::GetInstance();
auto status = ServerConfig::GetInstance().ValidateConfig(); Status s = config.LoadConfigFile(config_filename_);
if (!status.ok()) { if (!s.ok()) {
std::cerr << "Failed to load config file: " << config_filename_ << std::endl; std::cerr << "Failed to load config file: " << config_filename_ << std::endl;
exit(0); exit(0);
} }
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
......
cpp/src/server/Server.h
浏览文件 @ 5ffaa4fd
...@@ -29,7 +29,7 @@ namespace server { ...@@ -29,7 +29,7 @@ namespace server {
class Server { class Server {
public: public:
static Server &Instance(); static Server &GetInstance();
void Init(int64_t daemonized, void Init(int64_t daemonized,
const std::string &pid_filename, const std::string &pid_filename,
...@@ -40,8 +40,8 @@ class Server { ...@@ -40,8 +40,8 @@ class Server {
void Stop(); void Stop();
private: private:
Server(); Server() = default;
~Server(); ~Server() = default;
void Daemonize(); void Daemonize();
......
cpp/src/server/ServerConfig.cpp 已删除 100644 → 0
浏览文件 @ a8f6e5dc
此差异已折叠。
cpp/src/server/ServerConfig.h 已删除 100644 → 0
浏览文件 @ a8f6e5dc
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "utils/Status.h"
#include "config/ConfigNode.h"
#include "yaml-cpp/yaml.h"
namespace zilliz {
namespace milvus {
namespace server {
static const char* CONFIG_SERVER = "server_config";
static const char* CONFIG_SERVER_ADDRESS = "address";
static const char* CONFIG_SERVER_PORT = "port";
static const char* CONFIG_CLUSTER_MODE = "mode";
static const char* CONFIG_TIME_ZONE = "time_zone";
static const char* CONFIG_DB = "db_config";
static const char* CONFIG_DB_URL = "db_backend_url";
static const char* CONFIG_DB_PATH = "db_path";
static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path";
static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char* CONFIG_DB_PARALLEL_REDUCE = "parallel_reduce";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_LOG = "log_config";
static const char* CONFIG_CACHE = "cache_config";
static const char* CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity";
static const char* CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity";
static const char* CACHE_FREE_PERCENT = "cpu_cache_free_percent";
static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately";
static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent";
static const char* CONFIG_METRIC = "metric_config";
static const char* CONFIG_METRIC_IS_STARTUP = "is_startup";
static const char* CONFIG_METRIC_COLLECTOR = "collector";
static const char* CONFIG_PROMETHEUS = "prometheus_config";
static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port";
static const char* CONFIG_ENGINE = "engine_config";
static const char* CONFIG_DCBT = "use_blas_threshold";
static const char* CONFIG_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCES = "resources";
static const char* CONFIG_RESOURCE_TYPE = "type";
static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id";
static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor";
static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num";
static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory";
static const char* CONFIG_RESOURCE_TEMP_MEMORY = "temp_memory";
static const char* CONFIG_RESOURCE_CONNECTIONS = "connections";
static const char* CONFIG_SPEED_CONNECTIONS = "speed";
static const char* CONFIG_ENDPOINT_CONNECTIONS = "endpoint";
class ServerConfig {
public:
static ServerConfig &GetInstance();
Status LoadConfigFile(const std::string& config_filename);
Status ValidateConfig();
void PrintAll() const;
ConfigNode GetConfig(const std::string& name) const;
ConfigNode& GetConfig(const std::string& name);
private:
Status CheckServerConfig();
Status CheckDBConfig();
Status CheckMetricConfig();
Status CheckCacheConfig();
Status CheckEngineConfig();
Status CheckResourceConfig();
};
}
}
}
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @ 5ffaa4fd
...@@ -15,8 +15,9 @@ ...@@ -15,8 +15,9 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include <string.h>
#include "GrpcRequestTask.h" #include "GrpcRequestTask.h"
#include "../ServerConfig.h"
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "utils/TimeRecorder.h" #include "utils/TimeRecorder.h"
...@@ -28,9 +29,8 @@ ...@@ -28,9 +29,8 @@
#include "scheduler/SchedInst.h" #include "scheduler/SchedInst.h"
//#include <gperftools/profiler.h> //#include <gperftools/profiler.h>
#include "src/server/Server.h" #include "server/Server.h"
#include <string.h>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
......
cpp/src/server/grpc_impl/GrpcServer.cpp
浏览文件 @ 5ffaa4fd
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include "milvus.grpc.pb.h" #include "milvus.grpc.pb.h"
#include "GrpcServer.h" #include "GrpcServer.h"
#include "server/ServerConfig.h" #include "server/Config.h"
#include "server/DBWrapper.h" #include "server/DBWrapper.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "GrpcRequestHandler.h" #include "GrpcRequestHandler.h"
...@@ -73,13 +73,20 @@ GrpcServer::Stop() { ...@@ -73,13 +73,20 @@ GrpcServer::Stop() {
Status Status
GrpcServer::StartService() { GrpcServer::StartService() {
ServerConfig &config = ServerConfig::GetInstance(); Config &config = Config::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER); std::string address, port;
ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); Status s;
std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1");
int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530);
std::string server_address(address + ":" + std::to_string(port)); s = config.GetServerConfigAddress(address);
if (!s.ok()) {
return s;
}
s = config.GetServerConfigPort(port);
if (!s.ok()) {
return s;
}
std::string server_address(address + ":" + port);
::grpc::ServerBuilder builder; ::grpc::ServerBuilder builder;
builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption)); builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption));
......
cpp/src/utils/LogUtil.cpp
浏览文件 @ 5ffaa4fd
...@@ -15,13 +15,11 @@ ...@@ -15,13 +15,11 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "LogUtil.h"
#include "server/ServerConfig.h"
#include <ctype.h> #include <ctype.h>
#include <string> #include <string>
#include <libgen.h> #include <libgen.h>
#include "LogUtil.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
......
cpp/src/utils/SignalUtil.cpp
浏览文件 @ 5ffaa4fd
...@@ -34,7 +34,7 @@ void SignalUtil::HandleSignal(int signum) { ...@@ -34,7 +34,7 @@ void SignalUtil::HandleSignal(int signum) {
case SIGUSR2: { case SIGUSR2: {
SERVER_LOG_INFO << "Server received signal: " << signum; SERVER_LOG_INFO << "Server received signal: " << signum;
server::Server &server = server::Server::Instance(); server::Server &server = server::Server::GetInstance();
server.Stop(); server.Stop();
exit(0); exit(0);
...@@ -43,7 +43,7 @@ void SignalUtil::HandleSignal(int signum) { ...@@ -43,7 +43,7 @@ void SignalUtil::HandleSignal(int signum) {
SERVER_LOG_INFO << "Server received critical signal: " << signum; SERVER_LOG_INFO << "Server received critical signal: " << signum;
SignalUtil::PrintStacktrace(); SignalUtil::PrintStacktrace();
server::Server &server = server::Server::Instance(); server::Server &server = server::Server::GetInstance();
server.Stop(); server.Stop();
exit(1); exit(1);
......
cpp/src/utils/ValidationUtil.cpp
浏览文件 @ 5ffaa4fd
...@@ -206,38 +206,40 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { ...@@ -206,38 +206,40 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) {
} }
Status Status
ValidationUtil::ValidateStringIsNumber(const std::string &string) { ValidationUtil::ValidateStringIsNumber(const std::string &str) {
if (!string.empty() && std::all_of(string.begin(), string.end(), ::isdigit)) { if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
return Status::OK(); return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
} }
else { try {
return Status(SERVER_INVALID_ARGUMENT, "Not a number"); int32_t value = std::stoi(str);
} catch(...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
} }
return Status::OK();
} }
Status Status
ValidationUtil::ValidateStringIsBool(std::string &str) { ValidationUtil::ValidateStringIsBool(const std::string &str) {
std::transform(str.begin(), str.end(), str.begin(), ::tolower); std::string s = str;
if (str == "true" || str == "on" || str == "yes" || str == "1" || std::transform(s.begin(), s.end(), s.begin(), ::tolower);
str == "false" || str == "off" || str == "no" || str == "0" || if (s == "true" || s == "on" || s == "yes" || s == "1" ||
str.empty()) { s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) {
return Status::OK(); return Status::OK();
} }
else { else {
return Status(SERVER_INVALID_ARGUMENT, "Not a boolean: " + str); return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
} }
} }
Status Status
ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) { ValidationUtil::ValidateStringIsFloat(const std::string &str) {
char *end = nullptr; try {
val = std::strtod(str.c_str(), &end); float val = std::stof(str);
if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) { } catch(...) {
return Status::OK(); return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
}
else {
return Status(SERVER_INVALID_ARGUMENT, "Not a double value: " + str);
} }
return Status::OK();
} }
Status Status
......
cpp/src/utils/ValidationUtil.h
浏览文件 @ 5ffaa4fd
...@@ -67,10 +67,10 @@ public: ...@@ -67,10 +67,10 @@ public:
ValidateStringIsNumber(const std::string &str); ValidateStringIsNumber(const std::string &str);
static Status static Status
ValidateStringIsBool(std::string &str); ValidateStringIsBool(const std::string &str);
static Status static Status
ValidateStringIsDouble(const std::string &str, double &val); ValidateStringIsFloat(const std::string &str);
static Status static Status
ValidateDbURI(const std::string &uri); ValidateDbURI(const std::string &uri);
......
cpp/src/wrapper/KnowhereResource.cpp
浏览文件 @ 5ffaa4fd
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
#include "KnowhereResource.h" #include "KnowhereResource.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "server/ServerConfig.h" #include "server/Config.h"
#include <map> #include <map>
...@@ -37,19 +37,24 @@ KnowhereResource::Initialize() { ...@@ -37,19 +37,24 @@ KnowhereResource::Initialize() {
}; };
using GpuResourcesArray = std::map<int64_t , GpuResourceSetting>; using GpuResourcesArray = std::map<int64_t , GpuResourceSetting>;
GpuResourcesArray gpu_resources; GpuResourcesArray gpu_resources;
Status s;
//get build index gpu resource //get build index gpu resource
server::ServerConfig& root_config = server::ServerConfig::GetInstance(); server::Config& config = server::Config::GetInstance();
server::ConfigNode& db_config = root_config.GetConfig(server::CONFIG_DB);
int32_t build_index_gpu;
s = config.GetDBConfigBuildIndexGPU(build_index_gpu);
if (!s.ok()) return s;
int32_t build_index_gpu = db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, 0);
gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting()));
//get search gpu resource //get search gpu resource
server::ConfigNode& res_config = root_config.GetConfig(server::CONFIG_RESOURCE); std::vector<std::string> pool;
auto resources = res_config.GetSequence("resources"); s = config.GetResourceConfigPool(pool);
if (!s.ok()) return s;
std::set<uint64_t> gpu_ids; std::set<uint64_t> gpu_ids;
for (auto &resource : resources) { for (auto &resource : pool) {
if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") {
// invalid // invalid
continue; continue;
......
cpp/unittest/db/engine_test.cpp
浏览文件 @ 5ffaa4fd
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
#include "db/engine/EngineFactory.h" #include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h" #include "db/engine/ExecutionEngineImpl.h"
#include "server/ServerConfig.h"
#include "utils.h" #include "utils.h"
using namespace zilliz::milvus; using namespace zilliz::milvus;
......
cpp/unittest/db/search_test.cpp
浏览文件 @ 5ffaa4fd
...@@ -15,13 +15,12 @@ ...@@ -15,13 +15,12 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "server/ServerConfig.h"
#include "utils/TimeRecorder.h"
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <cmath> #include <cmath>
#include <vector> #include <vector>
#include <src/scheduler/task/SearchTask.h>
#include "scheduler/task/SearchTask.h"
#include "utils/TimeRecorder.h"
using namespace zilliz::milvus; using namespace zilliz::milvus;
......
cpp/unittest/main.cpp
浏览文件 @ 5ffaa4fd
...@@ -15,13 +15,12 @@ ...@@ -15,13 +15,12 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "utils/easylogging++.h"
#include "server/ServerConfig.h"
#include "utils/CommonUtil.h"
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <gmock/gmock.h> #include <gmock/gmock.h>
#include "utils/easylogging++.h"
#include "utils/CommonUtil.h"
INITIALIZE_EASYLOGGINGPP INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus; using namespace zilliz::milvus;
......
cpp/unittest/scheduler/schedinst_test.cpp
浏览文件 @ 5ffaa4fd
...@@ -15,12 +15,10 @@ ...@@ -15,12 +15,10 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "scheduler/SchedInst.h"
#include "server/ServerConfig.h"
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "scheduler/SchedInst.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
......
cpp/unittest/server/cache_test.cpp
浏览文件 @ 5ffaa4fd
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#include "cache/GpuCacheMgr.h" #include "cache/GpuCacheMgr.h"
#include "server/ServerConfig.h" #include "server/ServerConfig.h"
#include "utils/Error.h" #include "utils/Error.h"
#include "src/wrapper/vec_index.h" #include "wrapper/vec_index.h"
using namespace zilliz::milvus; using namespace zilliz::milvus;
......
cpp/unittest/server/config_test.cpp
浏览文件 @ 5ffaa4fd
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
#include <gtest/gtest-death-test.h> #include <gtest/gtest-death-test.h>
#include "config/ConfigMgr.h" #include "config/ConfigMgr.h"
#include "server/ServerConfig.h"
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h" #include "utils/ValidationUtil.h"
......
cpp/unittest/server/rpc_test.cpp
浏览文件 @ 5ffaa4fd
...@@ -29,7 +29,6 @@ ...@@ -29,7 +29,6 @@
#include "grpc/gen-status/status.pb.h" #include "grpc/gen-status/status.pb.h"
#include "server/DBWrapper.h" #include "server/DBWrapper.h"
#include "server/ServerConfig.h"
#include "scheduler/SchedInst.h" #include "scheduler/SchedInst.h"
#include "scheduler/ResourceFactory.h" #include "scheduler/ResourceFactory.h"
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
...@@ -67,7 +66,7 @@ class RpcHandlerTest : public testing::Test { ...@@ -67,7 +66,7 @@ class RpcHandlerTest : public testing::Test {
engine::DBOptions opt; engine::DBOptions opt;
server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB); server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB);
db_config.SetValue(server::CONFIG_DB_URL, "sqlite://:@:/"); db_config.SetValue(server::CONFIG_DB_BACKEND_URL, "sqlite://:@:/");
db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test"); db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test");
db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, ""); db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, "");
db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, ""); db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, "");
...@@ -77,7 +76,7 @@ class RpcHandlerTest : public testing::Test { ...@@ -77,7 +76,7 @@ class RpcHandlerTest : public testing::Test {
cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, ""); cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, "");
server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE); server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE);
engine_config.SetValue(server::CONFIG_OMP_THREAD_NUM, ""); engine_config.SetValue(server::CONFIG_ENGINE_OMP_THREAD_NUM, "");
server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER); server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER);
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或