未验证 提交 3be8aad1 编写于 作者: W Wang XiangYu 提交者: GitHub

Merge config file (#2168)

* Using el::Configurations Class init easylog
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* add logs config constant
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* add config check function
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* add config get function
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* logs config set function
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update InitLog function
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* fix clang-format
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update server_config.template
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update changelog
Signed-off-by: Nwxyu <xy.wang@zilliz.com>
上级 0ba67710
......@@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub
- \#221 Refactor LOG macro
- \#2039 Support Milvus run on SSE CPUs
- \#2149 Merge server_cpu_config.template and server_gpu_config.template
- \#2167 Merge config file
## Task
......
......@@ -186,3 +186,13 @@ wal_config:
recovery_error_ignore: true
buffer_size: 256
wal_path: @MILVUS_DB_PATH@/wal
logs:
trace.enable: true
debug.enable: true
info.enable: true
warning.enable: true
error.enable: true
fatal.enable: true
path: @MILVUS_DB_PATH@/logs
......@@ -153,6 +153,23 @@ const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 64;
const char* CONFIG_WAL_WAL_PATH = "wal_path";
const char* CONFIG_WAL_WAL_PATH_DEFAULT = "/tmp/milvus/wal";
/* logs config */
const char* CONFIG_LOGS = "logs";
const char* CONFIG_LOGS_TRACE_ENABLE = "trace.enable";
const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_DEBUG_ENABLE = "debug.enable";
const char* CONFIG_LOGS_DEBUG_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_INFO_ENABLE = "info.enable";
const char* CONFIG_LOGS_INFO_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_WARNING_ENABLE = "warning.enable";
const char* CONFIG_LOGS_WARNING_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_ERROR_ENABLE = "error.enable";
const char* CONFIG_LOGS_ERROR_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_FATAL_ENABLE = "fatal.enable";
const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT = "true";
const char* CONFIG_LOGS_PATH = "path";
const char* CONFIG_LOGS_PATH_DEFAULT = "/tmp/milvus/logs";
constexpr int64_t GB = 1UL << 30;
constexpr int32_t PORT_NUMBER_MIN = 1024;
constexpr int32_t PORT_NUMBER_MAX = 65535;
......@@ -367,6 +384,28 @@ Config::ValidateConfig() {
std::string wal_path;
CONFIG_CHECK(GetWalConfigWalPath(wal_path));
/* logs config */
bool trace_enable;
CONFIG_CHECK(GetLogsTraceEnable(trace_enable));
bool debug_enable;
CONFIG_CHECK(GetLogsDebugEnable(trace_enable));
bool info_enable;
CONFIG_CHECK(GetLogsInfoEnable(trace_enable));
bool warning_enable;
CONFIG_CHECK(GetLogsWarningEnable(trace_enable));
bool error_enable;
CONFIG_CHECK(GetLogsErrorEnable(trace_enable));
bool fatal_enable;
CONFIG_CHECK(GetLogsFatalEnable(trace_enable));
std::string logs_path;
CONFIG_CHECK(GetLogsPath(logs_path));
return Status::OK();
}
......@@ -418,6 +457,16 @@ Config::ResetDefaultConfig() {
CONFIG_CHECK(SetWalConfigRecoveryErrorIgnore(CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT));
CONFIG_CHECK(SetWalConfigBufferSize(CONFIG_WAL_BUFFER_SIZE_DEFAULT));
CONFIG_CHECK(SetWalConfigWalPath(CONFIG_WAL_WAL_PATH_DEFAULT));
/* logs config */
CONFIG_CHECK(SetLogsTraceEnable(CONFIG_LOGS_TRACE_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsDebugEnable(CONFIG_LOGS_DEBUG_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsInfoEnable(CONFIG_LOGS_INFO_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsWarningEnable(CONFIG_LOGS_WARNING_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsErrorEnable(CONFIG_LOGS_ERROR_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsFatalEnable(CONFIG_LOGS_FATAL_ENABLE_DEFAULT));
CONFIG_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT));
#ifdef MILVUS_GPU_VERSION
CONFIG_CHECK(SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT));
#endif
......@@ -1492,6 +1541,89 @@ Config::CheckWalConfigWalPath(const std::string& value) {
return ValidationUtil::ValidateStoragePath(value);
}
/* logs config */
Status
Config::CheckLogsTraceEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_trace_enable_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.trace.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsDebugEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_debug_enable_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.debug.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsInfoEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_info_enable_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.info.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsWarningEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_warning_enable_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.warning.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsErrorEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_error_enable_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.error.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsFatalEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_fatal_enable_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid logs config: " + value + ". Possible reason: logs.fatal.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckLogsPath(const std::string& value) {
fiu_return_on("check_logs_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "logs.path is empty!");
}
return ValidationUtil::ValidateStoragePath(value);
}
////////////////////////////////////////////////////////////////////////////////
ConfigNode&
Config::GetConfigRoot() {
......@@ -1958,6 +2090,62 @@ Config::GetWalConfigWalPath(std::string& wal_path) {
return Status::OK();
}
/* logs config */
Status
Config::GetLogsTraceEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_TRACE_ENABLE, CONFIG_LOGS_TRACE_ENABLE_DEFAULT);
CONFIG_CHECK(CheckLogsTraceEnable(str));
CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsDebugEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_DEBUG_ENABLE, CONFIG_LOGS_DEBUG_ENABLE_DEFAULT);
CONFIG_CHECK(CheckLogsDebugEnable(str));
CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsInfoEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_INFO_ENABLE, CONFIG_LOGS_INFO_ENABLE_DEFAULT);
CONFIG_CHECK(CheckLogsInfoEnable(str));
CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsWarningEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_WARNING_ENABLE, CONFIG_LOGS_WARNING_ENABLE_DEFAULT);
CONFIG_CHECK(CheckLogsWarningEnable(str));
CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsErrorEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_ERROR_ENABLE, CONFIG_LOGS_ERROR_ENABLE_DEFAULT);
CONFIG_CHECK(CheckLogsErrorEnable(str));
CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsFatalEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_FATAL_ENABLE, CONFIG_LOGS_FATAL_ENABLE_DEFAULT);
CONFIG_CHECK(CheckLogsFatalEnable(str));
CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetLogsPath(std::string& value) {
value = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_PATH, CONFIG_LOGS_PATH_DEFAULT);
CONFIG_CHECK(CheckWalConfigWalPath(value));
return Status::OK();
}
Status
Config::GetServerRestartRequired(bool& required) {
required = restart_required_;
......@@ -2182,6 +2370,49 @@ Config::SetWalConfigWalPath(const std::string& value) {
return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_WAL_PATH, value);
}
/* logs config */
Status
Config::SetLogsTraceEnable(const std::string& value) {
CONFIG_CHECK(CheckLogsTraceEnable(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_TRACE_ENABLE, value);
}
Status
Config::SetLogsDebugEnable(const std::string& value) {
CONFIG_CHECK(CheckLogsDebugEnable(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_DEBUG_ENABLE, value);
}
Status
Config::SetLogsInfoEnable(const std::string& value) {
CONFIG_CHECK(CheckLogsInfoEnable(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_INFO_ENABLE, value);
}
Status
Config::SetLogsWarningEnable(const std::string& value) {
CONFIG_CHECK(CheckLogsWarningEnable(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_WARNING_ENABLE, value);
}
Status
Config::SetLogsErrorEnable(const std::string& value) {
CONFIG_CHECK(CheckLogsErrorEnable(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_ERROR_ENABLE, value);
}
Status
Config::SetLogsFatalEnable(const std::string& value) {
CONFIG_CHECK(CheckLogsFatalEnable(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_FATAL_ENABLE, value);
}
Status
Config::SetLogsPath(const std::string& value) {
CONFIG_CHECK(CheckLogsPath(value));
return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_PATH, value);
}
#ifdef MILVUS_GPU_VERSION
Status
Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
......
......@@ -149,6 +149,22 @@ extern const int64_t CONFIG_WAL_BUFFER_SIZE_MIN;
extern const char* CONFIG_WAL_WAL_PATH;
extern const char* CONFIG_WAL_WAL_PATH_DEFAULT;
/* logs config */
extern const char* CONFIG_LOGS;
extern const char* CONFIG_LOGS_TRACE_ENABLE;
extern const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_DEBUG_ENABLE;
extern const char* CONFIG_LOGS_DEBUG_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_INFO_ENABLE;
extern const char* CONFIG_LOGS_INFO_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_WARNING_ENABLE;
extern const char* CONFIG_LOGS_WARNING_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_ERROR_ENABLE;
extern const char* CONFIG_LOGS_ERROR_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_FATAL_ENABLE;
extern const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_PATH;
class Config {
private:
Config();
......@@ -301,6 +317,22 @@ class Config {
Status
CheckWalConfigWalPath(const std::string& value);
/* logs config */
Status
CheckLogsTraceEnable(const std::string& value);
Status
CheckLogsDebugEnable(const std::string& value);
Status
CheckLogsInfoEnable(const std::string& value);
Status
CheckLogsWarningEnable(const std::string& value);
Status
CheckLogsErrorEnable(const std::string& value);
Status
CheckLogsFatalEnable(const std::string& value);
Status
CheckLogsPath(const std::string& value);
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
std::string
......@@ -414,6 +446,22 @@ class Config {
Status
GetWalConfigWalPath(std::string& value);
/* logs config */
Status
GetLogsTraceEnable(bool& value);
Status
GetLogsDebugEnable(bool& value);
Status
GetLogsInfoEnable(bool& value);
Status
GetLogsWarningEnable(bool& value);
Status
GetLogsErrorEnable(bool& value);
Status
GetLogsFatalEnable(bool& value);
Status
GetLogsPath(std::string& value);
Status
GetServerRestartRequired(bool& required);
......@@ -502,6 +550,22 @@ class Config {
Status
SetWalConfigWalPath(const std::string& value);
/* logs config */
Status
SetLogsTraceEnable(const std::string& value);
Status
SetLogsDebugEnable(const std::string& value);
Status
SetLogsInfoEnable(const std::string& value);
Status
SetLogsWarningEnable(const std::string& value);
Status
SetLogsErrorEnable(const std::string& value);
Status
SetLogsFatalEnable(const std::string& value);
Status
SetLogsPath(const std::string& value);
#ifdef MILVUS_GPU_VERSION
Status
SetEngineConfigGpuSearchThreshold(const std::string& value);
......
......@@ -190,7 +190,44 @@ Server::Start() {
}
tzset();
InitLog(log_config_file_);
{
bool trace_enable = false;
bool debug_enable = false;
bool info_enable = false;
bool warning_enable = false;
bool error_enable = false;
bool fatal_enable = false;
std::string logs_path;
s = config.GetLogsTraceEnable(trace_enable);
if (!s.ok()) {
return s;
}
s = config.GetLogsDebugEnable(debug_enable);
if (!s.ok()) {
return s;
}
s = config.GetLogsInfoEnable(info_enable);
if (!s.ok()) {
return s;
}
s = config.GetLogsWarningEnable(warning_enable);
if (!s.ok()) {
return s;
}
s = config.GetLogsErrorEnable(error_enable);
if (!s.ok()) {
return s;
}
s = config.GetLogsFatalEnable(fatal_enable);
if (!s.ok()) {
return s;
}
s = config.GetLogsPath(logs_path);
if (!s.ok()) {
return s;
}
InitLog(trace_enable, debug_enable, info_enable, warning_enable, error_enable, fatal_enable, logs_path);
}
std::string deploy_mode;
s = config.GetServerConfigDeployMode(deploy_mode);
......
......@@ -83,9 +83,70 @@ RolloutHandler(const char* filename, std::size_t size, el::Level level) {
}
Status
InitLog(const std::string& log_config_file) {
el::Configurations conf(log_config_file);
el::Loggers::reconfigureAllLoggers(conf);
InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable,
bool fatal_enable, const std::string& logs_path) {
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
defaultConf.setGlobally(el::ConfigurationType::ToFile, "true");
defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, "false");
defaultConf.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, "209715200");
std::string global_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log";
defaultConf.set(el::Level::Global, el::ConfigurationType::Filename, global_log_path.c_str());
defaultConf.set(el::Level::Global, el::ConfigurationType::Enabled, "true");
std::string info_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log";
defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str());
if (info_enable) {
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
}
std::string debug_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log";
defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str());
if (debug_enable) {
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
}
std::string warning_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log";
defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str());
if (warning_enable) {
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
}
std::string trace_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log";
defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str());
if (trace_enable) {
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
}
std::string error_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log";
defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str());
if (error_enable) {
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
}
std::string fatal_log_path = logs_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log";
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str());
if (fatal_enable) {
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
}
el::Loggers::reconfigureLogger("default", defaultConf);
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(RolloutHandler);
......
......@@ -21,7 +21,8 @@ namespace milvus {
namespace server {
Status
InitLog(const std::string& log_config_file);
InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_enable, bool error_enable,
bool fatal_enable, const std::string& logs_path);
void
RolloutHandler(const char* filename, std::size_t size, el::Level level);
......
......@@ -246,7 +246,7 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) {
}
TEST(UtilTest, LOG_TEST) {
auto status = milvus::server::InitLog(LOG_FILE_PATH);
auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util");
ASSERT_TRUE(status.ok());
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册