support log to stdout (#3993)

* support log to stdout
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* justfy set config
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>
Co-authored-by: Nyangxuan <xuan.yang@zilliz.com>

CHANGELOG.md

@@ -67,6 +67,7 @@ Please mark all changes in change log and use the issue from GitHub
 -   \#3502 Normalize http method in web sever
 -   \#3732 Add new operation supporting multi-segments
 -   \#3828 Limit the number of insert requests processing concurrently.
+-   \#3977 Support logging to stdout.
 
 ## Improvement
 -   \#2543 Remove secondary_path related code

core/conf/milvus.template

@@ -158,6 +158,8 @@ logs:
   path: @MILVUS_DB_PATH@/logs
   max_log_file_size: 1024MB
   log_rotate_num: 0
+  log_to_stdout: false
+  log_to_file: true
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Metric Config        | Description                                                | Type       | Default         |

core/src/config/ConfigInit.cpp

@@ -147,6 +147,8 @@ InitConfig() {
                                                     &config.logs.max_log_file_size.value, 1024 * MB)},
         {"logs.log_rotate_num",
          CreateIntegerConfig("logs.log_rotate_num", 0, 1024, &config.logs.log_rotate_num.value, 0)},
+        {"logs.log_to_stdout", CreateBoolConfig("logs.log_to_stdout", &config.logs.log_to_stdout.value, false)},
+        {"logs.log_to_file", CreateBoolConfig("logs.log_to_file", &config.logs.log_to_file.value, true)},
 
         /* metric */
         {"metric.enable", CreateBoolConfig("metric.enable", &config.metric.enable.value, false)},
@@ -339,12 +341,18 @@ gpu:
 # log_rotate_num       | The maximum number of log files that Milvus keeps for each | Integer    | 0               |
 #                      | logging level, num range [0, 1024], 0 means unlimited.     |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
+# log_to_stdout        | Whether to write logs to standard output in Milvus.        | Boolean    | false           |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# log_to_file          | Whether to write logs to files in Milvus                   | Boolean    | true            |
+#----------------------+------------------------------------------------------------+------------+-----------------+
 logs:
   level: @logs.level@
   trace.enable: @logs.trace.enable@
   path: @logs.path@
   max_log_file_size: @logs.max_log_file_size@
   log_rotate_num: @logs.log_rotate_num@
+  log_to_stdout: @logs.log_to_stdout@
+  log_to_file: @logs.log_to_file@
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Metric Config        | Description                                                | Type       | Default         |

core/src/config/ServerConfig.h

@@ -151,6 +151,8 @@ struct ServerConfig {
         String path{"unknown"};
         Integer max_log_file_size{0};
         Integer log_rotate_num{0};
+        Bool log_to_stdout{false};
+        Bool log_to_file{true};
     } logs;
 
     struct System {

core/src/log/LogMgr.cpp

@@ -37,12 +37,16 @@ bool LogMgr::enable_log_delete = false;
 
 Status
 LogMgr::InitLog(bool trace_enable, const std::string& level, const std::string& logs_path, int64_t max_log_file_size,
-                int64_t log_rotate_num) {
+                int64_t log_rotate_num, bool log_to_stdout, bool log_to_file) {
     try {
         auto enables = parse_level(level);
         enables["trace"] = trace_enable;
         LogMgr log_mgr(logs_path);
-        log_mgr.Default().Level(enables).To().Rotate(max_log_file_size, log_rotate_num).Setup();
+        log_mgr.Default()
+            .Level(enables, log_to_file)
+            .To(log_to_stdout, log_to_file)
+            .Rotate(max_log_file_size, log_rotate_num)
+            .Setup();
     } catch (std::exception& ex) {
         return Status(SERVER_UNEXPECTED_ERROR, ex.what());
     }
@@ -121,7 +125,7 @@ LogMgr::Default() {
 }
 
 LogMgr&
-LogMgr::Level(std::unordered_map<std::string, bool>& enables) {
+LogMgr::Level(std::unordered_map<std::string, bool>& enables, bool log_to_file) {
     std::string logs_reg_path = logs_path_.rfind('/') == logs_path_.length() - 1 ? logs_path_ : logs_path_ + "/";
 
     /* If want to output all logs to one file, uncomment this line below and comment other set_level lines */
@@ -129,35 +133,35 @@ LogMgr::Level(std::unordered_map<std::string, bool>& enables) {
 
     fiu_do_on("LogMgr.Level.trace_enable_to_false", enables["trace"] = false);
     set_level(el_config_, el::Level::Trace, enables["trace"],
-              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log");
+              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log", log_to_file);
 
     fiu_do_on("LogMgr.Level.info_enable_to_false", enables["info"] = false);
-    set_level(el_config_, el::Level::Info, enables["info"],
-              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log");
+    set_level(el_config_, el::Level::Info, enables["info"], logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log",
+              log_to_file);
 
     fiu_do_on("LogMgr.Level.debug_enable_to_false", enables["debug"] = false);
     set_level(el_config_, el::Level::Debug, enables["debug"],
-              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log");
+              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log", log_to_file);
 
     fiu_do_on("LogMgr.Level.warning_enable_to_false", enables["warning"] = false);
     set_level(el_config_, el::Level::Warning, enables["warning"],
-              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log");
+              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log", log_to_file);
 
     fiu_do_on("LogMgr.Level.error_enable_to_false", enables["error"] = false);
     set_level(el_config_, el::Level::Error, enables["error"],
-              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log");
+              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log", log_to_file);
 
     fiu_do_on("LogMgr.Level.fatal_enable_to_false", enables["fatal"] = false);
     set_level(el_config_, el::Level::Fatal, enables["fatal"],
-              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log");
+              logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log", log_to_file);
 
     return *this;
 }
 
 LogMgr&
-LogMgr::To() {
-    el_config_.setGlobally(el::ConfigurationType::ToFile, "true");
-    el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, "false");
+LogMgr::To(bool log_to_stdout, bool log_to_file) {
+    el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, (log_to_stdout ? "true" : "false"));
+    el_config_.setGlobally(el::ConfigurationType::ToFile, (log_to_file ? "true" : "false"));
 
     return *this;
 }
@@ -240,8 +244,12 @@ LogMgr::parse_level(const std::string& level) {
 }
 
 void
-LogMgr::set_level(el::Configurations& default_conf, el::Level level, bool enable, const std::string& log_path) {
-    default_conf.set(level, el::ConfigurationType::Filename, log_path.c_str());
+LogMgr::set_level(el::Configurations& default_conf, el::Level level, bool enable, const std::string& log_path,
+                  bool log_to_file) {
+    if (log_to_file) {
+        default_conf.set(level, el::ConfigurationType::Filename, log_path.c_str());
+    }
+
     if (enable) {
         default_conf.set(level, el::ConfigurationType::Enabled, "true");
     } else {

core/src/log/LogMgr.h

@@ -25,7 +25,7 @@ class LogMgr {
  public:
     static Status
     InitLog(bool trace_enable, const std::string& level, const std::string& logs_path, int64_t max_log_file_size,
-            int64_t delete_exceeds);
+            int64_t delete_exceeds, bool log_to_stdout, bool log_to_file);
 
     static void
     RolloutHandler(const char* filename, std::size_t size, el::Level level);
@@ -38,10 +38,10 @@ class LogMgr {
 
     /* Non-const for fiu to injecting error */
     LogMgr&
-    Level(std::unordered_map<std::string, bool>& enables);
+    Level(std::unordered_map<std::string, bool>& enables, bool log_to_file);
 
     LogMgr&
-    To();
+    To(bool log_to_stdout, bool log_to_file);
 
     LogMgr&
     Rotate(int64_t max_log_file_size, int64_t log_rotate_num);
@@ -57,7 +57,8 @@ class LogMgr {
     parse_level(const std::string& level);
 
     static void
-    set_level(el::Configurations& default_conf, el::Level level, bool enable, const std::string& log_path);
+    set_level(el::Configurations& default_conf, el::Level level, bool enable, const std::string& log_path,
+              bool log_to_file);
 
  private:
     el::Configurations el_config_;

core/src/server/Server.cpp

@@ -160,7 +160,8 @@ Server::Start() {
 
         /* log path is defined in Config file, so InitLog must be called after LoadConfig */
         STATUS_CHECK(LogMgr::InitLog(config.logs.trace.enable(), config.logs.level(), config.logs.path(),
-                                     config.logs.max_log_file_size(), config.logs.log_rotate_num()));
+                                     config.logs.max_log_file_size(), config.logs.log_rotate_num(),
+                                     config.logs.log_to_stdout(), config.logs.log_to_file()));
 
         auto wal_path = config.wal.enable() ? config.wal.path() : "";
         STATUS_CHECK(Directory::Initialize(config.storage.path(), wal_path, config.logs.path()));