Add text_log.level to limit entries that goes to system.text_log table

v2: use INT_MAX as default (since 0 is none)

dbms/programs/server/Server.cpp

@@ -881,7 +881,11 @@ int Server::main(const std::vector<std::string> & /*args*/)
         for (auto & server : servers)
             server->start();
 
-        setTextLog(global_context->getTextLog());
+        {
+            String level_str = config().getString("text_log.level", "");
+            int level = level_str.empty() ? INT_MAX : Poco::Logger::parseLevel(level_str);
+            setTextLog(global_context->getTextLog(), level);
+        }
         buildLoggers(config(), logger());
 
         main_config_reloader->start();

dbms/programs/server/config.xml

@@ -389,10 +389,12 @@
 
     <!-- Uncomment to write text log into table.
          Text log contains all information from usual server log but stores it in structured and efficient way.
+         The level of the messages that goes to the table can be limited (<level>), if not specified all messages will go to the table.
     <text_log>
         <database>system</database>
         <table>text_log</table>
         <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <level></level>
     </text_log>
     -->
 

libs/libloggers/loggers/Loggers.cpp

@@ -27,16 +27,17 @@ static std::string createDirectory(const std::string & file)
     return path.toString();
 };
 
-void Loggers::setTextLog(std::shared_ptr<DB::TextLog> log)
+void Loggers::setTextLog(std::shared_ptr<DB::TextLog> log, int max_priority)
 {
     text_log = log;
+    text_log_max_priority = max_priority;
 }
 
 void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name)
 {
     if (split)
         if (auto log = text_log.lock())
-            split->addTextLog(log);
+            split->addTextLog(log, text_log_max_priority);
 
     auto current_logger = config.getString("logger", "");
     if (config_logger == current_logger)

libs/libloggers/loggers/Loggers.h

@@ -24,7 +24,7 @@ public:
         return layer; /// layer setted in inheritor class BaseDaemonApplication.
     }
 
-    void setTextLog(std::shared_ptr<DB::TextLog> log);
+    void setTextLog(std::shared_ptr<DB::TextLog> log, int max_priority);
 
 protected:
     std::optional<size_t> layer;
@@ -38,5 +38,7 @@ private:
     std::string config_logger;
 
     std::weak_ptr<DB::TextLog> text_log;
+    int text_log_max_priority = -1;
+
     Poco::AutoPtr<DB::OwnSplitChannel> split;
 };

libs/libloggers/loggers/OwnSplitChannel.cpp

@@ -70,34 +70,37 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
     }
 
 
-    /// Also log to system.text_log table
-    TextLogElement elem;
+    /// Also log to system.text_log table, if message is not too noisy
+    if (text_log_max_priority && msg.getPriority() <= text_log_max_priority)
+    {
+        TextLogElement elem;
 
-    elem.event_time = msg_ext.time_seconds;
-    elem.microseconds = msg_ext.time_microseconds;
+        elem.event_time = msg_ext.time_seconds;
+        elem.microseconds = msg_ext.time_microseconds;
 
-    elem.thread_name = getThreadName();
-    elem.thread_number = msg_ext.thread_number;
+        elem.thread_name = getThreadName();
+        elem.thread_number = msg_ext.thread_number;
 
-    if (CurrentThread::isInitialized())
-        elem.os_thread_id = CurrentThread::get().os_thread_id;
-    else
-        elem.os_thread_id = 0;
+        if (CurrentThread::isInitialized())
+            elem.os_thread_id = CurrentThread::get().os_thread_id;
+        else
+            elem.os_thread_id = 0;
 
-    elem.query_id = msg_ext.query_id;
+        elem.query_id = msg_ext.query_id;
 
-    elem.message = msg.getText();
-    elem.logger_name = msg.getSource();
-    elem.level = msg.getPriority();
+        elem.message = msg.getText();
+        elem.logger_name = msg.getSource();
+        elem.level = msg.getPriority();
 
-    if (msg.getSourceFile() != nullptr)
-        elem.source_file = msg.getSourceFile();
+        if (msg.getSourceFile() != nullptr)
+            elem.source_file = msg.getSourceFile();
 
-    elem.source_line = msg.getSourceLine();
+        elem.source_line = msg.getSourceLine();
 
-    std::lock_guard<std::mutex> lock(text_log_mutex);
-    if (auto log = text_log.lock())
-        log->add(elem);
+        std::lock_guard<std::mutex> lock(text_log_mutex);
+        if (auto log = text_log.lock())
+            log->add(elem);
+    }
 }
 
 
@@ -106,10 +109,11 @@ void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel)
     channels.emplace_back(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get()));
 }
 
-void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log)
+void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority)
 {
     std::lock_guard<std::mutex> lock(text_log_mutex);
     text_log = log;
+    text_log_max_priority = max_priority;
 }
 
 }

libs/libloggers/loggers/OwnSplitChannel.h

@@ -20,7 +20,7 @@ public:
     /// Adds a child channel
     void addChannel(Poco::AutoPtr<Poco::Channel> channel);
 
-    void addTextLog(std::shared_ptr<DB::TextLog> log);
+    void addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority);
 
 private:
     void logSplit(const Poco::Message & msg);
@@ -33,6 +33,7 @@ private:
     std::mutex text_log_mutex;
 
     std::weak_ptr<DB::TextLog> text_log;
+    int text_log_max_priority = -1;
 };
 
 }