Fix logger, return systemd service (#1798)

* Fix test 281 under asan

* Cmake fixes

* Revert "Disable usage of systemd (assumed that temporarily) [#CLICKHOUSE-2]"

This reverts commit f0aa21bc.

* Fixing init scripts and logging

* Fix logger

* Fxi

* fix

* fix

* fix

* Update clickhouse-server-base.postinst

* Update clickhouse-server.cron.d

* Update clickhouse-server-base.postinst

* Update clickhouse-server-base.postinst

* fix

* fix

* space

* better message

debian/clickhouse-client.postinst

@@ -2,4 +2,6 @@
 set -e
 
 mkdir -p /etc/clickhouse-client/conf.d
-chown -R clickhouse: /etc/clickhouse-client
+
+# user created by clickhouse-server package
+chown -R clickhouse /etc/clickhouse-client || true

debian/clickhouse-server-base.postinst

@@ -8,8 +8,21 @@ CLICKHOUSE_LOGDIR=/var/log/clickhouse-server
 
 
 if [ "$1" = configure ]; then
-    if [ -x "/etc/init.d/clickhouse-server" ]; then
-        update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $?
+    if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then
+        # if old rc.d service present - remove it
+        if [ -x "/etc/init.d/clickhouse-server" ]; then
+            update-rc.d clickhouse-server remove
+            chmod -x /etc/init.d/clickhouse-server
+            echo "ClickHouse init script has migrated to systemd. Please manually stop old server and restart the service: sudo killall clickhouse-server && sleep 5 && sudo service clickhouse-server restart"
+        fi
+
+        /bin/systemctl daemon-reload
+        /bin/systemctl enable clickhouse-server
+    else
+        # If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server
+        if [ -x "/etc/init.d/clickhouse-server" ]; then
+            update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $?
+        fi
     fi
 
     # Make sure the administrative user exists
@@ -60,7 +73,6 @@ Please fix this and reinstall this package." >&2
         su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}" || chown -R root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}; chmod -R ug+rw ${CLICKHOUSE_LOGDIR}
     fi
 
-
     # Clean old dynamic compilation results
     if [ -d "${CLICKHOUSE_DATADIR}/build" ]; then
         rm -f ${CLICKHOUSE_DATADIR}/build/*.cpp ${CLICKHOUSE_DATADIR}/build/*.so ||:

debian/clickhouse-server.cron.d

-#*/10 * * * * root /etc/init.d/clickhouse-server condstart 1>/dev/null 2>&1
+#*/10 * * * * root (which service && (service clickhouse-server condstart || true)) || /etc/init.d/clickhouse-server condstart 1>/dev/null 2>&1

debian/source/options

 tar-ignore
-tar-ignore="contrib/poco/openssl/*"
 tar-ignore="build"
 tar-ignore="build_*"
+tar-ignore="contrib/poco/openssl/*"
+tar-ignore="contrib/poco/gradle/*"
+tar-ignore="contrib/poco/Data/SQLite/*"
+tar-ignore="contrib/poco/PDF/*"

libs/libdaemon/src/BaseDaemon.cpp

@@ -526,14 +526,12 @@ void BaseDaemon::reloadConfiguration()
       *  instead of using files specified in config.xml.
       * (It's convenient to log in console when you start server without any command line parameters.)
       */
-    std::string log_command_line_option = config().getString("logger.log", "");
     config_path = config().getString("config-file", "config.xml");
     loaded_config = ConfigProcessor(config_path, false, true).loadConfig(/* allow_zk_includes = */ true);
     if (last_configuration != nullptr)
         config().removeConfiguration(last_configuration);
     last_configuration = loaded_config.configuration.duplicate();
     config().add(last_configuration, PRIO_DEFAULT, false);
-    log_to_console = !config().getBool("application.runAsDaemon", false) && log_command_line_option.empty();
 }
 
 
@@ -578,31 +576,31 @@ void BaseDaemon::wakeup()
 
 void BaseDaemon::buildLoggers()
 {
+    bool is_daemon = config().getBool("application.runAsDaemon", false);
+
     /// Change path for logging.
-    if (config().hasProperty("logger.log") && !log_to_console)
+    if (config().hasProperty("logger.log"))
     {
         std::string path = createDirectory(config().getString("logger.log"));
-        if (config().getBool("application.runAsDaemon", false)
+        if (is_daemon
             && chdir(path.c_str()) != 0)
             throw Poco::Exception("Cannot change directory to " + path);
     }
     else
     {
-        if (config().getBool("application.runAsDaemon", false)
+        if (is_daemon
             && chdir("/tmp") != 0)
             throw Poco::Exception("Cannot change directory to /tmp");
     }
 
-    if (config().hasProperty("logger.errorlog") && !log_to_console)
-        createDirectory(config().getString("logger.errorlog"));
+    // Split log and error log.
+    Poco::AutoPtr<SplitterChannel> split = new SplitterChannel;
 
-    if (config().hasProperty("logger.log") && !log_to_console)
+    if (config().hasProperty("logger.log"))
     {
+        createDirectory(config().getString("logger.log"));
         std::cerr << "Should logs to " << config().getString("logger.log") << std::endl;
 
-        // Split log and error log.
-        Poco::AutoPtr<SplitterChannel> split = new SplitterChannel;
-
         // Set up two channel chains.
         Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
         pf->setProperty("times", "local");
@@ -618,61 +616,60 @@ void BaseDaemon::buildLoggers()
         log->setChannel(log_file);
         split->addChannel(log);
         log_file->open();
+    }
 
-        if (config().hasProperty("logger.errorlog"))
-        {
-            std::cerr << "Should error logs to " << config().getString("logger.errorlog") << std::endl;
-            Poco::AutoPtr<Poco::LevelFilterChannel> level = new Poco::LevelFilterChannel;
-            level->setLevel(Message::PRIO_NOTICE);
-            Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
-            pf->setProperty("times", "local");
-            Poco::AutoPtr<FormattingChannel> errorlog = new FormattingChannel(pf);
-            error_log_file = new FileChannel;
-            error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(config().getString("logger.errorlog")).absolute().toString());
-            error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config().getRawString("logger.size", "100M"));
-            error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
-            error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config().getRawString("logger.compress", "true"));
-            error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config().getRawString("logger.count", "1"));
-            error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config().getRawString("logger.flush", "true"));
-            error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config().getRawString("logger.rotateOnOpen", "false"));
-            errorlog->setChannel(error_log_file);
-            level->setChannel(errorlog);
-            split->addChannel(level);
-            errorlog->open();
-        }
-
-        /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code.
-        /// We don't need this configuration parameter.
+    if (config().hasProperty("logger.errorlog"))
+    {
+        createDirectory(config().getString("logger.errorlog"));
+        std::cerr << "Should error logs to " << config().getString("logger.errorlog") << std::endl;
+        Poco::AutoPtr<Poco::LevelFilterChannel> level = new Poco::LevelFilterChannel;
+        level->setLevel(Message::PRIO_NOTICE);
+        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
+        pf->setProperty("times", "local");
+        Poco::AutoPtr<FormattingChannel> errorlog = new FormattingChannel(pf);
+        error_log_file = new FileChannel;
+        error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(config().getString("logger.errorlog")).absolute().toString());
+        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config().getRawString("logger.size", "100M"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
+        error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config().getRawString("logger.compress", "true"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config().getRawString("logger.count", "1"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config().getRawString("logger.flush", "true"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config().getRawString("logger.rotateOnOpen", "false"));
+        errorlog->setChannel(error_log_file);
+        level->setChannel(errorlog);
+        split->addChannel(level);
+        errorlog->open();
+    }
 
-        if (config().getBool("logger.use_syslog", false) || config().getBool("dynamic_layer_selection", false))
-        {
-            Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG);
-            pf->setProperty("times", "local");
-            Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
-            syslog_channel = new Poco::SyslogChannel(commandName(), Poco::SyslogChannel::SYSLOG_CONS | Poco::SyslogChannel::SYSLOG_PID, Poco::SyslogChannel::SYSLOG_DAEMON);
-            log->setChannel(syslog_channel);
-            split->addChannel(log);
-            syslog_channel->open();
-        }
+    /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code.
+    /// We don't need this configuration parameter.
 
-        split->open();
-        logger().close();
-        logger().setChannel(split);
+    if (config().getBool("logger.use_syslog", false) || config().getBool("dynamic_layer_selection", false))
+    {
+        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG);
+        pf->setProperty("times", "local");
+        Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
+        syslog_channel = new Poco::SyslogChannel(commandName(), Poco::SyslogChannel::SYSLOG_CONS | Poco::SyslogChannel::SYSLOG_PID, Poco::SyslogChannel::SYSLOG_DAEMON);
+        log->setChannel(syslog_channel);
+        split->addChannel(log);
+        syslog_channel->open();
     }
-    else
+
+    if (config().getBool("logger.console", false) || (!config().hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO))))
     {
-        // Print to the terminal.
         Poco::AutoPtr<ConsoleChannel> file = new ConsoleChannel;
         Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
         pf->setProperty("times", "local");
         Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
         log->setChannel(file);
-
-        logger().close();
-        logger().setChannel(log);
         logger().warning("Logging to console");
+        split->addChannel(log);
     }
 
+    split->open();
+    logger().close();
+    logger().setChannel(split);
+
     // Global logging level (it can be overridden for specific loggers).
     logger().setLevel(config().getString("logger.level", "trace"));