diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index f960e82a66a163fa2f68ab41e73db8b2103284db..7725f5f7f854e4abecc5f2173d8f850366641720 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -87,7 +87,7 @@ if (ARCH_FREEBSD) endif () add_subdirectory(src/Common/ZooKeeper) -add_subdirectory(src/Common/ConfigProcessor) +add_subdirectory(src/Common/Config) if (MAKE_STATIC_LIBRARIES) add_library(dbms ${dbms_headers} ${dbms_sources}) @@ -141,7 +141,7 @@ target_link_libraries (clickhouse_common_io target_link_libraries (dbms clickhouse_parsers - clickhouse_common_configprocessor + clickhouse_common_config clickhouse_common_io ${MYSQLXX_LIBRARY} ${FARMHASH_LIBRARIES} diff --git a/dbms/src/Common/Config/CMakeLists.txt b/dbms/src/Common/Config/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..ae7d623d4576576c7e1d87f793187c47c6835af9 --- /dev/null +++ b/dbms/src/Common/Config/CMakeLists.txt @@ -0,0 +1,9 @@ + +include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) + +add_headers_and_sources(clickhouse_common_config .) + +add_library(clickhouse_common_config ${SPLIT_SHARED} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources}) + +target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils) +target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR}) diff --git a/dbms/src/Common/ConfigProcessor/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp similarity index 100% rename from dbms/src/Common/ConfigProcessor/ConfigProcessor.cpp rename to dbms/src/Common/Config/ConfigProcessor.cpp diff --git a/dbms/src/Common/ConfigProcessor/ConfigProcessor.h b/dbms/src/Common/Config/ConfigProcessor.h similarity index 100% rename from dbms/src/Common/ConfigProcessor/ConfigProcessor.h rename to dbms/src/Common/Config/ConfigProcessor.h diff --git a/dbms/src/Server/ConfigReloader.cpp b/dbms/src/Common/Config/ConfigReloader.cpp similarity index 98% rename from dbms/src/Server/ConfigReloader.cpp rename to dbms/src/Common/Config/ConfigReloader.cpp index e7d6224ea01a3810b0fc7fd461f70ce6d0cd412b..53a5aa61cfa6e016da659a23e616975551bec10a 100644 --- a/dbms/src/Server/ConfigReloader.cpp +++ b/dbms/src/Common/Config/ConfigReloader.cpp @@ -2,12 +2,9 @@ #include #include - #include - -#include #include -#include +#include "ConfigProcessor.h" namespace DB diff --git a/dbms/src/Server/ConfigReloader.h b/dbms/src/Common/Config/ConfigReloader.h similarity index 97% rename from dbms/src/Server/ConfigReloader.h rename to dbms/src/Common/Config/ConfigReloader.h index 02301346e13c4199e23af83f335804a3e8578935..2dcbea7a8bc587bebc8b408ff9c9c25906a2fb39 100644 --- a/dbms/src/Server/ConfigReloader.h +++ b/dbms/src/Common/Config/ConfigReloader.h @@ -1,9 +1,8 @@ #pragma once -#include +#include "ConfigProcessor.h" #include #include - #include #include #include diff --git a/dbms/src/Common/ConfigProcessor/CMakeLists.txt b/dbms/src/Common/ConfigProcessor/CMakeLists.txt deleted file mode 100644 index 923955dc8c9c952ffc04d895639016bdac834cc3..0000000000000000000000000000000000000000 --- a/dbms/src/Common/ConfigProcessor/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) - -add_headers_and_sources(clickhouse_common_configprocessor .) - -add_library(clickhouse_common_configprocessor ${SPLIT_SHARED} ${clickhouse_common_configprocessor_headers} ${clickhouse_common_configprocessor_sources}) - -target_link_libraries (clickhouse_common_configprocessor clickhouse_common_zookeeper string_utils) -target_include_directories (clickhouse_common_configprocessor PRIVATE ${DBMS_INCLUDE_DIR}) diff --git a/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt b/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt index ed55b3482056546d68955c42ff314b576d7a1d70..a5a035e558bacaafa4a1ba09e376dc7c9d8cd1e9 100644 --- a/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt +++ b/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt @@ -14,4 +14,4 @@ add_executable(zkutil_zookeeper_holder zkutil_zookeeper_holder.cpp) target_link_libraries(zkutil_zookeeper_holder clickhouse_common_zookeeper) add_executable (zk_many_watches_reconnect zk_many_watches_reconnect.cpp) -target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_configprocessor) +target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_config) diff --git a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp index 62169096a2dbf23a4c2757297f851b5e070f8fc2..b1d1d5d31016c5d026640619bf1b2367dbf248a7 100644 --- a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp +++ b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d66265728d21ea11ae6ce9d27d6b444dabee4a17..bbb2bc5e28a84490bc9e9078484cfec042142af3 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -45,7 +45,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp b/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp index a1a6f270ef7299540730ec21f4c78ae9031c3f6c..bfc7965a5ad629a2dc4bdc25b7471dc1dcd960f9 100644 --- a/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp +++ b/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp index 84f09150cffb3845e4ac001428f49d5df4bb1890..5da4ed2613436880cb9e2821f39dcad32897c41d 100644 --- a/dbms/src/Interpreters/tests/users.cpp +++ b/dbms/src/Interpreters/tests/users.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index adfd3309f064690913e94c5ca11fe1111b131353..d9f2a3412378b308b6fd0399664b2d0719ddb24a 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -15,7 +15,6 @@ option (ENABLE_CLICKHOUSE_COPIER "Enable copier" ${ENABLE_CLICKHOUSE_COPIER}) configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) add_library (clickhouse-server-lib - ConfigReloader.cpp HTTPHandler.cpp InterserverIOHTTPHandler.cpp MetricsTransmitter.cpp @@ -35,7 +34,7 @@ add_library (clickhouse-local-lib LocalServer.cpp) target_link_libraries (clickhouse-local-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions) add_library (clickhouse-extract-from-config-lib ${SPLIT_SHARED} ExtractFromConfig.cpp) -target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_configprocessor clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) add_library (clickhouse-client-lib Client.cpp) target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) diff --git a/dbms/src/Server/ExtractFromConfig.cpp b/dbms/src/Server/ExtractFromConfig.cpp index 961f9e31463c5da589fa80c781f58192c35fdd5d..7f0122256dd7a869f46400bfc87e1dfbf21bdc22 100644 --- a/dbms/src/Server/ExtractFromConfig.cpp +++ b/dbms/src/Server/ExtractFromConfig.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include diff --git a/dbms/src/Server/LocalServer.cpp b/dbms/src/Server/LocalServer.cpp index 138f49461b092cc1d4b8396a7f47f1055042ed0b..ed25725f5d244c5df5f615474f4f0b754b832e3f 100644 --- a/dbms/src/Server/LocalServer.cpp +++ b/dbms/src/Server/LocalServer.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index c31f054aaacd31f860c40bb7c0797247a12e77c5..44d4bb602c5755e9d09762aaa1959c4f7c234c58 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -30,7 +30,7 @@ #include #include #include -#include "ConfigReloader.h" +#include #include "HTTPHandlerFactory.h" #include "MetricsTransmitter.h" #include "StatusFile.h" @@ -228,7 +228,11 @@ int Server::main(const std::vector & /*args*/) auto main_config_reloader = std::make_unique(config_path, include_from_path, std::move(main_config_zk_node_cache), - [&](ConfigurationPtr config) { global_context->setClustersConfig(config); }, + [&](ConfigurationPtr config) + { + buildLoggers(*config); + global_context->setClustersConfig(config); + }, /* already_loaded = */ true); /// Initialize users config reloader. diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml index e0b4ea563acff96115cbee371a5388586bc2a4c2..ac871674e1e6ced07ccb37cc8b543637098d795d 100644 --- a/dbms/src/Server/config.xml +++ b/dbms/src/Server/config.xml @@ -1,6 +1,7 @@ + trace /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log diff --git a/libs/libdaemon/CMakeLists.txt b/libs/libdaemon/CMakeLists.txt index 292903b4a47438fa75899c1ee41101500ff064eb..6f31c4e5b385da6af0eeaabb917f03f495731cd3 100644 --- a/libs/libdaemon/CMakeLists.txt +++ b/libs/libdaemon/CMakeLists.txt @@ -17,4 +17,4 @@ endif () target_include_directories (daemon PUBLIC include) target_include_directories (daemon PRIVATE ${ClickHouse_SOURCE_DIR}/libs/libpocoext/include) -target_link_libraries (daemon clickhouse_common_io clickhouse_common_configprocessor ${EXECINFO_LIBRARY}) +target_link_libraries (daemon clickhouse_common_io clickhouse_common_config ${EXECINFO_LIBRARY}) diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h index c59ba187277db4ac739e987289338962f05b190e..20c38469d1600c09e8a1f0f7d3fc494804cbd649 100644 --- a/libs/libdaemon/include/daemon/BaseDaemon.h +++ b/libs/libdaemon/include/daemon/BaseDaemon.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include namespace Poco { class TaskManager; } @@ -67,7 +67,7 @@ public: void reloadConfiguration(); /// Строит необходимые логгеры - void buildLoggers(); + void buildLoggers(Poco::Util::AbstractConfiguration & config); /// Определяет параметр командной строки void defineOptions(Poco::Util::OptionSet & _options) override; @@ -228,6 +228,11 @@ protected: std::string config_path; ConfigProcessor::LoadedConfig loaded_config; Poco::Util::AbstractConfiguration * last_configuration = nullptr; + +private: + + /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed. + std::string config_logger; }; diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 1629fee91138808de8952d704e13d8b26c67be15..3c0576dcd92ea5a49d8e5c2751b21adb6432a075 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -565,67 +565,58 @@ void BaseDaemon::wakeup() } -void BaseDaemon::buildLoggers() +void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) { - bool is_daemon = config().getBool("application.runAsDaemon", false); + auto current_logger = config.getString("logger"); + if (config_logger == current_logger) + return; + config_logger = current_logger; - /// Change path for logging. - if (config().hasProperty("logger.log")) - { - std::string path = createDirectory(config().getString("logger.log")); - if (is_daemon - && chdir(path.c_str()) != 0) - throw Poco::Exception("Cannot change directory to " + path); - } - else - { - if (is_daemon - && chdir("/tmp") != 0) - throw Poco::Exception("Cannot change directory to /tmp"); - } + bool is_daemon = config.getBool("application.runAsDaemon", false); // Split log and error log. Poco::AutoPtr split = new SplitterChannel; - if (config().hasProperty("logger.log")) + auto log_level = config.getString("logger.level", "trace"); + if (config.hasProperty("logger.log")) { - createDirectory(config().getString("logger.log")); - std::cerr << "Logging to " << config().getString("logger.log") << std::endl; + createDirectory(config.getString("logger.log")); + std::cerr << "Logging " << log_level << " to " << config.getString("logger.log") << std::endl; // Set up two channel chains. Poco::AutoPtr pf = new OwnPatternFormatter(this); pf->setProperty("times", "local"); Poco::AutoPtr log = new FormattingChannel(pf); log_file = new FileChannel; - log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(config().getString("logger.log")).absolute().toString()); - log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config().getRawString("logger.size", "100M")); + log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(config.getString("logger.log")).absolute().toString()); + log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); - log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config().getRawString("logger.compress", "true")); - log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config().getRawString("logger.count", "1")); - log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config().getRawString("logger.flush", "true")); - log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config().getRawString("logger.rotateOnOpen", "false")); + log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); + log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); + log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); + log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); log->setChannel(log_file); split->addChannel(log); log_file->open(); } - if (config().hasProperty("logger.errorlog")) + if (config.hasProperty("logger.errorlog")) { - createDirectory(config().getString("logger.errorlog")); - std::cerr << "Logging errors to " << config().getString("logger.errorlog") << std::endl; + createDirectory(config.getString("logger.errorlog")); + std::cerr << "Logging errors to " << config.getString("logger.errorlog") << std::endl; Poco::AutoPtr level = new Poco::LevelFilterChannel; level->setLevel(Message::PRIO_NOTICE); Poco::AutoPtr pf = new OwnPatternFormatter(this); pf->setProperty("times", "local"); Poco::AutoPtr errorlog = new FormattingChannel(pf); error_log_file = new FileChannel; - error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(config().getString("logger.errorlog")).absolute().toString()); - error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config().getRawString("logger.size", "100M")); + error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(config.getString("logger.errorlog")).absolute().toString()); + error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); - error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config().getRawString("logger.compress", "true")); - error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config().getRawString("logger.count", "1")); - error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config().getRawString("logger.flush", "true")); - error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config().getRawString("logger.rotateOnOpen", "false")); + error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); + error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); + error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); + error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); errorlog->setChannel(error_log_file); level->setChannel(errorlog); split->addChannel(level); @@ -635,7 +626,7 @@ void BaseDaemon::buildLoggers() /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code. /// We don't need this configuration parameter. - if (config().getBool("logger.use_syslog", false) || config().getBool("dynamic_layer_selection", false)) + if (config.getBool("logger.use_syslog", false) || config.getBool("dynamic_layer_selection", false)) { Poco::AutoPtr pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG); pf->setProperty("times", "local"); @@ -646,14 +637,14 @@ void BaseDaemon::buildLoggers() syslog_channel->open(); } - if (config().getBool("logger.console", false) || (!config().hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO)))) + if (config.getBool("logger.console", false) || (!config.hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO)))) { Poco::AutoPtr file = new ConsoleChannel; Poco::AutoPtr pf = new OwnPatternFormatter(this); pf->setProperty("times", "local"); Poco::AutoPtr log = new FormattingChannel(pf); log->setChannel(file); - logger().warning("Logging to console"); + logger().warning("Logging " + log_level + " to console"); split->addChannel(log); } @@ -662,19 +653,25 @@ void BaseDaemon::buildLoggers() logger().setChannel(split); // Global logging level (it can be overridden for specific loggers). - logger().setLevel(config().getString("logger.level", "trace")); + logger().setLevel(log_level); + + // Set level to all already created loggers + std::vector names; + Logger::root().names(names); + for (const auto & name : names) + Logger::root().get(name).setLevel(log_level); // Attach to the root logger. - Logger::root().setLevel(logger().getLevel()); + Logger::root().setLevel(log_level); Logger::root().setChannel(logger().getChannel()); // Explicitly specified log levels for specific loggers. AbstractConfiguration::Keys levels; - config().keys("logger.levels", levels); + config.keys("logger.levels", levels); if(!levels.empty()) for(AbstractConfiguration::Keys::iterator it = levels.begin(); it != levels.end(); ++it) - Logger::get(*it).setLevel(config().getString("logger.levels." + *it, "trace")); + Logger::get(*it).setLevel(config.getString("logger.levels." + *it, "trace")); } @@ -839,7 +836,22 @@ void BaseDaemon::initialize(Application & self) pid.seed(config().getString("pid")); } - buildLoggers(); + /// Change path for logging. + if (config().hasProperty("logger.log")) + { + std::string path = createDirectory(config().getString("logger.log")); + if (is_daemon + && chdir(path.c_str()) != 0) + throw Poco::Exception("Cannot change directory to " + path); + } + else + { + if (is_daemon + && chdir("/tmp") != 0) + throw Poco::Exception("Cannot change directory to /tmp"); + } + + buildLoggers(config()); if (is_daemon) { diff --git a/utils/config-processor/CMakeLists.txt b/utils/config-processor/CMakeLists.txt index 965258803fa99473ac441a4fae99d86cac3c9500..d25c25d2f0251a79336f8d9aed16a3e610abac50 100644 --- a/utils/config-processor/CMakeLists.txt +++ b/utils/config-processor/CMakeLists.txt @@ -1,4 +1,4 @@ add_executable (config-processor config-processor.cpp) -target_link_libraries (config-processor clickhouse_common_configprocessor) +target_link_libraries (config-processor clickhouse_common_config) INSTALL(TARGETS config-processor RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT config-processor) diff --git a/utils/config-processor/config-processor.cpp b/utils/config-processor/config-processor.cpp index ff3a3e6160c9cc4aa6033460ceff9d3dd0c718f9..31eaf8522a4cd39884d20a567abf6f2b6ce68fa7 100644 --- a/utils/config-processor/config-processor.cpp +++ b/utils/config-processor/config-processor.cpp @@ -1,4 +1,4 @@ -#include +#include #include int main(int argc, char ** argv)