Allow redefine server config via command line: clickhouse-server --...

Allow redefine server config via command line: clickhouse-server -- --logger.level=information (#1811)

* Aloow redefine server config via command line: clickhouse-server -- --logger.level=information

* style

* fix

* BaseDaemon: Simpler createDirectory

* Allow --key value

* Update BaseDaemon.cpp

* Update BaseDaemon.cpp

dbms/src/Server/Server.cpp

@@ -2,16 +2,13 @@
 
 #include <memory>
 #include <sys/resource.h>
-
 #include <Poco/DirectoryIterator.h>
 #include <Poco/Net/HTTPServer.h>
 #include <Poco/Net/NetException.h>
-
 #include <ext/scope_guard.h>
-
+#include <common/logger_useful.h>
 #include <common/ErrorHandlers.h>
 #include <common/getMemoryAmount.h>
-
 #include <Common/ClickHouseRevision.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/Macros.h>
@@ -22,22 +19,17 @@
 #include <Common/getFQDNOrHostName.h>
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
-
 #include <IO/HTTPCommon.h>
-
 #include <Interpreters/AsynchronousMetrics.h>
 #include <Interpreters/DDLWorker.h>
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/loadMetadata.h>
-
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/System/attachSystemTables.h>
-
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <Functions/registerFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/registerStorages.h>
-
 #include "ConfigReloader.h"
 #include "HTTPHandlerFactory.h"
 #include "MetricsTransmitter.h"
@@ -49,7 +41,6 @@
 #include <Poco/Net/SecureServerSocket.h>
 #endif
 
-
 namespace CurrentMetrics
 {
     extern const Metric Revision;
@@ -76,6 +67,18 @@ static std::string getCanonicalPath(std::string && path)
     return path;
 }
 
+void Server::uninitialize()
+{
+    logger().information("shutting down");
+    BaseDaemon::uninitialize();
+}
+
+void Server::initialize(Poco::Util::Application & self)
+{
+    BaseDaemon::initialize(self);
+    logger().information("starting up");
+}
+
 std::string Server::getDefaultCorePath() const
 {
     return getCanonicalPath(config().getString("path")) + "cores";

dbms/src/Server/Server.h

@@ -2,7 +2,6 @@
 
 #include "IServer.h"
 
-#include <common/logger_useful.h>
 #include <daemon/BaseDaemon.h>
 
 /** Server provides three interfaces:
@@ -43,17 +42,9 @@ public:
     }
 
 protected:
-    void initialize(Application & self) override
-    {
-        BaseDaemon::initialize(self);
-        logger().information("starting up");
-    }
+    void initialize(Application & self) override;
 
-    void uninitialize() override
-    {
-        logger().information("shutting down");
-        BaseDaemon::uninitialize();
-    }
+    void uninitialize() override;
 
     int main(const std::vector<std::string> & args) override;
 

dbms/src/Server/config.xml

@@ -6,6 +6,7 @@
         <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
         <size>1000M</size>
         <count>10</count>
+        <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
     </logger>
 
     <http_port>8123</http_port>

libs/libcommon/CMakeLists.txt

@@ -17,7 +17,7 @@ if (APPLE)
     target_include_directories (apple_rt PUBLIC ${COMMON_INCLUDE_DIR})
 endif ()
 
-add_library (common
+add_library (common ${SPLIT_SHARED}
     src/DateLUT.cpp
     src/DateLUTImpl.cpp
     src/exp10.c

libs/libdaemon/CMakeLists.txt

-add_library (daemon
+add_library (daemon ${SPLIT_SHARED}
     src/BaseDaemon.cpp
     src/GraphiteWriter.cpp
     src/OwnPatternFormatter.cpp

libs/libdaemon/src/BaseDaemon.cpp

@@ -47,6 +47,8 @@
 #include <Poco/Message.h>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Poco/Util/XMLConfiguration.h>
+#include <Poco/Util/MapConfiguration.h>
+#include <Poco/Util/Application.h>
 #include <Poco/ScopedLock.h>
 #include <Poco/Exception.h>
 #include <Poco/ErrorHandler.h>
@@ -486,21 +488,9 @@ static void terminate_handler()
 
 static std::string createDirectory(const std::string & _path)
 {
-    Poco::Path path(_path);
-    std::string str;
-    for(int j=0;j<path.depth();++j)
-    {
-        str += "/";
-        str += path[j];
-
-        int res = ::mkdir(str.c_str(), 0700);
-        if( res && (errno!=EEXIST) )
-        {
-            throw std::runtime_error(std::string("Can't create dir - ") + str + " - " + strerror(errno));
-        }
-    }
-
-    return str;
+    auto path = Poco::Path(_path).makeDirectory().popDirectory();
+    Poco::File(path).createDirectories();
+    return path.toString();
 };
 
 static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path)
@@ -708,6 +698,63 @@ void BaseDaemon::initialize(Application & self)
     task_manager.reset(new Poco::TaskManager);
     ServerApplication::initialize(self);
 
+    {
+        /// Parsing all args and converting to config layer
+        /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4
+        Poco::AutoPtr<Poco::Util::MapConfiguration> map_config = new Poco::Util::MapConfiguration;
+        std::string key;
+        for(auto & arg : argv())
+        {
+            auto key_start = arg.find_first_not_of('-');
+            auto pos_minus = arg.find('-');
+            auto pos_eq = arg.find('=');
+
+            // old saved '--key', will set to some true value "1"
+            if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start)
+            {
+                map_config->setString(key, "1");
+                key = "";
+            }
+
+            if (pos_eq == std::string::npos)
+            {
+                if (!key.empty())
+                {
+                    if (pos_minus == std::string::npos || pos_minus > key_start)
+                    {
+                        map_config->setString(key, arg);
+                    }
+                    key = "";
+                }
+                if (pos_minus != std::string::npos && key_start != std::string::npos && pos_minus < key_start)
+                    key = arg.substr(key_start);
+                continue;
+            }
+            else
+            {
+                key = "";
+            }
+
+            if (key_start == std::string::npos)
+                continue;
+
+            if (pos_minus > key_start)
+                continue;
+
+            key = arg.substr(key_start, pos_eq - key_start);
+            if (key.empty())
+                continue;
+            std::string value;
+            if (arg.size() > pos_eq)
+                value = arg.substr(pos_eq+1);
+
+            map_config->setString(key, value);
+            key = "";
+        }
+        /// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher!
+        config().add(map_config, PRIO_APPLICATION - 100);
+    }
+
     bool is_daemon = config().getBool("application.runAsDaemon", false);
 
     if (is_daemon)

libs/libmysqlxx/CMakeLists.txt

 
-add_library (mysqlxx
+add_library (mysqlxx ${SPLIT_SHARED}
     src/Connection.cpp
     src/Exception.cpp
     src/Query.cpp

libs/libpocoext/CMakeLists.txt

-add_library (pocoext
+add_library (pocoext ${SPLIT_SHARED}
     src/LevelFilterChannel.cpp
     src/ThreadNumber.cpp