fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

fix build [#CLICKHOUSE-3305]

dbms/src/Dictionaries/IDictionary.h

@@ -8,16 +8,14 @@
 #include <Common/PODArray.h>
 #include <memory>
 #include <chrono>
+#include <Dictionaries/IDictionarySource.h>
 
 namespace DB
 {
 
-class IDictionarySource;
-
 struct IDictionaryBase;
 using DictionaryPtr = std::unique_ptr<IDictionaryBase>;
 
-struct DictionaryLifetime;
 struct DictionaryStructure;
 class ColumnString;
 
@@ -57,9 +55,14 @@ struct IDictionaryBase : public IExternalLoadable
 
     bool supportUpdates() const override { return !isCached(); }
 
-    virtual std::shared_ptr<IExternalLoadable> cloneObject() const override
+    bool isModified() const override
+    {   auto source = getSource();
+        return source && source->isModified();
+    }
+
+    std::unique_ptr<IExternalLoadable> cloneObject() const override
     {
-        return std::static_pointer_cast<IDictionaryBase>(clone());
+        return clone();
     };
 
     std::shared_ptr<IDictionaryBase> shared_from_this()

dbms/src/Interpreters/ExternalDictionaries.cpp

@@ -44,13 +44,13 @@ ExternalDictionaries::ExternalDictionaries(Context & context, bool throw_on_erro
                          getExternalDictionariesUpdateSettings(),
                          getExternalDictionariesConfigSettings(),
                          &Logger::get("ExternalDictionaries"),
-                         "external dictionary", throw_on_error),
+                         "external dictionary"),
           context(context)
 {
-
+    init(throw_on_error);
 }
 
-ExternalDictionaries::LoadablePtr ExternalDictionaries::create(
+std::unique_ptr<IExternalLoadable> ExternalDictionaries::create(
         const std::string & name, const Configuration & config, const std::string & config_prefix)
 {
     return DictionaryFactory::instance().create(name, config, config_prefix, context);

dbms/src/Interpreters/ExternalDictionaries.h

@@ -30,8 +30,8 @@ public:
 
 protected:
 
-    LoadablePtr create(const std::string & name, const Configuration & config,
-                       const std::string & config_prefix) override;
+    std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
+                                              const std::string & config_prefix) override;
 
     using ExternalLoader::getObjectsMap;
 

dbms/src/Interpreters/ExternalLoader.cpp

 #include <Interpreters/ExternalLoader.h>
 #include <Common/StringUtils.h>
 #include <Common/MemoryTracker.h>
+#include <Common/Exception.h>
 #include <Common/getMultipleKeysFromConfig.h>
+#include <Common/setThreadName.h>
 #include <ext/scope_guard.h>
 #include <Poco/Util/Application.h>
 #include <Poco/Glob.h>
@@ -46,10 +48,19 @@ void ExternalLoader::reloadPeriodically()
 ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config,
                                const ExternalLoaderUpdateSettings & update_settings,
                                const ExternalLoaderConfigSettings & config_settings,
-                               Logger * log, const std::string & loadable_object_name, bool throw_on_error)
+                               Logger * log, const std::string & loadable_object_name)
         : config(config), update_settings(update_settings), config_settings(config_settings),
           log(log), object_name(loadable_object_name)
 {
+}
+
+void ExternalLoader::init(bool throw_on_error)
+{
+    if (is_initialized)
+        return;
+
+    is_initialized = true;
+
     {
         /// During synchronous loading of external dictionaries at moment of query execution,
         /// we should not use per query memory limit.
@@ -116,7 +127,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
 
         try
         {
-            auto loadable_ptr = failed_loadable_object.second.loadable->clone();
+            auto loadable_ptr = failed_loadable_object.second.loadable->cloneObject();
             if (const auto exception_ptr = loadable_ptr->getCreationException())
             {
                 /// recalculate next attempt time
@@ -143,7 +154,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
                 const auto dict_it = loadable_objects.find(name);
 
                 dict_it->second.loadable.reset();
-                dict_it->second.loadable = loadable_ptr;
+                dict_it->second.loadable = std::move(loadable_ptr);
 
                 /// clear stored exception on success
                 dict_it->second.exception = std::exception_ptr{};
@@ -200,13 +211,13 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
                 if (current->isModified())
                 {
                     /// create new version of loadable object
-                    auto new_version = current->clone();
+                    auto new_version = current->cloneObject();
 
                     if (const auto exception_ptr = new_version->getCreationException())
                         std::rethrow_exception(exception_ptr);
 
                     loadable_object.second.loadable.reset();
-                    loadable_object.second.loadable = new_version;
+                    loadable_object.second.loadable = std::move(new_version);
                 }
             }
 
@@ -321,9 +332,9 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
                         const auto failed_dict_it = failed_loadable_objects.find(name);
                         FailedLoadableInfo info{std::move(object_ptr), std::chrono::system_clock::now() + delay, 0};
                         if (failed_dict_it != std::end(failed_loadable_objects))
-                            failed_dict_it->second = info;
+                            (*failed_dict_it).second = std::move(info);
                         else
-                            failed_loadable_objects.emplace(name, info);
+                            failed_loadable_objects.emplace(name, std::move(info));
 
                         std::rethrow_exception(exception_ptr);
                     }
@@ -343,12 +354,12 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
 
                     /// add new loadable object or update an existing version
                     if (object_it == std::end(loadable_objects))
-                        loadable_objects.emplace(name, LoadableInfo{object_ptr, config_path});
+                        loadable_objects.emplace(name, LoadableInfo{std::move(object_ptr), config_path});
                     else
                     {
                         if (object_it->second.loadable)
                             object_it->second.loadable.reset();
-                        object_it->second.loadable = object_ptr;
+                        object_it->second.loadable = std::move(object_ptr);
 
                         /// erase stored exception on success
                         object_it->second.exception = std::exception_ptr{};
@@ -414,4 +425,9 @@ ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name
     return it->second.loadable;
 }
 
+std::tuple<std::unique_lock<std::mutex>, const ExternalLoader::ObjectsMap &> ExternalLoader::getObjectsMap() const
+{
+    return std::make_tuple(std::unique_lock<std::mutex>(map_mutex), std::cref(loadable_objects));
+}
+
 }

dbms/src/Interpreters/ExternalLoader.h

 #pragma once
 
-#include <Common/Exception.h>
-#include <Common/setThreadName.h>
-#include <Common/randomSeed.h>
-#include <common/MultiVersion.h>
 #include <common/logger_useful.h>
 #include <Poco/Event.h>
-#include <unistd.h>
-#include <ctime>
 #include <mutex>
 #include <thread>
 #include <unordered_map>
 #include <chrono>
-#include <pcg_random.hpp>
-#include <Poco/Util/AbstractConfiguration.h>
+#include <tuple>
 #include <Interpreters/IExternalLoadable.h>
+#include <Core/Types.h>
+#include <pcg_random.hpp>
+#include <Common/randomSeed.h>
 
 
 namespace DB
@@ -64,16 +60,34 @@ struct ExternalLoaderConfigSettings
 */
 class ExternalLoader
 {
+public:
+    using LoadablePtr = std::shared_ptr<IExternalLoadable>;
+
+private:
+    struct LoadableInfo final
+    {
+        LoadablePtr loadable;
+        std::string origin;
+        std::exception_ptr exception;
+    };
+
+    struct FailedLoadableInfo final
+    {
+        std::unique_ptr<IExternalLoadable> loadable;
+        std::chrono::system_clock::time_point next_attempt_time;
+        UInt64 error_count;
+    };
+
 public:
     using Configuration = Poco::Util::AbstractConfiguration;
     using ObjectsMap = std::unordered_map<std::string, LoadableInfo>;
 
-    /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
+    /// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
     ExternalLoader(const Configuration & config,
                    const ExternalLoaderUpdateSettings & update_settings,
                    const ExternalLoaderConfigSettings & config_settings,
-                   Logger * log, const std::string & loadable_object_name, bool throw_on_error);
-    ~ExternalLoader();
+                   Logger * log, const std::string & loadable_object_name);
+    virtual ~ExternalLoader();
 
     /// Forcibly reloads all loadable objects.
     void reload();
@@ -84,48 +98,33 @@ public:
     LoadablePtr getLoadable(const std::string & name) const;
 
 protected:
-    virtual LoadablePtr create(const std::string & name, const Configuration & config,
-                               const std::string & config_prefix) = 0;
+    virtual std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
+                                                      const std::string & config_prefix) = 0;
 
     /// Direct access to objects.
-    std::tuple<std::lock_guard<std::mutex>, const ObjectsMap &> getObjectsMap()
-    {
-        return std::make_tuple(std::lock_guard<std::mutex>(map_mutex), std::cref(loadable_objects));
-    }
+    std::tuple<std::unique_lock<std::mutex>, const ObjectsMap &> getObjectsMap() const;
+
+    /// Should be called in derived constructor (to avoid pure virtual call).
+    void init(bool throw_on_error);
 
 private:
 
-    /// Protects only dictionaries map.
+    bool is_initialized = false;
+
+    /// Protects only objects map.
     mutable std::mutex map_mutex;
 
     /// Protects all data, currently used to avoid races between updating thread and SYSTEM queries
     mutable std::mutex all_mutex;
 
-    using LoadablePtr = std::shared_ptr<IExternalLoadable>;
-    struct LoadableInfo final
-    {
-        LoadablePtr loadable;
-        std::string origin;
-        std::exception_ptr exception;
-    };
-
-    struct FailedLoadableInfo final
-    {
-        std::unique_ptr<IExternalLoadable> loadable;
-        std::chrono::system_clock::time_point next_attempt_time;
-        UInt64 error_count;
-    };
-
     /// name -> loadable.
     ObjectsMap loadable_objects;
 
-    /** Here are loadable objects, that has been never loaded successfully.
-      * They are also in 'loadable_objects', but with nullptr as 'loadable'.
-      */
+    /// Here are loadable objects, that has been never loaded successfully.
+    /// They are also in 'loadable_objects', but with nullptr as 'loadable'.
     std::unordered_map<std::string, FailedLoadableInfo> failed_loadable_objects;
 
-    /** Both for dictionaries and failed_dictionaries.
-      */
+    /// Both for loadable_objects and failed_loadable_objects.
     std::unordered_map<std::string, std::chrono::system_clock::time_point> update_times;
 
     pcg64 rnd_engine{randomSeed()};
@@ -143,7 +142,7 @@ private:
 
     std::unordered_map<std::string, Poco::Timestamp> last_modification_times;
 
-    /// Check dictionaries definitions in config files and reload or/and add new ones if the definition is changed
+    /// Check objects definitions in config files and reload or/and add new ones if the definition is changed
     /// If loadable_name is not empty, load only loadable object with name loadable_name
     void reloadFromConfigFiles(bool throw_on_error, bool force_reload = false, const std::string & loadable_name = "");
     void reloadFromConfigFile(const std::string & config_path, bool throw_on_error, bool force_reload,

dbms/src/Interpreters/IExternalLoadable.h

+#pragma once
 #include <string>
 #include <memory>
+#include <Core/Types.h>
 
-namespace DB
-{
 
 namespace Poco::Util
 {
-class AbstractConfiguration;
+    class AbstractConfiguration;
 }
 
+
+namespace DB
+{
+
 /// Min and max lifetimes for a loadable object or it's entry
 struct ExternalLoadableLifetime final
 {
@@ -31,7 +35,7 @@ public:
 
     virtual bool isModified() const = 0;
 
-    virtual std::shared_ptr<IExternalLoadable> cloneObject() const = 0;
+    virtual std::unique_ptr<IExternalLoadable> cloneObject() const = 0;
 
     virtual std::exception_ptr getCreationException() const = 0;
 };