diff --git a/dbms/src/Dictionaries/IDictionary.h b/dbms/src/Dictionaries/IDictionary.h index f378a9a7b07a9deb7c16492c48e0056c9d735698..48b8206fbca3116551b7d36856faa76c4e2ac4b9 100644 --- a/dbms/src/Dictionaries/IDictionary.h +++ b/dbms/src/Dictionaries/IDictionary.h @@ -8,16 +8,14 @@ #include #include #include +#include namespace DB { -class IDictionarySource; - struct IDictionaryBase; using DictionaryPtr = std::unique_ptr; -struct DictionaryLifetime; struct DictionaryStructure; class ColumnString; @@ -57,9 +55,14 @@ struct IDictionaryBase : public IExternalLoadable bool supportUpdates() const override { return !isCached(); } - virtual std::shared_ptr cloneObject() const override + bool isModified() const override + { auto source = getSource(); + return source && source->isModified(); + } + + std::unique_ptr cloneObject() const override { - return std::static_pointer_cast(clone()); + return clone(); }; std::shared_ptr shared_from_this() diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index b42d703d090aca102e7e366801176e4a639ba84b..4c1e563e1f21222d078ecaf43ede961f91788103 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -44,13 +44,13 @@ ExternalDictionaries::ExternalDictionaries(Context & context, bool throw_on_erro getExternalDictionariesUpdateSettings(), getExternalDictionariesConfigSettings(), &Logger::get("ExternalDictionaries"), - "external dictionary", throw_on_error), + "external dictionary"), context(context) { - + init(throw_on_error); } -ExternalDictionaries::LoadablePtr ExternalDictionaries::create( +std::unique_ptr ExternalDictionaries::create( const std::string & name, const Configuration & config, const std::string & config_prefix) { return DictionaryFactory::instance().create(name, config, config_prefix, context); diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h index 31ff6ad53411e4efb50ba08a8844dc6ab4587bc5..9ed41f89d509880ed0f6fb11637f1aafcf9456ff 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.h +++ b/dbms/src/Interpreters/ExternalDictionaries.h @@ -30,8 +30,8 @@ public: protected: - LoadablePtr create(const std::string & name, const Configuration & config, - const std::string & config_prefix) override; + std::unique_ptr create(const std::string & name, const Configuration & config, + const std::string & config_prefix) override; using ExternalLoader::getObjectsMap; diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 42967f541c8b1e3cf28b69538d610651d12e609b..9b78463936637bad37f56e1f48d300ed993a63d9 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -46,10 +48,19 @@ void ExternalLoader::reloadPeriodically() ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config, const ExternalLoaderUpdateSettings & update_settings, const ExternalLoaderConfigSettings & config_settings, - Logger * log, const std::string & loadable_object_name, bool throw_on_error) + Logger * log, const std::string & loadable_object_name) : config(config), update_settings(update_settings), config_settings(config_settings), log(log), object_name(loadable_object_name) { +} + +void ExternalLoader::init(bool throw_on_error) +{ + if (is_initialized) + return; + + is_initialized = true; + { /// During synchronous loading of external dictionaries at moment of query execution, /// we should not use per query memory limit. @@ -116,7 +127,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) try { - auto loadable_ptr = failed_loadable_object.second.loadable->clone(); + auto loadable_ptr = failed_loadable_object.second.loadable->cloneObject(); if (const auto exception_ptr = loadable_ptr->getCreationException()) { /// recalculate next attempt time @@ -143,7 +154,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) const auto dict_it = loadable_objects.find(name); dict_it->second.loadable.reset(); - dict_it->second.loadable = loadable_ptr; + dict_it->second.loadable = std::move(loadable_ptr); /// clear stored exception on success dict_it->second.exception = std::exception_ptr{}; @@ -200,13 +211,13 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) if (current->isModified()) { /// create new version of loadable object - auto new_version = current->clone(); + auto new_version = current->cloneObject(); if (const auto exception_ptr = new_version->getCreationException()) std::rethrow_exception(exception_ptr); loadable_object.second.loadable.reset(); - loadable_object.second.loadable = new_version; + loadable_object.second.loadable = std::move(new_version); } } @@ -321,9 +332,9 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const const auto failed_dict_it = failed_loadable_objects.find(name); FailedLoadableInfo info{std::move(object_ptr), std::chrono::system_clock::now() + delay, 0}; if (failed_dict_it != std::end(failed_loadable_objects)) - failed_dict_it->second = info; + (*failed_dict_it).second = std::move(info); else - failed_loadable_objects.emplace(name, info); + failed_loadable_objects.emplace(name, std::move(info)); std::rethrow_exception(exception_ptr); } @@ -343,12 +354,12 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const /// add new loadable object or update an existing version if (object_it == std::end(loadable_objects)) - loadable_objects.emplace(name, LoadableInfo{object_ptr, config_path}); + loadable_objects.emplace(name, LoadableInfo{std::move(object_ptr), config_path}); else { if (object_it->second.loadable) object_it->second.loadable.reset(); - object_it->second.loadable = object_ptr; + object_it->second.loadable = std::move(object_ptr); /// erase stored exception on success object_it->second.exception = std::exception_ptr{}; @@ -414,4 +425,9 @@ ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name return it->second.loadable; } +std::tuple, const ExternalLoader::ObjectsMap &> ExternalLoader::getObjectsMap() const +{ + return std::make_tuple(std::unique_lock(map_mutex), std::cref(loadable_objects)); +} + } diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index d330bae2633f3600b892b3b1e3e13317f7e89e57..672c801424eb53a88213cd251fcca2177df60c67 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -1,20 +1,16 @@ #pragma once -#include -#include -#include -#include #include #include -#include -#include #include #include #include #include -#include -#include +#include #include +#include +#include +#include namespace DB @@ -64,16 +60,34 @@ struct ExternalLoaderConfigSettings */ class ExternalLoader { +public: + using LoadablePtr = std::shared_ptr; + +private: + struct LoadableInfo final + { + LoadablePtr loadable; + std::string origin; + std::exception_ptr exception; + }; + + struct FailedLoadableInfo final + { + std::unique_ptr loadable; + std::chrono::system_clock::time_point next_attempt_time; + UInt64 error_count; + }; + public: using Configuration = Poco::Util::AbstractConfiguration; using ObjectsMap = std::unordered_map; - /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. + /// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. ExternalLoader(const Configuration & config, const ExternalLoaderUpdateSettings & update_settings, const ExternalLoaderConfigSettings & config_settings, - Logger * log, const std::string & loadable_object_name, bool throw_on_error); - ~ExternalLoader(); + Logger * log, const std::string & loadable_object_name); + virtual ~ExternalLoader(); /// Forcibly reloads all loadable objects. void reload(); @@ -84,48 +98,33 @@ public: LoadablePtr getLoadable(const std::string & name) const; protected: - virtual LoadablePtr create(const std::string & name, const Configuration & config, - const std::string & config_prefix) = 0; + virtual std::unique_ptr create(const std::string & name, const Configuration & config, + const std::string & config_prefix) = 0; /// Direct access to objects. - std::tuple, const ObjectsMap &> getObjectsMap() - { - return std::make_tuple(std::lock_guard(map_mutex), std::cref(loadable_objects)); - } + std::tuple, const ObjectsMap &> getObjectsMap() const; + + /// Should be called in derived constructor (to avoid pure virtual call). + void init(bool throw_on_error); private: - /// Protects only dictionaries map. + bool is_initialized = false; + + /// Protects only objects map. mutable std::mutex map_mutex; /// Protects all data, currently used to avoid races between updating thread and SYSTEM queries mutable std::mutex all_mutex; - using LoadablePtr = std::shared_ptr; - struct LoadableInfo final - { - LoadablePtr loadable; - std::string origin; - std::exception_ptr exception; - }; - - struct FailedLoadableInfo final - { - std::unique_ptr loadable; - std::chrono::system_clock::time_point next_attempt_time; - UInt64 error_count; - }; - /// name -> loadable. ObjectsMap loadable_objects; - /** Here are loadable objects, that has been never loaded successfully. - * They are also in 'loadable_objects', but with nullptr as 'loadable'. - */ + /// Here are loadable objects, that has been never loaded successfully. + /// They are also in 'loadable_objects', but with nullptr as 'loadable'. std::unordered_map failed_loadable_objects; - /** Both for dictionaries and failed_dictionaries. - */ + /// Both for loadable_objects and failed_loadable_objects. std::unordered_map update_times; pcg64 rnd_engine{randomSeed()}; @@ -143,7 +142,7 @@ private: std::unordered_map last_modification_times; - /// Check dictionaries definitions in config files and reload or/and add new ones if the definition is changed + /// Check objects definitions in config files and reload or/and add new ones if the definition is changed /// If loadable_name is not empty, load only loadable object with name loadable_name void reloadFromConfigFiles(bool throw_on_error, bool force_reload = false, const std::string & loadable_name = ""); void reloadFromConfigFile(const std::string & config_path, bool throw_on_error, bool force_reload, diff --git a/dbms/src/Interpreters/IExternalLoadable.h b/dbms/src/Interpreters/IExternalLoadable.h index 841aba86dfa72aebed6c69fea4a31ae6ffa05e8e..9e52b9e6864f0dcaf1507b6f7d989bf8e7db1e52 100644 --- a/dbms/src/Interpreters/IExternalLoadable.h +++ b/dbms/src/Interpreters/IExternalLoadable.h @@ -1,14 +1,18 @@ +#pragma once #include #include +#include -namespace DB -{ namespace Poco::Util { -class AbstractConfiguration; + class AbstractConfiguration; } + +namespace DB +{ + /// Min and max lifetimes for a loadable object or it's entry struct ExternalLoadableLifetime final { @@ -31,7 +35,7 @@ public: virtual bool isModified() const = 0; - virtual std::shared_ptr cloneObject() const = 0; + virtual std::unique_ptr cloneObject() const = 0; virtual std::exception_ptr getCreationException() const = 0; };