diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 05d7438a8d169c173649ae34bebbea89e4cbe91a..ce3243862801ccbd7b635c76a6f55266adfcf831 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -474,6 +474,7 @@ namespace ErrorCodes extern const int NOT_ENOUGH_PRIVILEGES = 497; extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498; extern const int S3_ERROR = 499; + extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA = 500; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 52dde4efe9bb0792b526dfd9611a6a5619d0542c..d39ed5ec1932b7c00b22f7e61382a94cd5c29b41 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -102,13 +102,12 @@ void DatabaseDictionary::removeDictionary( } void DatabaseDictionary::attachDictionary( - const String & /*dictionary_name*/, const Context & /*context*/, bool /*reload*/) + const String & /*dictionary_name*/, const Context & /*context*/) { throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD); } -void DatabaseDictionary::detachDictionary( - const String & /*dictionary_name*/, const Context & /*context*/, bool /*reload*/) +void DatabaseDictionary::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/) { throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD); } diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 9e88b50a68ef08c68e559ac86e87599aea23fa3d..64acdad8645092484d5b6ef0fc4c5510898dd815 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -89,9 +89,9 @@ public: ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & table_name) const override; - void attachDictionary(const String & dictionary_name, const Context & context, bool reload) override; + void attachDictionary(const String & dictionary_name, const Context & context) override; - void detachDictionary(const String & dictionary_name, const Context & context, bool reload) override; + void detachDictionary(const String & dictionary_name, const Context & context) override; void shutdown() override; diff --git a/dbms/src/Databases/DatabaseLazy.cpp b/dbms/src/Databases/DatabaseLazy.cpp index b232bf49392c4c7dd2ee23d82ab95efe390c687f..68f2b4e0e22aa9550c3d9925dc5eb40610d6371d 100644 --- a/dbms/src/Databases/DatabaseLazy.cpp +++ b/dbms/src/Databases/DatabaseLazy.cpp @@ -123,13 +123,12 @@ DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator( void DatabaseLazy::attachDictionary( const String & /*dictionary_name*/, - const Context & /*context*/, - bool /*load*/) + const Context & /*context*/) { throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD); } -void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/, bool /*reload*/) +void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/) { throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD); } diff --git a/dbms/src/Databases/DatabaseLazy.h b/dbms/src/Databases/DatabaseLazy.h index c268f58945c9dabb1d69590b76a090cc2d243370..75f130ff4c5c79deb4386652ae633b4076fb1bd7 100644 --- a/dbms/src/Databases/DatabaseLazy.h +++ b/dbms/src/Databases/DatabaseLazy.h @@ -111,9 +111,9 @@ public: StoragePtr detachTable(const String & table_name) override; - void attachDictionary(const String & dictionary_name, const Context & context, bool reload) override; + void attachDictionary(const String & dictionary_name, const Context & context) override; - void detachDictionary(const String & dictionary_name, const Context & context, bool reload) override; + void detachDictionary(const String & dictionary_name, const Context & context) override; void shutdown() override; diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 0badc9b4df436727e4ddd701da69c16e75f62252..7d7f101a88cbe0fa63a255fbc9d05143c6b711f7 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -35,7 +35,7 @@ void DatabaseMemory::createTable( } -void DatabaseMemory::attachDictionary(const String & /*name*/, const Context & /*context*/, bool /*reload*/) +void DatabaseMemory::attachDictionary(const String & /*name*/, const Context & /*context*/) { throw Exception("There is no ATTACH DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD); } @@ -57,7 +57,7 @@ void DatabaseMemory::removeTable( } -void DatabaseMemory::detachDictionary(const String & /*name*/, const Context & /*context*/, bool /*reload*/) +void DatabaseMemory::detachDictionary(const String & /*name*/, const Context & /*context*/) { throw Exception("There is no DETACH DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD); } diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 45f51a177f7cfa8ac3aef625cbea46bc416654f9..40f54c793e688a997f580a58187496da6a58b7c7 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -40,8 +40,7 @@ public: void attachDictionary( const String & name, - const Context & context, - bool reload) override; + const Context & context) override; void removeTable( const Context & context, @@ -53,8 +52,7 @@ public: void detachDictionary( const String & name, - const Context & context, - bool reload) override; + const Context & context) override; time_t getObjectMetadataModificationTime(const Context & context, const String & table_name) override; diff --git a/dbms/src/Databases/DatabaseMySQL.h b/dbms/src/Databases/DatabaseMySQL.h index 3ca8722b24888addb5ad3b2a564196662a17678f..17c056e8dacea8396efad081c33d3d00f6016fc4 100644 --- a/dbms/src/Databases/DatabaseMySQL.h +++ b/dbms/src/Databases/DatabaseMySQL.h @@ -64,7 +64,7 @@ public: throw Exception("MySQL database engine does not support detach table.", ErrorCodes::NOT_IMPLEMENTED); } - void detachDictionary(const String &, const Context &, bool) override + void detachDictionary(const String &, const Context &) override { throw Exception("MySQL database engine does not support detach dictionary.", ErrorCodes::NOT_IMPLEMENTED); } @@ -90,7 +90,7 @@ public: throw Exception("MySQL database engine does not support attach table.", ErrorCodes::NOT_IMPLEMENTED); } - void attachDictionary(const String &, const Context &, bool) override + void attachDictionary(const String &, const Context &) override { throw Exception("MySQL database engine does not support attach dictionary.", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/dbms/src/Databases/DatabaseOnDisk.cpp b/dbms/src/Databases/DatabaseOnDisk.cpp index 03119743a86bc2e1885efa48f0a6dc0cef89a124..52bccc471ef85249bd1cfccc709a58bd8f8f9532 100644 --- a/dbms/src/Databases/DatabaseOnDisk.cpp +++ b/dbms/src/Databases/DatabaseOnDisk.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include +#include #include @@ -267,9 +269,11 @@ void DatabaseOnDisk::createDictionary( { const auto & settings = context.getSettingsRef(); - /** The code is based on the assumption that all threads share the same order of operations - * - creating the .sql.tmp file; - * - adding a dictionary to `dictionaries`; + /** The code is based on the assumption that all threads share the same order of operations: + * - create the .sql.tmp file; + * - add the dictionary to ExternalDictionariesLoader; + * - load the dictionary in case dictionaries_lazy_load == false; + * - attach the dictionary; * - rename .sql.tmp to .sql. */ @@ -278,17 +282,20 @@ void DatabaseOnDisk::createDictionary( if (database.isDictionaryExist(context, dictionary_name)) throw Exception("Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS); + /// A dictionary with the same full name could be defined in *.xml config files. + String full_name = database.getDatabaseName() + "." + dictionary_name; + auto & external_loader = const_cast(context.getExternalDictionariesLoader()); + if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST) + throw Exception("Dictionary " + backQuote(full_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS); + if (database.isTableExist(context, dictionary_name)) throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name); String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp"; - String statement; + String statement = getObjectDefinitionFromCreateQuery(query); { - statement = getObjectDefinitionFromCreateQuery(query); - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. WriteBufferFromFile out(dictionary_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); writeString(statement, out); @@ -298,27 +305,48 @@ void DatabaseOnDisk::createDictionary( out.close(); } - try + bool succeeded = false; + SCOPE_EXIT({ + if (!succeeded) + Poco::File(dictionary_metadata_tmp_path).remove(); + }); + + /// Add a temporary repository containing the dictionary. + /// We need this temp repository to try loading the dictionary before actually attaching it to the database. + static std::atomic counter = 0; + String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " " + + std::to_string(++counter); + external_loader.addConfigRepository( + temp_repository_name, + std::make_unique( + std::vector{std::pair{dictionary_metadata_tmp_path, + getDictionaryConfigurationFromAST(query->as(), database.getDatabaseName())}})); + SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); }); + + bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true); + if (!lazy_load) { - /// Do not load it now because we want more strict loading - database.attachDictionary(dictionary_name, context, false); - /// Load dictionary - bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true); - String dict_name = database.getDatabaseName() + "." + dictionary_name; - context.getExternalDictionariesLoader().addDictionaryWithConfig( - dict_name, database.getDatabaseName(), query->as(), !lazy_load); - - /// If it was ATTACH query and file with dictionary metadata already exist - /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. - Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path); - - } - catch (...) - { - database.detachDictionary(dictionary_name, context); - Poco::File(dictionary_metadata_tmp_path).remove(); - throw; + /// loadStrict() is called here to force loading the dictionary, wait until the loading is finished, + /// and throw an exception if the loading is failed. + external_loader.loadStrict(full_name); } + + database.attachDictionary(dictionary_name, context); + SCOPE_EXIT({ + if (!succeeded) + database.detachDictionary(dictionary_name, context); + }); + + /// If it was ATTACH query and file with dictionary metadata already exist + /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. + Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path); + + /// ExternalDictionariesLoader doesn't know we renamed the metadata path. + /// So we have to manually call reloadConfig() here. + external_loader.reloadConfig(database.getDatabaseName(), full_name); + + /// Everything's ok. + succeeded = true; } @@ -362,16 +390,18 @@ void DatabaseOnDisk::removeDictionary( database.detachDictionary(dictionary_name, context); String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name); - - try - { - Poco::File(dictionary_metadata_path).remove(); - } - catch (...) + if (Poco::File(dictionary_metadata_path).exists()) { - /// If remove was not possible for some reason - database.attachDictionary(dictionary_name, context); - throw; + try + { + Poco::File(dictionary_metadata_path).remove(); + } + catch (...) + { + /// If remove was not possible for some reason + database.attachDictionary(dictionary_name, context); + throw; + } } } diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 57138f8237f0198b6f5cf16649ac2af4aa201427..b4a900399a915f4eae297168ebfd0c4b1d2ce1c1 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -52,52 +52,68 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256; static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5; static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; -namespace -{ - -void loadObject( - Context & context, - const ASTCreateQuery & query, - DatabaseOrdinary & database, - const String database_data_path, - const String & database_name, - bool has_force_restore_data_flag) -try +namespace { - if (query.is_dictionary) + void tryAttachTable( + Context & context, + const ASTCreateQuery & query, + DatabaseOrdinary & database, + const String database_data_path, + const String & database_name, + bool has_force_restore_data_flag) { - String dictionary_name = query.table; - database.attachDictionary(dictionary_name, context, false); + assert(!query.is_dictionary); + try + { + String table_name; + StoragePtr table; + std::tie(table_name, table) + = createTableFromAST(query, database_name, database_data_path, context, has_force_restore_data_flag); + database.attachTable(table_name, table); + } + catch (const Exception & e) + { + throw Exception( + "Cannot attach table '" + query.table + "' from query " + serializeAST(query) + + ". Error: " + DB::getCurrentExceptionMessage(true), + e, + DB::ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA); + } } - else + + + void tryAttachDictionary( + Context & context, + const ASTCreateQuery & query, + DatabaseOrdinary & database) { - String table_name; - StoragePtr table; - std::tie(table_name, table) - = createTableFromAST(query, database_name, database_data_path, context, has_force_restore_data_flag); - database.attachTable(table_name, table); + assert(query.is_dictionary); + try + { + database.attachDictionary(query.table, context); + } + catch (const Exception & e) + { + throw Exception( + "Cannot create dictionary '" + query.table + "' from query " + serializeAST(query) + + ". Error: " + DB::getCurrentExceptionMessage(true), + e, + DB::ErrorCodes::CANNOT_CREATE_DICTIONARY_FROM_METADATA); + } } -} -catch (const Exception & e) -{ - throw Exception( - "Cannot create object '" + query.table + "' from query " + serializeAST(query) + ". Error: " + DB::getCurrentExceptionMessage(true), - e, DB::ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA); -} -void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch) -{ - if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) + void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch) { - LOG_INFO(log, std::fixed << std::setprecision(2) << processed * 100.0 / total << "%"); - watch.restart(); + if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) + { + LOG_INFO(log, std::fixed << std::setprecision(2) << processed * 100.0 / total << "%"); + watch.restart(); + } } } -} - DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context) : DatabaseWithOwnTablesBase(std::move(name_)) @@ -151,22 +167,20 @@ void DatabaseOrdinary::loadStoredObjects( std::atomic tables_processed{0}; std::atomic dictionaries_processed{0}; - auto loadOneObject = [&](const ASTCreateQuery & query) - { - loadObject(context, query, *this, getDataPath(), getDatabaseName(), has_force_restore_data_flag); - - /// Messages, so that it's not boring to wait for the server to load for a long time. - if (query.is_dictionary) - logAboutProgress(log, ++dictionaries_processed, total_dictionaries, watch); - else - logAboutProgress(log, ++tables_processed, total_tables, watch); - }; - ThreadPool pool(SettingMaxThreads().getAutoValue()); + /// Attach tables. for (const auto & name_with_query : file_names) { - pool.scheduleOrThrowOnError([&]() { loadOneObject(name_with_query.second->as()); }); + const auto & create_query = name_with_query.second->as(); + if (!create_query.is_dictionary) + pool.scheduleOrThrowOnError([&]() + { + tryAttachTable(context, create_query, *this, getDataPath(), getDatabaseName(), has_force_restore_data_flag); + + /// Messages, so that it's not boring to wait for the server to load for a long time. + logAboutProgress(log, ++tables_processed, total_tables, watch); + }); } pool.wait(); @@ -178,16 +192,19 @@ void DatabaseOrdinary::loadStoredObjects( auto dictionaries_repository = std::make_unique(shared_from_this(), context); auto & external_loader = context.getExternalDictionariesLoader(); external_loader.addConfigRepository(getDatabaseName(), std::move(dictionaries_repository)); - bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true); - auto filter = [this](const std::string & dictionary_name) -> bool + /// Attach dictionaries. + for (const auto & name_with_query : file_names) { - if (!startsWith(dictionary_name, name + "." /* db name */)) - return false; - LOG_INFO(log, "Loading dictionary " << backQuote(dictionary_name) << ", for database " << backQuote(name)); - return true; - }; - external_loader.reload(filter, !lazy_load); + auto create_query = name_with_query.second->as(); + if (create_query.is_dictionary) + { + tryAttachDictionary(context, create_query, *this); + + /// Messages, so that it's not boring to wait for the server to load for a long time. + logAboutProgress(log, ++dictionaries_processed, total_dictionaries, watch); + } + } } diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index 2feda6fc2b3a1b70d24d93e26b26d4aa21549981..3322d9d56bbd9e8afc5fd8db45b1827d5d472f1c 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -161,19 +161,21 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) return res; } -void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name, const Context & context, bool reload) +void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name, const Context & context) { + String full_name = getDatabaseName() + "." + dictionary_name; { std::lock_guard lock(mutex); auto it = dictionaries.find(dictionary_name); if (it == dictionaries.end()) - throw Exception("Dictionary " + name + "." + dictionary_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); dictionaries.erase(it); } - if (reload) - context.getExternalDictionariesLoader().reload(getDatabaseName() + "." + dictionary_name); - + /// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed + /// and therefore it will unload the dictionary. + const auto & external_loader = context.getExternalDictionariesLoader(); + external_loader.reloadConfig(getDatabaseName(), full_name); } void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table) @@ -184,22 +186,19 @@ void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const Sto } -void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const Context & context, bool load) +void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const Context & context) { - const auto & external_loader = context.getExternalDictionariesLoader(); - String full_name = getDatabaseName() + "." + dictionary_name; { std::lock_guard lock(mutex); - auto status = external_loader.getCurrentStatus(full_name); - if (status != ExternalLoader::Status::NOT_EXIST || !dictionaries.emplace(dictionary_name).second) - throw Exception( - "Dictionary " + full_name + " already exists.", - ErrorCodes::DICTIONARY_ALREADY_EXISTS); + if (!dictionaries.emplace(dictionary_name).second) + throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS); } - if (load) - external_loader.reload(full_name, true); + /// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added + /// and in case `dictionaries_lazy_load == false` it will load the dictionary. + const auto & external_loader = context.getExternalDictionariesLoader(); + external_loader.reloadConfig(getDatabaseName(), full_name); } void DatabaseWithOwnTablesBase::shutdown() diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h index f07136aa927efab97d20940a91e0b189a1146d2d..b277e8cd3d186666e9c32abae1059df0771fd6ca 100644 --- a/dbms/src/Databases/DatabasesCommon.h +++ b/dbms/src/Databases/DatabasesCommon.h @@ -33,11 +33,11 @@ public: void attachTable(const String & table_name, const StoragePtr & table) override; - void attachDictionary(const String & name, const Context & context, bool reload) override; + void attachDictionary(const String & name, const Context & context) override; StoragePtr detachTable(const String & table_name) override; - void detachDictionary(const String & name, const Context & context, bool reload) override; + void detachDictionary(const String & name, const Context & context) override; DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override; diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index ac1ccc096d5da9b8a542537f35abb050fd38e834..6b9d29b070ea8a0322551af90e92792d44294d19 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -165,14 +165,14 @@ public: virtual void attachTable(const String & name, const StoragePtr & table) = 0; /// Add dictionary to the database, but do not add it to the metadata. The database may not support this method. - /// load is false when we starting up and lazy_load is true, so we don't want to load dictionaries synchronously. - virtual void attachDictionary(const String & name, const Context & context, bool reload = true) = 0; + /// If dictionaries_lazy_load is false it also starts loading the dictionary asynchronously. + virtual void attachDictionary(const String & name, const Context & context) = 0; /// Forget about the table without deleting it, and return it. The database may not support this method. virtual StoragePtr detachTable(const String & name) = 0; - /// Forget about the dictionary without deleting it, and return it. The database may not support this method. - virtual void detachDictionary(const String & name, const Context & context, bool reload = true) = 0; + /// Forget about the dictionary without deleting it. The database may not support this method. + virtual void detachDictionary(const String & name, const Context & context) = 0; /// Rename the table and possibly move the table to another database. virtual void renameTable( diff --git a/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index f767440eb8b035c6adaec4df3408d3af12a78358..03e3a8c381211f49e92d0a6c8d28ea24fcea2609 100644 --- a/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -414,7 +414,7 @@ void checkPrimaryKey(const std::unordered_set & all_attrs, const Na } -DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query) +DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name) { checkAST(query); @@ -427,7 +427,8 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer AutoPtr name_element(xml_document->createElement("name")); current_dictionary->appendChild(name_element); - AutoPtr name(xml_document->createTextNode(query.database + "." + query.table)); + String full_name = (!database_name.empty() ? database_name : query.database) + "." + query.table; + AutoPtr name(xml_document->createTextNode(full_name)); name_element->appendChild(name); AutoPtr structure_element(xml_document->createElement("structure")); diff --git a/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h b/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h index bb48765c492a572e0565f39584b34ad05d40440d..adfcd7f276880c0f826bbf4da7f34f6ae6300870 100644 --- a/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h +++ b/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h @@ -10,6 +10,6 @@ using DictionaryConfigurationPtr = Poco::AutoPtr -#include #include -#include namespace DB { @@ -28,15 +26,4 @@ void ExternalDictionariesLoader::addConfigRepository( ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name"}); } - -void ExternalDictionariesLoader::addDictionaryWithConfig( - const String & dictionary_name, const String & repo_name, const ASTCreateQuery & query, bool load_never_loading) const -{ - ExternalLoader::addObjectAndLoad( - dictionary_name, /// names are equal - dictionary_name, - repo_name, - getDictionaryConfigurationFromAST(query), - "dictionary", load_never_loading); -} } diff --git a/dbms/src/Interpreters/ExternalDictionariesLoader.h b/dbms/src/Interpreters/ExternalDictionariesLoader.h index ae2ffc8bcc8d722749ef9e14e70318b9d4b03a1a..4348e22e9acc8c1c79a952df61804aba2058db35 100644 --- a/dbms/src/Interpreters/ExternalDictionariesLoader.h +++ b/dbms/src/Interpreters/ExternalDictionariesLoader.h @@ -1,17 +1,14 @@ #pragma once #include -#include #include -#include -#include #include namespace DB { - class Context; +class IExternalLoaderConfigRepository; /// Manages user-defined dictionaries. class ExternalDictionariesLoader : public ExternalLoader @@ -36,14 +33,6 @@ public: const std::string & repository_name, std::unique_ptr config_repository); - /// Starts reloading of a specified object. - void addDictionaryWithConfig( - const String & dictionary_name, - const String & repo_name, - const ASTCreateQuery & query, - bool load_never_loading = false) const; - - protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config) const override; @@ -52,7 +41,6 @@ protected: friend class DatabaseDictionary; private: - Context & context; }; diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index c7c883fde7983265e3311d9d5f957c112607fbad..4cbd8848d71f7bfa50a5c738a117ec3bad015fd8 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -10,6 +10,8 @@ #include #include #include +#include +#include namespace DB @@ -39,10 +41,10 @@ struct LoadingGuardForAsyncLoad struct ExternalLoader::ObjectConfig { - String config_path; Poco::AutoPtr config; String key_in_config; String repository_name; + String path; }; @@ -57,226 +59,258 @@ public: } ~LoadablesConfigReader() = default; - void addConfigRepository( - const String & name, - std::unique_ptr repository, - const ExternalLoaderConfigSettings & settings) + using RepositoryPtr = std::unique_ptr; + + void addConfigRepository(const String & repository_name, RepositoryPtr repository, const ExternalLoaderConfigSettings & settings) { std::lock_guard lock{mutex}; - repositories.emplace(name, std::make_pair(std::move(repository), settings)); + RepositoryInfo repository_info{std::move(repository), settings, {}}; + repositories.emplace(repository_name, std::move(repository_info)); + need_collect_object_configs = true; } - void removeConfigRepository(const String & name) + RepositoryPtr removeConfigRepository(const String & repository_name) { std::lock_guard lock{mutex}; - repositories.erase(name); + auto it = repositories.find(repository_name); + if (it == repositories.end()) + return nullptr; + auto repository = std::move(it->second.repository); + repositories.erase(it); + need_collect_object_configs = true; + return repository; } using ObjectConfigsPtr = std::shared_ptr>; - - /// Reads configurations. + /// Reads all repositories. ObjectConfigsPtr read() { std::lock_guard lock(mutex); - // Check last modification times of files and read those files which are new or changed. - if (!readLoadablesInfos()) - return configs; // Nothing changed, so we can return the previous result. - - return collectConfigs(); + readRepositories(); + collectObjectConfigs(); + return object_configs; } - ObjectConfig updateLoadableInfo( - const String & external_name, - const String & object_name, - const String & repo_name, - const Poco::AutoPtr & config, - const String & key) + /// Reads only a specified repository. + /// This functions checks only a specified repository but returns configs from all repositories. + ObjectConfigsPtr read(const String & repository_name) { std::lock_guard lock(mutex); + readRepositories(repository_name); + collectObjectConfigs(); + return object_configs; + } - auto it = loadables_infos.find(object_name); - if (it == loadables_infos.end()) - { - LoadablesInfos loadable_info; - loadables_infos[object_name] = loadable_info; - } - auto & loadable_info = loadables_infos[object_name]; - ObjectConfig object_config{object_name, config, key, repo_name}; - bool found = false; - for (auto iter = loadable_info.configs.begin(); iter != loadable_info.configs.end(); ++iter) - { - if (iter->first == external_name) - { - iter->second = object_config; - found = true; - break; - } - } - - if (!found) - loadable_info.configs.emplace_back(external_name, object_config); - loadable_info.last_update_time = Poco::Timestamp{}; /// now - loadable_info.in_use = true; - return object_config; + /// Reads only a specified path from a specified repository. + /// This functions checks only a specified repository but returns configs from all repositories. + ObjectConfigsPtr read(const String & repository_name, const String & path) + { + std::lock_guard lock(mutex); + readRepositories(repository_name, path); + collectObjectConfigs(); + return object_configs; } private: - struct LoadablesInfos + struct FileInfo { Poco::Timestamp last_update_time = 0; - std::vector> configs; // Parsed loadable's contents. - bool in_use = true; // Whether the `LoadablesInfos` should be destroyed because the correspondent loadable is deleted. + std::vector> objects; // Parsed contents of the file. + bool in_use = true; // Whether the `FileInfo` should be destroyed because the correspondent file is deleted. }; - /// Collect current configurations - ObjectConfigsPtr collectConfigs() + struct RepositoryInfo { - // Generate new result. - auto new_configs = std::make_shared>(); - for (const auto & [path, loadable_info] : loadables_infos) - { - for (const auto & [name, config] : loadable_info.configs) - { - auto already_added_it = new_configs->find(name); - if (already_added_it != new_configs->end()) - { - const auto & already_added = already_added_it->second; - LOG_WARNING(log, path << ": " << type_name << " '" << name << "' is found " - << ((path == already_added.config_path) - ? ("twice in the same file") - : ("both in file '" + already_added.config_path + "' and '" + path + "'"))); - continue; - } - new_configs->emplace(name, config); - } - } - - configs = new_configs; - return configs; - } + RepositoryPtr repository; + ExternalLoaderConfigSettings settings; + std::unordered_map files; + }; - /// Read files and store them to the map ` loadables_infos`. - bool readLoadablesInfos() + /// Reads the repositories. + /// Checks last modification times of files and read those files which are new or changed. + void readRepositories(const std::optional & only_repository_name = {}, const std::optional & only_path = {}) { - bool changed = false; - - for (auto & name_and_loadable_info : loadables_infos) + Strings repository_names; + if (only_repository_name) { - LoadablesInfos & loadable_info = name_and_loadable_info.second; - loadable_info.in_use = false; + if (repositories.count(*only_repository_name)) + repository_names.push_back(*only_repository_name); } + else + boost::copy(repositories | boost::adaptors::map_keys, std::back_inserter(repository_names)); - for (const auto & [repo_name, repo_with_settings] : repositories) + for (const auto & repository_name : repository_names) { - const auto names = repo_with_settings.first->getAllLoadablesDefinitionNames(); - for (const auto & loadable_name : names) + auto & repository_info = repositories[repository_name]; + + for (auto & file_info : repository_info.files | boost::adaptors::map_values) + file_info.in_use = false; + + Strings existing_paths; + if (only_path) + { + if (repository_info.repository->exists(*only_path)) + existing_paths.push_back(*only_path); + } + else + boost::copy(repository_info.repository->getAllLoadablesDefinitionNames(), std::back_inserter(existing_paths)); + + for (const auto & path : existing_paths) { - auto it = loadables_infos.find(loadable_name); - if (it != loadables_infos.end()) + auto it = repository_info.files.find(path); + if (it != repository_info.files.end()) { - LoadablesInfos & loadable_info = it->second; - if (readLoadablesInfo(repo_name, *repo_with_settings.first, loadable_name, repo_with_settings.second, loadable_info)) - changed = true; + FileInfo & file_info = it->second; + if (readFileInfo(file_info, *repository_info.repository, path, repository_info.settings)) + need_collect_object_configs = true; } else { - LoadablesInfos loadable_info; - if (readLoadablesInfo(repo_name, *repo_with_settings.first, loadable_name, repo_with_settings.second, loadable_info)) + FileInfo file_info; + if (readFileInfo(file_info, *repository_info.repository, path, repository_info.settings)) { - loadables_infos.emplace(loadable_name, std::move(loadable_info)); - changed = true; + repository_info.files.emplace(path, std::move(file_info)); + need_collect_object_configs = true; } } } - } - std::vector deleted_names; - for (auto & [path, loadable_info] : loadables_infos) - if (!loadable_info.in_use) - deleted_names.emplace_back(path); - if (!deleted_names.empty()) - { - for (const String & deleted_name : deleted_names) - loadables_infos.erase(deleted_name); - changed = true; + Strings deleted_paths; + for (auto & [path, file_info] : repository_info.files) + { + if (file_info.in_use) + continue; + + if (only_path && (*only_path != path)) + continue; + + deleted_paths.emplace_back(path); + } + + if (!deleted_paths.empty()) + { + for (const String & deleted_path : deleted_paths) + repository_info.files.erase(deleted_path); + need_collect_object_configs = true; + } } - return changed; } - bool readLoadablesInfo( - const String & repo_name, + /// Reads a file, returns true if the file is new or changed. + bool readFileInfo( + FileInfo & file_info, IExternalLoaderConfigRepository & repository, - const String & object_name, - const ExternalLoaderConfigSettings & settings, - LoadablesInfos & loadable_info) const + const String & path, + const ExternalLoaderConfigSettings & settings) const { try { - if (object_name.empty() || !repository.exists(object_name)) + if (path.empty() || !repository.exists(path)) { - LOG_WARNING(log, "Config file '" + object_name + "' does not exist"); + LOG_WARNING(log, "Config file '" + path + "' does not exist"); return false; } - auto update_time_from_repository = repository.getUpdateTime(object_name); + auto update_time_from_repository = repository.getUpdateTime(path); /// Actually it can't be less, but for sure we check less or equal - if (update_time_from_repository <= loadable_info.last_update_time) + if (update_time_from_repository <= file_info.last_update_time) { - loadable_info.in_use = true; + file_info.in_use = true; return false; } - auto file_contents = repository.load(object_name); + auto file_contents = repository.load(path); /// get all objects' definitions Poco::Util::AbstractConfiguration::Keys keys; file_contents->keys(keys); /// for each object defined in repositories - std::vector> configs_from_file; + std::vector> object_configs_from_file; for (const auto & key : keys) { if (!startsWith(key, settings.external_config)) { if (!startsWith(key, "comment") && !startsWith(key, "include_from")) - LOG_WARNING(log, object_name << ": file contains unknown node '" << key << "', expected '" << settings.external_config << "'"); + LOG_WARNING(log, path << ": file contains unknown node '" << key << "', expected '" << settings.external_config << "'"); continue; } - String external_name = file_contents->getString(key + "." + settings.external_name); - if (external_name.empty()) + String object_name = file_contents->getString(key + "." + settings.external_name); + if (object_name.empty()) { - LOG_WARNING(log, object_name << ": node '" << key << "' defines " << type_name << " with an empty name. It's not allowed"); + LOG_WARNING(log, path << ": node '" << key << "' defines " << type_name << " with an empty name. It's not allowed"); continue; } - configs_from_file.emplace_back(external_name, ObjectConfig{object_name, file_contents, key, repo_name}); + object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}}); } - loadable_info.configs = std::move(configs_from_file); - loadable_info.last_update_time = update_time_from_repository; - loadable_info.in_use = true; + file_info.objects = std::move(object_configs_from_file); + file_info.last_update_time = update_time_from_repository; + file_info.in_use = true; return true; } catch (...) { - tryLogCurrentException(log, "Failed to load config for dictionary '" + object_name + "'"); + tryLogCurrentException(log, "Failed to load config file '" + path + "'"); return false; } } + /// Builds a map of current configurations of objects. + void collectObjectConfigs() + { + if (!need_collect_object_configs) + return; + need_collect_object_configs = false; + + // Generate new result. + auto new_configs = std::make_shared>(); + + for (const auto & [repository_name, repository_info] : repositories) + { + for (const auto & [path, file_info] : repository_info.files) + { + for (const auto & [object_name, object_config] : file_info.objects) + { + auto already_added_it = new_configs->find(object_name); + if (already_added_it == new_configs->end()) + { + auto & new_config = new_configs->emplace(object_name, object_config).first->second; + new_config.repository_name = repository_name; + new_config.path = path; + } + else + { + const auto & already_added = already_added_it->second; + if (!startsWith(repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) && + !startsWith(already_added.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX)) + { + LOG_WARNING( + log, + type_name << " '" << object_name << "' is found " + << (((path == already_added.path) && repository_name == already_added.repository_name) + ? ("twice in the same file '" + path + "'") + : ("both in file '" + already_added.path + "' and '" + path + "'"))); + } + } + } + } + } + + object_configs = new_configs; + } const String type_name; Logger * log; std::mutex mutex; - using RepositoryPtr = std::unique_ptr; - using RepositoryWithSettings = std::pair; - std::unordered_map repositories; - ObjectConfigsPtr configs; - std::unordered_map loadables_infos; + std::unordered_map repositories; + ObjectConfigsPtr object_configs; + bool need_collect_object_configs = false; }; @@ -338,10 +372,11 @@ public: else { const auto & new_config = new_config_it->second; - if (!isSameConfiguration(*info.object_config.config, info.object_config.key_in_config, *new_config.config, new_config.key_in_config)) + bool config_is_same = isSameConfiguration(*info.object_config.config, info.object_config.key_in_config, *new_config.config, new_config.key_in_config); + info.object_config = new_config; + if (!config_is_same) { /// Configuration has been changed. - info.object_config = new_config; info.config_changed = true; if (info.wasLoading()) @@ -376,12 +411,6 @@ public: event.notify_all(); } - void setSingleObjectConfigurationWithoutLoading(const String & external_name, const ObjectConfig & config) - { - std::lock_guard lock{mutex}; - infos.emplace(external_name, Info{config}); - } - /// Sets whether all the objects from the configuration should be always loaded (even if they aren't used). void enableAlwaysLoadEverything(bool enable) { @@ -662,7 +691,7 @@ private: result.exception = exception; result.loading_start_time = loading_start_time; result.loading_duration = loadingDuration(); - result.origin = object_config.config_path; + result.origin = object_config.path; result.repository_name = object_config.repository_name; return result; } @@ -1091,12 +1120,14 @@ void ExternalLoader::addConfigRepository( const ExternalLoaderConfigSettings & config_settings) { config_files_reader->addConfigRepository(repository_name, std::move(config_repository), config_settings); - loading_dispatcher->setConfiguration(config_files_reader->read()); + reloadConfig(repository_name); } -void ExternalLoader::removeConfigRepository(const std::string & repository_name) +std::unique_ptr ExternalLoader::removeConfigRepository(const std::string & repository_name) { - config_files_reader->removeConfigRepository(repository_name); + auto repository = config_files_reader->removeConfigRepository(repository_name); + reloadConfig(repository_name); + return repository; } void ExternalLoader::enableAlwaysLoadEverything(bool enable) @@ -1189,40 +1220,36 @@ void ExternalLoader::load(Loadables & loaded_objects, Duration timeout) const void ExternalLoader::reload(const String & name, bool load_never_loading) const { - auto configs = config_files_reader->read(); - loading_dispatcher->setConfiguration(configs); + reloadConfig(); loading_dispatcher->reload(name, load_never_loading); } void ExternalLoader::reload(bool load_never_loading) const { - loading_dispatcher->setConfiguration(config_files_reader->read()); + reloadConfig(); loading_dispatcher->reload(load_never_loading); } void ExternalLoader::reload(const FilterByNameFunction & filter_by_name, bool load_never_loading) const { - loading_dispatcher->setConfiguration(config_files_reader->read()); + reloadConfig(); loading_dispatcher->reload(filter_by_name, load_never_loading); } -void ExternalLoader::addObjectAndLoad( - const String & name, - const String & external_name, - const String & repo_name, - const Poco::AutoPtr & config, - const String & key, - bool load_never_loading) const +void ExternalLoader::reloadConfig() const { - auto object_config = config_files_reader->updateLoadableInfo(external_name, name, repo_name, config, key); - loading_dispatcher->setSingleObjectConfigurationWithoutLoading(external_name, object_config); - LoadablePtr loaded_object; - if (load_never_loading) - loading_dispatcher->loadStrict(name, loaded_object); - else - loading_dispatcher->load(name, loaded_object, Duration::zero()); + loading_dispatcher->setConfiguration(config_files_reader->read()); } +void ExternalLoader::reloadConfig(const String & repository_name) const +{ + loading_dispatcher->setConfiguration(config_files_reader->read(repository_name)); +} + +void ExternalLoader::reloadConfig(const String & repository_name, const String & path) const +{ + loading_dispatcher->setConfiguration(config_files_reader->read(repository_name, path)); +} ExternalLoader::LoadablePtr ExternalLoader::createObject( const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index 16cdbd449f21463cd09e5fe5806882a59cd9181d..f7089ef0974b1e65ff457532ef60153bd06b225d 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -87,7 +87,7 @@ public: const ExternalLoaderConfigSettings & config_settings); /// Removes a repository which were used to read configurations. - void removeConfigRepository(const std::string & repository_name); + std::unique_ptr removeConfigRepository(const std::string & repository_name); /// Sets whether all the objects from the configuration should be always loaded (even those which are never used). void enableAlwaysLoadEverything(bool enable); @@ -128,15 +128,18 @@ public: /// Tries to finish loading of a specified object during the timeout. /// Returns nullptr if the loading is unsuccessful or if there is no such object. void load(const String & name, LoadablePtr & loaded_object, Duration timeout = NO_TIMEOUT) const; + void load(const String & name) const { LoadablePtr object; load(name, object, Duration::zero()); } LoadablePtr loadAndGet(const String & name, Duration timeout = NO_TIMEOUT) const { LoadablePtr object; load(name, object, timeout); return object; } LoadablePtr tryGetLoadable(const String & name) const { return loadAndGet(name); } /// Tries to finish loading of a specified object during the timeout. /// Throws an exception if the loading is unsuccessful or if there is no such object. void loadStrict(const String & name, LoadablePtr & loaded_object) const; + void loadStrict(const String & name) const { LoadablePtr object; loadStrict(name, object); } LoadablePtr getLoadable(const String & name) const { LoadablePtr object; loadStrict(name, object); return object; } /// Tries to finish loading of the objects for which the specified function returns true. + void load(const FilterByNameFunction & filter_by_name) const { Loadables objects; load(filter_by_name, objects, Duration::zero()); } void load(const FilterByNameFunction & filter_by_name, Loadables & loaded_objects, Duration timeout = NO_TIMEOUT) const; void load(const FilterByNameFunction & filter_by_name, LoadResults & load_results, Duration timeout = NO_TIMEOUT) const; Loadables loadAndGet(const FilterByNameFunction & filter_by_name, Duration timeout = NO_TIMEOUT) const { Loadables loaded_objects; load(filter_by_name, loaded_objects, timeout); return loaded_objects; } @@ -160,18 +163,18 @@ public: /// The function can either skip them (false) or load for the first time (true). void reload(const FilterByNameFunction & filter_by_name, bool load_never_loading = false) const; + /// Reloads all config repositories. + void reloadConfig() const; + + /// Reloads only a specified config repository. + void reloadConfig(const String & repository_name) const; + + /// Reload only a specified path in a specified config repository. + void reloadConfig(const String & repository_name, const String & path) const; + protected: virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const = 0; - /// Reload object with already parsed configuration - void addObjectAndLoad( - const String & name, /// name of dictionary - const String & external_name, /// name of source (example xml-file, may contain more than dictionary) - const String & repo_name, /// name of repository (database name, or all xml files) - const Poco::AutoPtr & config, - const String & key_in_config, /// key where we can start search of loadables (, , etc) - bool load_never_loading = false) const; - private: struct ObjectConfig; diff --git a/dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.cpp b/dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.cpp new file mode 100644 index 0000000000000000000000000000000000000000..16c8a3aa59c3f1cb8f0ca9b31e21ac2fc9be8177 --- /dev/null +++ b/dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +ExternalLoaderPresetConfigRepository::ExternalLoaderPresetConfigRepository(const std::vector> & preset_) +{ + boost::range::copy(preset_, std::inserter(preset, preset.end())); +} + +ExternalLoaderPresetConfigRepository::~ExternalLoaderPresetConfigRepository() = default; + +std::set ExternalLoaderPresetConfigRepository::getAllLoadablesDefinitionNames() const +{ + std::set paths; + boost::range::copy(preset | boost::adaptors::map_keys, std::inserter(paths, paths.end())); + return paths; +} + +bool ExternalLoaderPresetConfigRepository::exists(const String& path) const +{ + return preset.count(path); +} + +Poco::Timestamp ExternalLoaderPresetConfigRepository::getUpdateTime(const String & path) +{ + if (!exists(path)) + throw Exception("Loadable " + path + " not found", ErrorCodes::BAD_ARGUMENTS); + return creation_time; +} + +/// May contain definition about several entities (several dictionaries in one .xml file) +LoadablesConfigurationPtr ExternalLoaderPresetConfigRepository::load(const String & path) const +{ + auto it = preset.find(path); + if (it == preset.end()) + throw Exception("Loadable " + path + " not found", ErrorCodes::BAD_ARGUMENTS); + return it->second; +} + +} diff --git a/dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.h b/dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.h new file mode 100644 index 0000000000000000000000000000000000000000..b35209a7fb9d6153d1acc1018dadfc7a1034fa87 --- /dev/null +++ b/dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ +/// A config repository filled with preset loadables used by ExternalLoader. +class ExternalLoaderPresetConfigRepository : public IExternalLoaderConfigRepository +{ +public: + ExternalLoaderPresetConfigRepository(const std::vector> & preset_); + ~ExternalLoaderPresetConfigRepository() override; + + std::set getAllLoadablesDefinitionNames() const override; + bool exists(const String & path) const override; + Poco::Timestamp getUpdateTime(const String & path) override; + LoadablesConfigurationPtr load(const String & path) const override; + +private: + std::unordered_map preset; + Poco::Timestamp creation_time; +}; + +} diff --git a/dbms/src/Interpreters/IExternalLoaderConfigRepository.cpp b/dbms/src/Interpreters/IExternalLoaderConfigRepository.cpp new file mode 100644 index 0000000000000000000000000000000000000000..968a9bca9de94b34eeaa4b4b787a58e877361bde --- /dev/null +++ b/dbms/src/Interpreters/IExternalLoaderConfigRepository.cpp @@ -0,0 +1,7 @@ +#include + + +namespace DB +{ +const char * IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX = "\xFF internal repo "; +} diff --git a/dbms/src/Interpreters/IExternalLoaderConfigRepository.h b/dbms/src/Interpreters/IExternalLoaderConfigRepository.h index 93cefe0a0d49a2d2b7e7f4b1fc229a84969b7d08..bcac36d9807b744ccdba6d2b4b13267423057576 100644 --- a/dbms/src/Interpreters/IExternalLoaderConfigRepository.h +++ b/dbms/src/Interpreters/IExternalLoaderConfigRepository.h @@ -36,6 +36,8 @@ public: virtual LoadablesConfigurationPtr load(const std::string & loadable_definition_name) const = 0; virtual ~IExternalLoaderConfigRepository() = default; + + static const char * INTERNAL_REPOSITORY_NAME_PREFIX; }; using ExternalLoaderConfigRepositoryPtr = std::unique_ptr; diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp index 73896c10e1bcdc0a2a893c8b653d9c32907baa0c..12c882dee9d6b642a7c5520fc9ee5a1546050932 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp @@ -50,14 +50,21 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con const auto & external_dictionaries = context.getExternalDictionariesLoader(); for (const auto & [dict_name, load_result] : external_dictionaries.getCurrentLoadResults()) { + if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX)) + continue; + size_t i = 0; + String database; + String short_name = dict_name; - res_columns[i++]->insert(load_result.repository_name); - if (!load_result.repository_name.empty()) - res_columns[i++]->insert(dict_name.substr(load_result.repository_name.length() + 1)); - else - res_columns[i++]->insert(dict_name); + if (!load_result.repository_name.empty() && startsWith(dict_name, load_result.repository_name + ".")) + { + database = load_result.repository_name; + short_name = dict_name.substr(load_result.repository_name.length() + 1); + } + res_columns[i++]->insert(database); + res_columns[i++]->insert(short_name); res_columns[i++]->insert(static_cast(load_result.status)); res_columns[i++]->insert(load_result.origin);