diff --git a/dbms/include/DB/Common/MemoryTracker.h b/dbms/include/DB/Common/MemoryTracker.h index b17862bf5be4ce1bb37d17c2a477024c0421e886..97992e7948b66e7fb00cdc733deff32c089742f1 100644 --- a/dbms/include/DB/Common/MemoryTracker.h +++ b/dbms/include/DB/Common/MemoryTracker.h @@ -86,3 +86,22 @@ public: * Таким образом, его нужно всего-лишь протащить во все потоки, в которых обрабатывается один запрос. */ extern __thread MemoryTracker * current_memory_tracker; + + +#include + +struct TemporarilyDisableMemoryTracker : private boost::noncopyable +{ + MemoryTracker * memory_tracker; + + TemporarilyDisableMemoryTracker() + { + memory_tracker = current_memory_tracker; + current_memory_tracker = nullptr; + } + + ~TemporarilyDisableMemoryTracker() + { + current_memory_tracker = memory_tracker; + } +}; diff --git a/dbms/include/DB/Interpreters/ExternalDictionaries.h b/dbms/include/DB/Interpreters/ExternalDictionaries.h index 67c79c5b20641015dc90d54c0c6c4c19ec11340f..139a0c941b4f75975361df38f47e3d424849e65e 100644 --- a/dbms/include/DB/Interpreters/ExternalDictionaries.h +++ b/dbms/include/DB/Interpreters/ExternalDictionaries.h @@ -107,21 +107,7 @@ public: /** During synchronous loading of external dictionaries at moment of query execution, * we should not use per query memory limit. */ - struct TemporarilyDisableMemoryTracker - { - MemoryTracker * memory_tracker; - - TemporarilyDisableMemoryTracker() - { - memory_tracker = current_memory_tracker; - current_memory_tracker = nullptr; - } - - ~TemporarilyDisableMemoryTracker() - { - current_memory_tracker = memory_tracker; - } - } temporarily_disable_memory_tracker; + TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker; reloadImpl(throw_on_error); } diff --git a/dbms/include/DB/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/include/DB/Storages/MergeTree/MergedBlockOutputStream.h index ff1916f3016c6f4e978626f0f93300e95f64a151..17b22048ad0d5b41c971f01837f1778304bd108e 100644 --- a/dbms/include/DB/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/include/DB/Storages/MergeTree/MergedBlockOutputStream.h @@ -442,20 +442,30 @@ private: } } - /// Пишем индекс. Индекс содержит значение Primary Key для каждой index_granularity строки. - for (size_t i = index_offset; i < rows; i += storage.index_granularity) { - if (storage.merging_params.mode != MergeTreeData::MergingParams::Unsorted) + /** While filling index (index_columns), disable memory tracker. + * Because memory is allocated here (maybe in context of INSERT query), + * but then freed in completely different place (while merging parts), where query memory_tracker is not available. + * And otherwise it will look like excessively growing memory consumption in context of query. + * (observed in long INSERT SELECTs) + */ + TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker; + + /// Пишем индекс. Индекс содержит значение Primary Key для каждой index_granularity строки. + for (size_t i = index_offset; i < rows; i += storage.index_granularity) { - for (size_t j = 0, size = primary_columns.size(); j < size; ++j) + if (storage.merging_params.mode != MergeTreeData::MergingParams::Unsorted) { - const IColumn & primary_column = *primary_columns[j].column.get(); - index_columns[j].get()->insertFrom(primary_column, i); - primary_columns[j].type.get()->serializeBinary(primary_column, i, *index_stream); + for (size_t j = 0, size = primary_columns.size(); j < size; ++j) + { + const IColumn & primary_column = *primary_columns[j].column.get(); + index_columns[j].get()->insertFrom(primary_column, i); + primary_columns[j].type.get()->serializeBinary(primary_column, i, *index_stream); + } } - } - ++marks_count; + ++marks_count; + } } size_t written_for_last_mark = (storage.index_granularity - index_offset + rows) % storage.index_granularity;