diff --git a/dbms/include/DB/Common/HashTable/TwoLevelHashTable.h b/dbms/include/DB/Common/HashTable/TwoLevelHashTable.h index 2dd99f8cfe02bd0845d7aab5fbfa4082550d8283..916c27b7a156ac11d8e419e316bcbaddc64f85cb 100644 --- a/dbms/include/DB/Common/HashTable/TwoLevelHashTable.h +++ b/dbms/include/DB/Common/HashTable/TwoLevelHashTable.h @@ -95,6 +95,7 @@ public: { typename Source::const_iterator it = src.begin(); + /// Предполагается, что нулевой ключ (хранящийся отдельно) при итерировании идёт первым. if (it != src.end() && it.getPtr()->isZero(src)) { insert(*it); @@ -142,6 +143,8 @@ public: value_type & operator* () const { return *current_it; } value_type * operator->() const { return &*current_it; } + + Cell * getPtr() const { return current_it.getPtr(); } }; @@ -177,6 +180,8 @@ public: const value_type & operator* () const { return *current_it; } const value_type * operator->() const { return &*current_it; } + + const Cell * getPtr() const { return current_it.getPtr(); } }; @@ -205,6 +210,10 @@ public: std::pair res; emplace(Cell::getKey(x), res.first, res.second, hash_value); + + if (res.second) + res.first.getPtr()->setMapped(x); + return res; } diff --git a/dbms/include/DB/Common/VirtualColumnUtils.h b/dbms/include/DB/Common/VirtualColumnUtils.h index 0d873ce3cbab05628bef62b708daa8ac1cfb197c..a4848d5e17b63a44f3bda0e56498b0def459c73e 100644 --- a/dbms/include/DB/Common/VirtualColumnUtils.h +++ b/dbms/include/DB/Common/VirtualColumnUtils.h @@ -1,19 +1,18 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include + +#include +#include +#include + namespace DB { +class Context; + + namespace VirtualColumnUtils { diff --git a/dbms/include/DB/DataStreams/RemoteBlockInputStream.h b/dbms/include/DB/DataStreams/RemoteBlockInputStream.h index 8efbf9dfae59efd58f519bbc23e33d6f45eb5709..d31f1e68ea7e52856990880af08733b0bfcf1536 100644 --- a/dbms/include/DB/DataStreams/RemoteBlockInputStream.h +++ b/dbms/include/DB/DataStreams/RemoteBlockInputStream.h @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include diff --git a/dbms/include/DB/Interpreters/AggregationCommon.h b/dbms/include/DB/Interpreters/AggregationCommon.h index 5d06d11c03a774ae77344ae219735f2c951a3b6f..23c092da299845081c0a7b944a7efe822fb958d6 100644 --- a/dbms/include/DB/Interpreters/AggregationCommon.h +++ b/dbms/include/DB/Interpreters/AggregationCommon.h @@ -9,8 +9,6 @@ #include #include #include -#include -#include template <> diff --git a/dbms/include/DB/Interpreters/Aggregator.h b/dbms/include/DB/Interpreters/Aggregator.h index a1e2213d52cec37f981be218228ff61b8d80b59c..f45c30a6c55809f926f115178db3b904aa2ee49a 100644 --- a/dbms/include/DB/Interpreters/Aggregator.h +++ b/dbms/include/DB/Interpreters/Aggregator.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include diff --git a/dbms/src/Common/VirtualColumnUtils.cpp b/dbms/src/Common/VirtualColumnUtils.cpp index bec2bca6787116ced35662e16f1f765b8a8c9def..d7d69eaa293b540081d89f6ea47618bcb05ceeca 100644 --- a/dbms/src/Common/VirtualColumnUtils.cpp +++ b/dbms/src/Common/VirtualColumnUtils.cpp @@ -1,16 +1,12 @@ -#include - #include -#include -#include -#include -#include +#include #include #include #include #include -#include -#include + +#include + namespace DB { diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index a7dd3973a21c1d66735368e87430364adaab8787..35cbfbaaf36ff3dae9b9c34ffcab16876513b2d3 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -442,10 +442,7 @@ void NO_INLINE Aggregator::mergeTwoLevelDataImpl( } }; - /// future и packaged_task используются, чтобы исключения автоматически прокидывались в основной поток. - - std::vector> futures; - futures.reserve(Method::Data::NUM_BUCKETS); + /// packaged_task используются, чтобы исключения автоматически прокидывались в основной поток. std::vector> tasks; tasks.reserve(Method::Data::NUM_BUCKETS); @@ -453,7 +450,6 @@ void NO_INLINE Aggregator::mergeTwoLevelDataImpl( for (size_t bucket = 0; bucket < Method::Data::NUM_BUCKETS; ++bucket) { tasks.emplace_back(std::bind(merge_bucket, bucket, current_memory_tracker)); - futures.emplace_back(tasks.back().get_future()); if (thread_pool) thread_pool->schedule([bucket, &tasks] { tasks[bucket](); }); @@ -464,8 +460,8 @@ void NO_INLINE Aggregator::mergeTwoLevelDataImpl( if (thread_pool) thread_pool->wait(); - for (auto & future : futures) - future.get(); + for (auto & task : tasks) + task.get_future().get(); } @@ -635,16 +631,17 @@ bool Aggregator::executeOnBlock(Block & block, AggregatedDataVariants & result, /** Почему выбрано 30 000? Потому что при таком количестве элементов, в TwoLevelHashTable, * скорее всего, хватит места на все ключи, с размером таблицы по-умолчанию * (256 корзин по 256 ячеек, fill factor = 0.5) + * TODO Не конвертировать, если запрос выполняется в один поток. */ if (result.isConvertibleToTwoLevel() && result_size >= TWO_LEVEL_HASH_TABLE_THRESHOLD) result.convertToTwoLevel(); /// Проверка ограничений. - if (!no_more_keys && max_rows_to_group_by && result.size() > max_rows_to_group_by) + if (!no_more_keys && max_rows_to_group_by && result_size > max_rows_to_group_by) { if (group_by_overflow_mode == OverflowMode::THROW) - throw Exception("Limit for rows to GROUP BY exceeded: has " + toString(result.size()) + throw Exception("Limit for rows to GROUP BY exceeded: has " + toString(result_size) + " rows, maximum: " + toString(max_rows_to_group_by), ErrorCodes::TOO_MUCH_ROWS); else if (group_by_overflow_mode == OverflowMode::BREAK) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index e2367f44dcb4fd5f4b2532a32e2777fcfe21ed9f..df0175ef9251e0ad8fdf44444137071711cdd9d0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/Storages/StorageChunkMerger.cpp b/dbms/src/Storages/StorageChunkMerger.cpp index 7462efa9f206a6b163726abc9a346f014d9065a5..0c3b66615d7b46ff557bcfb714fc9d539a339233 100644 --- a/dbms/src/Storages/StorageChunkMerger.cpp +++ b/dbms/src/Storages/StorageChunkMerger.cpp @@ -14,7 +14,10 @@ #include #include #include +#include #include +#include +#include namespace DB diff --git a/dbms/src/Storages/StorageChunks.cpp b/dbms/src/Storages/StorageChunks.cpp index c3aad961a39532b01b1a87fc56e5e9d7370df272..6617e1788eb07028727868e78cf943956f435809 100644 --- a/dbms/src/Storages/StorageChunks.cpp +++ b/dbms/src/Storages/StorageChunks.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace DB diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index fcb99636408438bd2a2605efc992883b84d39c3c..309826e9059cb7a18b1d1cc332a0a0dd4ea70c14 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -1,8 +1,13 @@ #include +#include #include #include #include #include +#include +#include +#include + namespace DB { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 8f669e5d25d275be4f0346111a7f864e18c8f6c5..715aabf6011dbbb1ea525fc5527739d1658b597c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -8,8 +8,10 @@ #include #include #include +#include #include + namespace DB {