Merge pull request #8242 from achimbab/bugfix_bloomfilter

Fixed two bugs in the bloom_filter index.

dbms/src/Interpreters/BloomFilterHash.h

@@ -85,8 +85,16 @@ struct BloomFilterHash
                 throw Exception("Unexpected type " + data_type->getName() + " of bloom filter index.", ErrorCodes::LOGICAL_ERROR);
 
             const auto & offsets = array_col->getOffsets();
-            size_t offset = (pos == 0) ? 0 : offsets[pos - 1];
-            limit = std::max(array_col->getData().size() - offset, limit);
+            limit = offsets[pos + limit - 1] - offsets[pos - 1];    /// PaddedPODArray allows access on index -1.
+            pos = offsets[pos - 1];
+
+            if (limit == 0)
+            {
+                auto index_column = ColumnUInt64::create(1);
+                ColumnUInt64::Container & index_column_vec = index_column->getData();
+                index_column_vec[0] = 0;
+                return index_column;
+            }
         }
 
         const ColumnPtr actual_col = BloomFilter::getPrimitiveColumn(column);

dbms/tests/performance/bloom_filter.xml

+<test>
+    <type>once</type>
+
+    <stop_conditions>
+        <all_of>
+            <total_time_ms>30000</total_time_ms>
+        </all_of>
+    </stop_conditions>
+
+    <create_query>DROP TABLE IF EXISTS test_bf</create_query>
+    <create_query>SET allow_experimental_data_skipping_indices = 1</create_query>
+    <create_query>CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id</create_query>
+    <query>SYSTEM STOP MERGES</query>
+    <query>INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM system.numbers LIMIT 3000000</query>
+    <query>SYSTEM START MERGES</query>
+    <drop_query>DROP TABLE IF EXISTS test_bf</drop_query>
+</test>

dbms/tests/queries/0_stateless/00945_bloom_filter_index.reference

@@ -175,3 +175,6 @@
 1
 1
 1
+5000
+5000
+5000

dbms/tests/queries/0_stateless/00945_bloom_filter_index.sql

@@ -245,3 +245,21 @@ SELECT COUNT() FROM test.bloom_filter_array_lc_null_types_test WHERE has(str, '1
 SELECT COUNT() FROM test.bloom_filter_array_lc_null_types_test WHERE has(fixed_string, toFixedString('100', 5));
 
 DROP TABLE IF EXISTS test.bloom_filter_array_lc_null_types_test;
+
+DROP TABLE IF EXISTS test.bloom_filter_array_offsets_lc_str;
+CREATE TABLE test.bloom_filter_array_offsets_lc_str (order_key int, str Array(LowCardinality((String))), INDEX idx str TYPE bloom_filter(1.01) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024;
+INSERT INTO test.bloom_filter_array_offsets_lc_str SELECT number AS i, if(i%2, ['value'], []) FROM system.numbers LIMIT 10000;
+SELECT count() FROM test.bloom_filter_array_offsets_lc_str WHERE has(str, 'value');
+DROP TABLE IF EXISTS test.bloom_filter_array_offsets_lc_str;
+
+DROP TABLE IF EXISTS test.bloom_filter_array_offsets_str;
+CREATE TABLE test.bloom_filter_array_offsets_str (order_key int, str Array(String), INDEX idx str TYPE bloom_filter(1.01) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024;
+INSERT INTO test.bloom_filter_array_offsets_str SELECT number AS i, if(i%2, ['value'], []) FROM system.numbers LIMIT 10000;
+SELECT count() FROM test.bloom_filter_array_offsets_str WHERE has(str, 'value');
+DROP TABLE IF EXISTS test.bloom_filter_array_offsets_str;
+
+DROP TABLE IF EXISTS test.bloom_filter_array_offsets_i;
+CREATE TABLE test.bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX idx i TYPE bloom_filter(1.01) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024;
+INSERT INTO test.bloom_filter_array_offsets_i SELECT number AS i, if(i%2, [99999], []) FROM system.numbers LIMIT 10000;
+SELECT count() FROM test.bloom_filter_array_offsets_i WHERE has(i, 99999);
+DROP TABLE IF EXISTS test.bloom_filter_array_offsets_i;

dbms/tests/queries/0_stateless/01045_bloom_filter_null_array.reference

+7
+1
+2
+1
+0

dbms/tests/queries/0_stateless/01045_bloom_filter_null_array.sql

+SET allow_experimental_data_skipping_indices = 1;
+
+
+DROP TABLE IF EXISTS test.bloom_filter_null_array;
+
+CREATE TABLE test.bloom_filter_null_array (v Array(LowCardinality(Nullable(String))), INDEX idx v TYPE bloom_filter(0.1) GRANULARITY 1) ENGINE = MergeTree() ORDER BY v;
+
+INSERT INTO test.bloom_filter_null_array VALUES ([]);
+INSERT INTO test.bloom_filter_null_array VALUES (['1', '2']) ([]) ([]);
+INSERT INTO test.bloom_filter_null_array VALUES ([]) ([]) (['2', '3']);
+
+SELECT COUNT() FROM test.bloom_filter_null_array;
+SELECT COUNT() FROM test.bloom_filter_null_array WHERE has(v, '1');
+SELECT COUNT() FROM test.bloom_filter_null_array WHERE has(v, '2');
+SELECT COUNT() FROM test.bloom_filter_null_array WHERE has(v, '3');
+SELECT COUNT() FROM test.bloom_filter_null_array WHERE has(v, '4');
+
+DROP TABLE IF EXISTS test.bloom_filter_null_array;