add tests for mutations cleaner [#CLICKHOUSE-3828]

dbms/tests/queries/0_stateless/00652_mergetree_mutations.lib

+#!/usr/bin/env bash
+
+function wait_for_mutation()
+{
+    local table=$1
+    local mutation_id=$2
+
+    for i in {1..100}
+    do
+        sleep 0.1
+        if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT is_done FROM system.mutations WHERE table='$table' AND mutation_id='$mutation_id'") -eq 1 ]]; then
+            break
+        fi
+
+        if [[ $i -eq 100 ]]; then
+            echo "Timed out while waiting for mutation to execute!"
+        fi
+
+    done
+}

dbms/tests/queries/0_stateless/00652_mergetree_mutations.reference

@@ -8,3 +8,6 @@ Query involving aliases should fail on submission
 mutation_1.txt	DELETE WHERE x = 1	['']	[1]	0	1
 mutation_5.txt	DELETE WHERE (x % 2) = 1	['']	[5]	0	1
 mutation_6.txt	DELETE WHERE s = \'d\'	['']	[6]	0	1
+*** Test mutations cleaner ***
+mutation_3.txt	DELETE WHERE x = 2	1
+mutation_4.txt	DELETE WHERE x = 3	1

dbms/tests/queries/0_stateless/00652_mergetree_mutations.sh

@@ -3,6 +3,8 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
+. $CURDIR/00652_mergetree_mutations.lib
+
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations"
 
 ${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations(d Date, x UInt32, s String, a UInt32 ALIAS x + 1) ENGINE MergeTree(d, intDiv(x, 10), 8192)"
@@ -31,18 +33,8 @@ ${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations DELETE WHERE s = 'd'"
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations(d, x, s) VALUES \
     ('2000-01-01', 5, 'e'), ('2000-02-01', 5, 'e')"
 
-# Wait until all mutations are done.
-for i in {1..100}
-do
-    sleep 0.1
-    if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT sum(is_done) FROM system.mutations WHERE table='mutations'") -eq 3 ]]; then
-        break
-    fi
-
-    if [[ $i -eq 100 ]]; then
-       echo "Timed out while waiting for mutations to execute!"
-    fi
-done
+# Wait until the last mutation is done.
+wait_for_mutation "mutations" "mutation_6.txt"
 
 # Check that the table contains only the data that should not be deleted.
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations ORDER BY d, x"
@@ -50,4 +42,31 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations ORDER BY d, x"
 ${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \
     FROM system.mutations WHERE table = 'mutations' ORDER BY mutation_id"
 
+
+${CLICKHOUSE_CLIENT} --query="SELECT '*** Test mutations cleaner ***'"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_cleaner"
+
+# Create a table with finished_mutations_to_keep = 2
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations_cleaner(x UInt32) ENGINE MergeTree ORDER BY x SETTINGS finished_mutations_to_keep = 2"
+
+# Insert some data
+${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_cleaner(x) VALUES (1), (2), (3), (4)"
+
+# Add some mutations and wait for their execution
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner DELETE WHERE x = 1"
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner DELETE WHERE x = 2"
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner DELETE WHERE x = 3"
+
+wait_for_mutation "mutations_cleaner" "mutation_4.txt"
+
+# Sleep and then do an INSERT to wakeup the background task that will clean up the old mutations
+sleep 1
+${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_cleaner(x) VALUES (4)"
+sleep 0.1
+
+# Check that the first mutation is cleaned
+${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, is_done FROM system.mutations WHERE table = 'mutations_cleaner' ORDER BY mutation_id"
+
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_cleaner"

dbms/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.reference

@@ -7,3 +7,7 @@ Query should fail 2
 0000000000	DELETE WHERE x = 1	[]	[]	0	1
 0000000001	DELETE WHERE (x % 2) = 1	['200001','200002']	[2,1]	0	1
 0000000002	DELETE WHERE s = \'d\'	['200001','200002']	[3,2]	0	1
+*** Test mutations cleaner ***
+0000000001	DELETE WHERE x = 2	1
+0000000002	DELETE WHERE x = 3	1
+0000000003	DELETE WHERE x = 4	0

dbms/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh

@@ -3,6 +3,8 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
+. $CURDIR/00652_mergetree_mutations.lib
+
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_r1"
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_r2"
 
@@ -31,18 +33,8 @@ ${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_r1 DELETE WHERE s = 'd'
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_r1(d, x, s) VALUES \
     ('2000-01-01', 5, 'e'), ('2000-02-01', 5, 'e')"
 
-# Wait until all mutations are done.
-for i in {1..100}
-do
-    sleep 0.1
-    if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT sum(is_done) FROM system.mutations WHERE table='mutations_r2'") -eq 3 ]]; then
-        break
-    fi
-
-    if [[ $i -eq 100 ]]; then
-       echo "Timed out while waiting for mutations to execute!"
-    fi
-done
+# Wait until the last mutation is done.
+wait_for_mutation "mutations_r2" "0000000002"
 
 # Check that the table contains only the data that should not be deleted.
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations_r2 ORDER BY d, x"
@@ -50,5 +42,44 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations_r2 ORDER BY d, x"
 ${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \
     FROM system.mutations WHERE table = 'mutations_r2' ORDER BY mutation_id"
 
+
+${CLICKHOUSE_CLIENT} --query="SELECT '*** Test mutations cleaner ***'"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_cleaner_r1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_cleaner_r2"
+
+# Create 2 replicas with finished_mutations_to_keep = 2
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations_cleaner_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/mutations_cleaner', 'r1') ORDER BY x SETTINGS \
+    finished_mutations_to_keep = 2,
+    cleanup_delay_period = 1,
+    cleanup_delay_period_random_add = 0"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations_cleaner_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/mutations_cleaner', 'r2') ORDER BY x SETTINGS \
+    finished_mutations_to_keep = 2,
+    cleanup_delay_period = 1,
+    cleanup_delay_period_random_add = 0"
+
+# Insert some data
+${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_cleaner_r1(x) VALUES (1), (2), (3), (4)"
+
+# Add some mutations and wait for their execution
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 1"
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 2"
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 3"
+
+wait_for_mutation "mutations_cleaner_r2" "0000000002"
+
+# Add another mutation and prevent its execution on the second replica
+${CLICKHOUSE_CLIENT} --query="SYSTEM STOP REPLICATION QUEUES test.mutations_cleaner_r2"
+${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 4"
+
+# Sleep for more than cleanup_delay_period
+sleep 1.5
+
+# Check that the first mutation is cleaned
+${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, is_done FROM system.mutations WHERE table = 'mutations_cleaner_r2' ORDER BY mutation_id"
+
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_r1"
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_r2"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_cleaner_r1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_cleaner_r2"