diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a5d4bd87a7a849090bf9ecf48ca618b065597734..a11dd25d8e3217a571a8701464ca4f1a0de1c80e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1217,6 +1217,9 @@ void MergeTreeData::clearOldWriteAheadLogs() void MergeTreeData::clearEmptyParts() { + if (!getSettings()->remove_empty_parts) + return; + auto parts = getDataPartsVector(); for (const auto & part : parts) { diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index fe97298fe75b2093d47b57ef42c6a14620e723d3..6f0401e061b90424b138799d7d5e56b4fb552495 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -105,6 +105,7 @@ struct Settings; M(UInt64, concurrent_part_removal_threshold, 100, "Activate concurrent part removal (see 'max_part_removal_threads') only if the number of inactive data parts is at least this.", 0) \ M(String, storage_policy, "default", "Name of storage disk policy", 0) \ M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ + M(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm", 0) \ \ /** Settings for testing purposes */ \ M(Bool, randomize_part_type, false, "For testing purposes only. Randomizes part type between wide and compact", 0) \ diff --git a/tests/queries/0_stateless/00933_alter_ttl.reference b/tests/queries/0_stateless/00933_alter_ttl.reference index 545f5644e944dda80f28538943ef39670131d427..ca46a6e70ce54855d91ed3b7ae76725ab894bbc5 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.reference +++ b/tests/queries/0_stateless/00933_alter_ttl.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.ttl\n(\n `d` Date,\n `a` Int32\n)\nENGINE = MergeTree\nPARTITION BY toDayOfMonth(d)\nORDER BY a\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl\n(\n `d` Date,\n `a` Int32\n)\nENGINE = MergeTree\nPARTITION BY toDayOfMonth(d)\nORDER BY a\nTTL d + toIntervalDay(1)\nSETTINGS remove_empty_parts = 0, index_granularity = 8192 2100-10-10 3 2100-10-10 4 d Date diff --git a/tests/queries/0_stateless/00933_alter_ttl.sql b/tests/queries/0_stateless/00933_alter_ttl.sql index 1aacb9a669485505c35a50a8478c0c01f8eaa842..f3f6858cbb44a0cf6e0177069f0dab0bcc5cd5a0 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.sql +++ b/tests/queries/0_stateless/00933_alter_ttl.sql @@ -2,14 +2,13 @@ set send_logs_level = 'fatal'; drop table if exists ttl; -create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d); +create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d) settings remove_empty_parts = 0; alter table ttl modify ttl d + interval 1 day; show create table ttl; insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3); insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4); -select sleep(1) format Null; -- wait if very fast merge happen optimize table ttl partition 10 final; select * from ttl order by d; @@ -18,7 +17,7 @@ alter table ttl modify ttl a; -- { serverError 450 } drop table if exists ttl; -create table ttl (d Date, a Int) engine = MergeTree order by tuple() partition by toDayOfMonth(d); +create table ttl (d Date, a Int) engine = MergeTree order by tuple() partition by toDayOfMonth(d) settings remove_empty_parts = 0; alter table ttl modify column a Int ttl d + interval 1 day; desc table ttl; alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 43 } diff --git a/tests/queries/0_stateless/00933_ttl_simple.sql b/tests/queries/0_stateless/00933_ttl_simple.sql index aa8b33b29997ec2a7b4bcef82b17983c2734ac0f..2bf686822d5fd5900bf10d2bfdca0f7c5cc67468 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.sql +++ b/tests/queries/0_stateless/00933_ttl_simple.sql @@ -11,7 +11,9 @@ select a, b from ttl_00933_1; drop table if exists ttl_00933_1; -create table ttl_00933_1 (d DateTime, a Int, b Int) engine = MergeTree order by toDate(d) partition by tuple() ttl d + interval 1 second; +create table ttl_00933_1 (d DateTime, a Int, b Int) + engine = MergeTree order by toDate(d) partition by tuple() ttl d + interval 1 second + settings remove_empty_parts = 0; insert into ttl_00933_1 values (now(), 1, 2); insert into ttl_00933_1 values (now(), 3, 4); insert into ttl_00933_1 values (now() + 1000, 5, 6); @@ -30,7 +32,9 @@ select * from ttl_00933_1 order by d; drop table if exists ttl_00933_1; -create table ttl_00933_1 (d DateTime, a Int) engine = MergeTree order by tuple() partition by tuple() ttl d + interval 1 day; +create table ttl_00933_1 (d DateTime, a Int) + engine = MergeTree order by tuple() partition by tuple() ttl d + interval 1 day + settings remove_empty_parts = 0; insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_1 values (toDateTime('2100-10-10 00:00:00'), 3); @@ -39,7 +43,9 @@ select * from ttl_00933_1 order by d; drop table if exists ttl_00933_1; -create table ttl_00933_1 (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d) ttl d + interval 1 day; +create table ttl_00933_1 (d Date, a Int) + engine = MergeTree order by a partition by toDayOfMonth(d) ttl d + interval 1 day + settings remove_empty_parts = 0; insert into ttl_00933_1 values (toDate('2000-10-10'), 1); insert into ttl_00933_1 values (toDate('2100-10-10'), 2); optimize table ttl_00933_1 final; diff --git a/tests/queries/0_stateless/00976_ttl_with_old_parts.sql b/tests/queries/0_stateless/00976_ttl_with_old_parts.sql index 0ed69850cf60e7f0661064fdcbcff0732afc2f92..60151595983945a9b664701e850e89661a2bbee4 100644 --- a/tests/queries/0_stateless/00976_ttl_with_old_parts.sql +++ b/tests/queries/0_stateless/00976_ttl_with_old_parts.sql @@ -1,6 +1,6 @@ drop table if exists ttl; -create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d); +create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d) settings remove_empty_parts = 0; insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3); diff --git a/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh b/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh index 3405b85089c408cf44acbeeaa866eda70fe8f56d..5473185af3dc0233c955c7198abe562ca8b047a0 100755 --- a/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh +++ b/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh @@ -16,7 +16,7 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE table_with_empty_part ENGINE = MergeTree() ORDER BY id PARTITION BY id -SETTINGS vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0 +SETTINGS vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0, remove_empty_parts = 0 " diff --git a/tests/queries/0_stateless/01047_simple_aggregate_sizes_of_columns_bug.sql b/tests/queries/0_stateless/01047_simple_aggregate_sizes_of_columns_bug.sql index 7213aaa16c7e8f30e91a9226d0c69a271b467835..a2ea13b26fc04c07d10fd6b7ad22395dca5c7a63 100644 --- a/tests/queries/0_stateless/01047_simple_aggregate_sizes_of_columns_bug.sql +++ b/tests/queries/0_stateless/01047_simple_aggregate_sizes_of_columns_bug.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS column_size_bug; -CREATE TABLE column_size_bug (date_time DateTime, value SimpleAggregateFunction(sum,UInt64)) ENGINE = AggregatingMergeTree PARTITION BY toStartOfInterval(date_time, INTERVAL 1 DAY) ORDER BY (date_time); +CREATE TABLE column_size_bug (date_time DateTime, value SimpleAggregateFunction(sum,UInt64)) ENGINE = AggregatingMergeTree PARTITION BY toStartOfInterval(date_time, INTERVAL 1 DAY) ORDER BY (date_time) SETTINGS remove_empty_parts = 0; INSERT INTO column_size_bug VALUES(now(),1); INSERT INTO column_size_bug VALUES(now(),1);