Merging: Fixed most of problems, actualized test #2260

dbms/src/Storages/IStorage.h

@@ -209,6 +209,7 @@ public:
     virtual void drop() {}
 
     /** Clear the table data and leave it empty.
+      * Must be called under lockForAlter.
       */
     virtual void truncate(const ASTPtr & /*query*/)
     {

dbms/src/Storages/StorageJoin.cpp

@@ -45,6 +45,7 @@ void StorageJoin::truncate(const ASTPtr &)
 {
     Poco::File(path).remove(true);
     Poco::File(path).createDirectories();
+    Poco::File(path + "tmp/").createDirectories();
 
     increment = 0;
     join = std::make_shared<Join>(key_names, key_names, false /* use_nulls */, SizeLimits(), kind, strictness);

dbms/src/Storages/StorageMaterializedView.cpp

@@ -209,7 +209,7 @@ void StorageMaterializedView::drop()
         DatabaseAndTableName(select_database_name, select_table_name),
         DatabaseAndTableName(database_name, table_name));
 
-    if (has_inner_table)
+    if (has_inner_table && tryGetTargetTable())
         executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name);
 }
 
@@ -270,10 +270,29 @@ StoragePtr StorageMaterializedView::getTargetTable() const
     return global_context.getTable(target_database_name, target_table_name);
 }
 
+StoragePtr StorageMaterializedView::tryGetTargetTable() const
+{
+    return global_context.tryGetTable(target_database_name, target_table_name);
+}
+
+String StorageMaterializedView::getDataPath() const
+{
+    if (auto table = tryGetTargetTable())
+        return table->getDataPath();
+    return {};
+}
+
 bool StorageMaterializedView::checkTableCanBeDropped() const
 {
     /// Don't drop the target table if it was created manually via 'TO inner_table' statement
-    return has_inner_table ? getTargetTable()->checkTableCanBeDropped() : true;
+    if (!has_inner_table)
+        return true;
+
+    auto target_table = tryGetTargetTable();
+    if (!target_table)
+        return true;
+
+    return target_table->checkTableCanBeDropped();
 }
 
 

dbms/src/Storages/StorageMaterializedView.h

@@ -51,7 +51,7 @@ public:
         size_t max_block_size,
         unsigned num_streams) override;
 
-    String getDataPath() const override { return getTargetTable()->getDataPath(); }
+    String getDataPath() const override;
 
 private:
     String select_database_name;
@@ -65,6 +65,7 @@ private:
     bool has_inner_table = false;
 
     StoragePtr getTargetTable() const;
+    StoragePtr tryGetTargetTable() const;
 
     void checkStatementCanBeForwarded() const;
 

dbms/src/Storages/StorageMergeTree.cpp

@@ -141,8 +141,8 @@ void StorageMergeTree::truncate(const ASTPtr &)
         /// Asks to complete merges and does not allow them to start.
         /// This protects against "revival" of data for a removed partition after completion of merge.
         auto merge_blocker = merger.actions_blocker.cancel();
-        /// Waits for completion of merge and does not start new ones.
-        auto lock = lockForAlter(__PRETTY_FUNCTION__);
+
+        /// NOTE: It's assumed that this method is called under lockForAlter.
 
         auto parts_to_remove = data.getDataPartsVector();
         data.removePartsFromWorkingSet(parts_to_remove, true);

dbms/src/Storages/StorageSet.cpp

@@ -121,15 +121,16 @@ size_t StorageSet::getSize() const { return set->getTotalRowCount(); }
 
 void StorageSet::truncate(const ASTPtr &)
 {
+    Poco::File(path).remove(true);
+    Poco::File(path).createDirectories();
+    Poco::File(path + "tmp/").createDirectories();
+
     Block header = getSampleBlock();
     header = header.sortColumns();
 
     increment = 0;
     set = std::make_shared<Set>(SizeLimits());
     set->setHeader(header);
-
-    Poco::File(path).remove(true);
-    Poco::File(path).createDirectories();
 };
 
 

dbms/tests/queries/0_stateless/00623_replicated_truncate_table_zookeeper.sql

@@ -7,16 +7,22 @@ CREATE TABLE test.replicated_truncate2 (d Date, k UInt64, i32 Int32) ENGINE=Repl
 SELECT '======Before Truncate======';
 INSERT INTO test.replicated_truncate1 VALUES ('2015-01-01', 10, 42);
 
+SYSTEM SYNC REPLICA test.replicated_truncate2;
+
 SELECT * FROM test.replicated_truncate1 ORDER BY k;
 SELECT * FROM test.replicated_truncate2 ORDER BY k;
 
 SELECT '======After Truncate And Empty======';
 TRUNCATE TABLE test.replicated_truncate1;
+
 SELECT * FROM test.replicated_truncate1 ORDER BY k;
 SELECT * FROM test.replicated_truncate2 ORDER BY k;
 
 SELECT '======After Truncate And Insert Data======';
 INSERT INTO test.replicated_truncate1 VALUES ('2015-01-01', 10, 42);
+
+SYSTEM SYNC REPLICA test.replicated_truncate2;
+
 SELECT * FROM test.replicated_truncate1 ORDER BY k;
 SELECT * FROM test.replicated_truncate2 ORDER BY k;
 

dbms/tests/queries/0_stateless/00623_truncate_table.reference

@@ -6,7 +6,6 @@
 1
 2000-01-01	1
 2000-01-01	1
-1	hello
 ======After Truncate And Empty======
 0
 ======After Truncate And Insert Data======
@@ -17,4 +16,3 @@
 1
 2000-01-01	1
 2000-01-01	1
-1	hello

dbms/tests/queries/0_stateless/00623_truncate_table.sql

@@ -6,8 +6,6 @@ DROP TABLE IF EXISTS truncate_test.test_stripe_log;
 DROP TABLE IF EXISTS truncate_test.test_merge_tree;
 DROP TABLE IF EXISTS truncate_test.test_materialized_view;
 DROP TABLE IF EXISTS truncate_test.test_materialized_depend;
-DROP TABLE IF EXISTS truncate_test.test_distributed_depend;
-DROP TABLE IF EXISTS truncate_test.test_distributed;
 
 CREATE DATABASE truncate_test;
 CREATE TABLE truncate_test.test_set(id UInt64) ENGINE = Set;
@@ -18,8 +16,6 @@ CREATE TABLE truncate_test.test_stripe_log(id UInt64) ENGINE = StripeLog;
 CREATE TABLE truncate_test.test_merge_tree(p Date, k UInt64) ENGINE = MergeTree(p, k, 1);
 CREATE TABLE truncate_test.test_materialized_depend(p Date, k UInt64) ENGINE = Null;
 CREATE MATERIALIZED VIEW truncate_test.test_materialized_view ENGINE = MergeTree(p, k, 1) AS SELECT * FROM truncate_test.test_materialized_depend;
-CREATE TABLE truncate_test.test_distributed_depend (x UInt64, s String) ENGINE = MergeTree ORDER BY x;
-CREATE TABLE truncate_test.test_distributed AS truncate_test.test_distributed_depend ENGINE = Distributed(test_shard_localhost, truncate_test, test_distributed_depend);
 
 SELECT '======Before Truncate======';
 INSERT INTO truncate_test.test_set VALUES(0);
@@ -29,7 +25,6 @@ INSERT INTO truncate_test.test_tiny_log VALUES(1);
 INSERT INTO truncate_test.test_stripe_log VALUES(1);
 INSERT INTO truncate_test.test_merge_tree VALUES('2000-01-01', 1);
 INSERT INTO truncate_test.test_materialized_depend VALUES('2000-01-01', 1);
-INSERT INTO truncate_test.test_distributed_depend VALUES (1, 'hello');
 SELECT * FROM system.numbers WHERE number NOT IN truncate_test.test_set LIMIT 1;
 SELECT * FROM truncate_test.test_log;
 SELECT * FROM truncate_test.test_memory;
@@ -37,7 +32,6 @@ SELECT * FROM truncate_test.test_tiny_log;
 SELECT * FROM truncate_test.test_stripe_log;
 SELECT * FROM truncate_test.test_merge_tree;
 SELECT * FROM truncate_test.test_materialized_view;
-SELECT * FROM truncate_test.test_distributed;
 
 SELECT '======After Truncate And Empty======';
 TRUNCATE TABLE truncate_test.test_set;
@@ -47,7 +41,6 @@ TRUNCATE TABLE truncate_test.test_tiny_log;
 TRUNCATE TABLE truncate_test.test_stripe_log;
 TRUNCATE TABLE truncate_test.test_merge_tree;
 TRUNCATE TABLE truncate_test.test_materialized_view;
-TRUNCATE TABLE truncate_test.test_distributed;
 SELECT * FROM system.numbers WHERE number NOT IN truncate_test.test_set LIMIT 1;
 SELECT * FROM truncate_test.test_log;
 SELECT * FROM truncate_test.test_memory;
@@ -55,7 +48,6 @@ SELECT * FROM truncate_test.test_tiny_log;
 SELECT * FROM truncate_test.test_stripe_log;
 SELECT * FROM truncate_test.test_merge_tree;
 SELECT * FROM truncate_test.test_materialized_view;
-SELECT * FROM truncate_test.test_distributed;
 
 SELECT '======After Truncate And Insert Data======';
 INSERT INTO truncate_test.test_set VALUES(0);
@@ -65,7 +57,6 @@ INSERT INTO truncate_test.test_tiny_log VALUES(1);
 INSERT INTO truncate_test.test_stripe_log VALUES(1);
 INSERT INTO truncate_test.test_merge_tree VALUES('2000-01-01', 1);
 INSERT INTO truncate_test.test_materialized_depend VALUES('2000-01-01', 1);
-INSERT INTO truncate_test.test_distributed_depend VALUES (1, 'hello');
 SELECT * FROM system.numbers WHERE number NOT IN truncate_test.test_set LIMIT 1;
 SELECT * FROM truncate_test.test_log;
 SELECT * FROM truncate_test.test_memory;
@@ -73,7 +64,6 @@ SELECT * FROM truncate_test.test_tiny_log;
 SELECT * FROM truncate_test.test_stripe_log;
 SELECT * FROM truncate_test.test_merge_tree;
 SELECT * FROM truncate_test.test_materialized_view;
-SELECT * FROM truncate_test.test_distributed;
 
 DROP TABLE IF EXISTS truncate_test.test_set;
 DROP TABLE IF EXISTS truncate_test.test_log;
@@ -83,6 +73,4 @@ DROP TABLE IF EXISTS truncate_test.test_stripe_log;
 DROP TABLE IF EXISTS truncate_test.test_merge_tree;
 DROP TABLE IF EXISTS truncate_test.test_materialized_view;
 DROP TABLE IF EXISTS truncate_test.test_materialized_depend;
-DROP TABLE IF EXISTS truncate_test.test_distributed;
-DROP TABLE IF EXISTS truncate_test.test_distributed_depend;
 DROP DATABASE IF EXISTS truncate_test;