Merge branch 'master' of github.com:yandex/ClickHouse into add-test-deadlock-rename-table

dbms/src/Interpreters/InterpreterKillQueryQuery.cpp

@@ -265,11 +265,6 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S
     if (where_expression)
         select_query += " WHERE " + queryToString(where_expression);
 
-    auto use_processors = context.getSettingsRef().experimental_use_processors;
-    context.getSettingsRef().experimental_use_processors = false;
-
-    SCOPE_EXIT(context.getSettingsRef().experimental_use_processors = use_processors);
-
     BlockIO block_io = executeQuery(select_query, context, true);
     Block res = block_io.in->read();
 

dbms/src/Storages/LiveView/StorageLiveView.cpp

@@ -593,7 +593,7 @@ void registerStorageLiveView(StorageFactory & factory)
 {
     factory.registerStorage("LiveView", [](const StorageFactory::Arguments & args)
     {
-        if (!args.local_context.getSettingsRef().allow_experimental_live_view)
+        if (!args.attach && !args.local_context.getSettingsRef().allow_experimental_live_view)
             throw Exception("Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')", ErrorCodes::SUPPORT_IS_DISABLED);
 
         return StorageLiveView::create(args.table_name, args.database_name, args.local_context, args.query, args.columns);

dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml

+<yandex>
+	<dictionary>
+		<name>cache_xypairs</name>
+		<source>
+            <clickhouse>
+                <host>localhost</host>
+                <port>9000</port>
+                <user>default</user>
+                <password></password>
+                <db>test</db>
+                <table>xypairs</table>
+            </clickhouse>
+		</source>
+		<lifetime>1</lifetime>
+		<layout>
+			<cache>
+				<size_in_cells>5</size_in_cells>
+			</cache>
+		</layout>
+		<structure>
+            <id>
+                <name>x</name>
+            </id>
+			<attribute>
+				<name>y</name>
+				<type>UInt64</type>
+                <null_value>0</null_value>
+			</attribute>
+		</structure>
+	</dictionary>
+</yandex>

dbms/tests/integration/test_dictionaries/test.py

@@ -17,6 +17,10 @@ def get_status(dictionary_name):
     return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n")
 
 
+def get_last_exception(dictionary_name):
+    return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'")
+
+
 def get_loading_start_time(dictionary_name):
     s = instance.query("SELECT loading_start_time FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n")
     if s == "0000-00-00 00:00:00":
@@ -350,3 +354,58 @@ def test_reload_after_fail_by_timer(started_cluster):
     time.sleep(6);
     query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n"
     assert get_status("no_file_2") == "LOADED"
+
+
+def test_reload_after_fail_in_cache_dictionary(started_cluster):
+    query = instance.query
+    query_and_get_error = instance.query_and_get_error
+
+    # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond.
+    expected_error = "Table test.xypairs doesn't exist"
+    assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))")
+    assert get_status("cache_xypairs") == "LOADED"
+    assert expected_error in get_last_exception("cache_xypairs")
+
+    # Create table `test.xypairs`.
+    query('''
+        drop table if exists test.xypairs;
+        create table test.xypairs (x UInt64, y UInt64) engine=Log;
+        insert into test.xypairs values (1, 56), (3, 78);
+        ''')
+
+    # Cache dictionary now works.
+    assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "56", ignore_error=True)
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0"
+    assert get_last_exception("cache_xypairs") == ""
+
+    # Drop table `test.xypairs`.
+    query('drop table if exists test.xypairs')
+
+    # Values are cached so we can get them.
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56"
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0"
+    assert get_last_exception("cache_xypairs") == ""
+
+    # But we can't get a value from the source table which isn't cached.
+    assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))")
+    assert expected_error in get_last_exception("cache_xypairs")
+
+    # Passed time should not spoil the cache.
+    time.sleep(5);
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56"
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0"
+    assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))")
+    assert expected_error in get_last_exception("cache_xypairs")
+
+    # Create table `test.xypairs` again with changed values.
+    query('''
+        drop table if exists test.xypairs;
+        create table test.xypairs (x UInt64, y UInt64) engine=Log;
+        insert into test.xypairs values (1, 57), (3, 79);
+        ''')
+
+    # The cache dictionary returns new values now.
+    assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57")
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0"
+    query("SELECT dictGet('cache_xypairs', 'y', toUInt64(3))") == "79"
+    assert get_last_exception("cache_xypairs") == ""

dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh

+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+set -e
+
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1";
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2";
+$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory";
+
+
+function thread1()
+{
+    while true; do 
+        seq 1 1000 | sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' | $CLICKHOUSE_CLIENT -n
+    done
+}
+
+function thread2()
+{
+    while true; do
+        $CLICKHOUSE_CLIENT --query "SELECT * FROM merge(currentDatabase(), '^test[12]$')"
+    done
+}
+
+# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout
+export -f thread1;
+export -f thread2;
+
+TIMEOUT=10
+
+timeout $TIMEOUT bash -c thread1 2> /dev/null &
+timeout $TIMEOUT bash -c thread2 2> /dev/null &
+
+wait
+
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1";
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2";

dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh

+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+set -e
+
+function thread1()
+{
+    while true; do 
+        $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM numbers(1000000000)" --format Null;
+    done
+}
+
+function thread2()
+{
+    while true; do
+        $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello'" --format Null;
+        sleep 0.$RANDOM
+    done
+}
+
+function thread3()
+{
+    while true; do
+        $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null;
+        $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null;
+    done
+}
+
+# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout
+export -f thread1;
+export -f thread2;
+export -f thread3;
+
+TIMEOUT=10
+
+timeout $TIMEOUT bash -c thread1 2> /dev/null &
+timeout $TIMEOUT bash -c thread1 2> /dev/null &
+timeout $TIMEOUT bash -c thread1 2> /dev/null &
+timeout $TIMEOUT bash -c thread1 2> /dev/null &
+
+timeout $TIMEOUT bash -c thread2 2> /dev/null &
+
+timeout $TIMEOUT bash -c thread3 2> /dev/null &
+timeout $TIMEOUT bash -c thread3 2> /dev/null &
+timeout $TIMEOUT bash -c thread3 2> /dev/null &
+timeout $TIMEOUT bash -c thread3 2> /dev/null &
+
+wait

dbms/tests/stress

+#!/usr/bin/env bash
+
+# https://stackoverflow.com/questions/360201/how-do-i-kill-background-processes-jobs-when-my-shell-script-exits
+trap 'kill -9 $(jobs -p)' EXIT
+
+function thread()
+{
+    while true; do
+        ./clickhouse-test --order random 2>&1 | awk '{ printf "'$1' " }'
+    done
+}
+
+# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout
+export -f thread;
+
+NUM_THREADS=${1:-"16"}
+TIMEOUT=${2:-"300"}
+
+for i in $(seq 1 $NUM_THREADS); do
+    timeout $TIMEOUT bash -c "thread $i" 2> /dev/null &
+done
+
+wait