diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile
index 76cadc3ce119b3fba7da9656519a0be53a3bcdf1..004bac02918a2c20fc1aba555d55d6cb2b286d3f 100644
--- a/docker/test/performance-comparison/Dockerfile
+++ b/docker/test/performance-comparison/Dockerfile
@@ -30,7 +30,7 @@ RUN apt-get update \
             tzdata \
             vim \
             wget \
-    && pip3 --no-cache-dir install clickhouse_driver scipy \
+    && pip3 --no-cache-dir install 'clickhouse-driver>=0.1.5' scipy \
     && apt-get purge --yes python3-dev g++ \
     && apt-get autoremove --yes \
     && apt-get clean \
diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py
index 337f13690b60039a654cd4fa4b50b41d057503d6..2568f7ac066daf64d09fbdd4d2edc9c699032c50 100755
--- a/docker/test/performance-comparison/perf.py
+++ b/docker/test/performance-comparison/perf.py
@@ -14,10 +14,12 @@ import string
 import sys
 import time
 import traceback
+import logging
 import xml.etree.ElementTree as et
 from threading import Thread
 from scipy import stats
 
+logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', level='WARNING')
 
 total_start_seconds = time.perf_counter()
 stage_start_seconds = total_start_seconds
@@ -171,12 +173,9 @@ reportStageEnd('drop-1')
 settings = root.findall('settings/*')
 for conn_index, c in enumerate(all_connections):
     for s in settings:
-        try:
-            q = f"set {s.tag} = '{s.text}'"
-            c.execute(q)
-            print(f'set\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
-        except:
-            print(traceback.format_exc(), file=sys.stderr)
+        # requires clickhouse-driver >= 1.1.5 to accept arbitrary new settings
+        # (https://github.com/mymarilyn/clickhouse-driver/pull/142)
+        c.settings[s.tag] = s.text
 
 reportStageEnd('settings')
 
diff --git a/tests/performance/joins_in_memory_pmj.xml b/tests/performance/joins_in_memory_pmj.xml
index 2b9596e15912f139153a0816576a42e05c84d679..1ca3fd49d5034ea16cca4ab609a127f14f58cb72 100644
--- a/tests/performance/joins_in_memory_pmj.xml
+++ b/tests/performance/joins_in_memory_pmj.xml
@@ -1,6 +1,9 @@
 <test max_ignored_relative_change="0.8">
     <create_query>CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory</create_query>
-    <create_query>SET join_algorithm = 'partial_merge'</create_query>
+
+    <settings>
+        <join_algorithm>partial_merge</join_algorithm>
+    </settings>
 
     <fill_query>INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000)</fill_query>
     <fill_query>INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000)</fill_query>
diff --git a/tests/performance/materialized_view_parallel_insert.xml b/tests/performance/materialized_view_parallel_insert.xml
index 4b71354dec3168d01a782defa93f12d34fd10566..24b8989443fb20c6289e1f63a0b75fd4944bf101 100644
--- a/tests/performance/materialized_view_parallel_insert.xml
+++ b/tests/performance/materialized_view_parallel_insert.xml
@@ -18,13 +18,13 @@
             -- do not select anything because we only need column types
             LIMIT 0
     </create_query>
-    <fill_query>SET max_insert_threads=8</fill_query>
     <fill_query>SYSTEM STOP MERGES</fill_query>
 
     <query>
         INSERT INTO hits_mv
         SELECT CounterID, EventDate, UserID, Title
         FROM hits_10m_single
+        SETTINGS max_insert_threads=8
     </query>
 
     <drop_query>SYSTEM START MERGES</drop_query>
diff --git a/tests/performance/parallel_insert.xml b/tests/performance/parallel_insert.xml
index 4050e771cb87e47d4a78166451cf4ca0c4e2bf64..08c1532e96358f5c55bae73ebf2f1ff0e9b8b29c 100644
--- a/tests/performance/parallel_insert.xml
+++ b/tests/performance/parallel_insert.xml
@@ -19,13 +19,13 @@
             -- do not select anything because we only need column types
             LIMIT 0
     </create_query>
-    <fill_query>SET max_insert_threads=8</fill_query>
     <fill_query>SYSTEM STOP MERGES</fill_query>
 
     <query>
         INSERT INTO hits2
         SELECT CounterID, EventDate, UserID, Title
         FROM hits_10m_single
+        SETTINGS max_insert_threads=8
     </query>
 
     <drop_query>SYSTEM START MERGES</drop_query>