performance comparison

docker/test/performance-comparison/compare.sh

@@ -204,6 +204,11 @@ function run_tests
 
         grep ^query "$test_name-raw.tsv" | cut -f2- > "$test_name-queries.tsv"
         grep ^client-time "$test_name-raw.tsv" | cut -f2- > "$test_name-client-time.tsv"
+        skipped=$(grep ^skipped "$test_name-raw.tsv" | cut -f2-)
+        if [ "$skipped" != "" ]
+        then
+            printf "$test_name""\t""$skipped""\n" >> skipped-tests.tsv
+        fi
     done
 
     unset TIMEFORMAT
@@ -399,7 +404,7 @@ IFS=$'\n'
 for query in $(cut -d'	' -f1 stacks.rep | sort | uniq)
 do
     query_file=$(echo "$query" | cut -c-120 | sed 's/[/]/_/g')
-    grep -F "$query" stacks.rep \
+    grep -F "$query	" stacks.rep \
         | cut -d'	' -f 2- \
         | tee "$query_file.stacks.rep" \
         | ~/fg/flamegraph.pl > "$query_file.svg" &
@@ -427,16 +432,17 @@ case "$stage" in
     time restart
     ;&
 "run_tests")
-    # If the tests fail with OOM or something, still try to restart the servers
-    # to collect the logs.
+    # Ignore the errors to collect the log anyway
     time run_tests ||:
-    time restart
     ;&
 "get_profiles")
-    time get_profiles
+    # If the tests fail with OOM or something, still try to restart the servers
+    # to collect the logs. Prefer not to restart, because addresses might change
+    # and we won't be able to process trace_log data.
+    time get_profiles || restart || get_profiles
     ;&
 "analyze_queries")
-    analyze_queries
+    time analyze_queries
     ;&
 "report")
     time report

docker/test/performance-comparison/perf.py

@@ -46,18 +46,23 @@ connections = [clickhouse_driver.Client(**server) for server in servers]
 
 report_stage_end('connect')
 
-# Check tables that should exist
-tables = [e.text for e in root.findall('preconditions/table_exists')]
-for t in tables:
-    for c in connections:
-        res = c.execute("show create table {}".format(t))
-
 # Apply settings
 settings = root.findall('settings/*')
 for c in connections:
     for s in settings:
         c.execute("set {} = '{}'".format(s.tag, s.text))
 
+# Check tables that should exist. If they don't exist, just skip this test.
+tables = [e.text for e in root.findall('preconditions/table_exists')]
+for t in tables:
+    for c in connections:
+        try:
+            res = c.execute("show create table {}".format(t))
+        except:
+            print('skipped\t' + traceback.format_exception_only(*sys.exc_info()[:2])[-1])
+            traceback.print_exc()
+            sys.exit(0)
+
 report_stage_end('preconditions')
 
 # Process substitutions

docker/test/performance-comparison/report.py

@@ -192,6 +192,9 @@ run_error_rows = tsvRows('run-errors.tsv')
 error_tests += len(run_error_rows)
 printSimpleTable('Run errors', ['Test', 'Error'], run_error_rows)
 
+skipped_tests_rows = tsvRows('skipped-tests.tsv')
+printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows)
+
 printSimpleTable('Tests with most unstable queries',
     ['Test', 'Unstable', 'Changed perf', 'Total not OK'],
     tsvRows('bad-tests.tsv'))