performance comparison

docker/test/performance-comparison/compare.sh

@@ -93,53 +93,52 @@ function run_tests
     # Just check that the script runs at all
     "$script_dir/perf.py" --help > /dev/null
 
-    # When testing commits from master, use the older test files. This allows the
-    # tests to pass even when we add new functions and tests for them, that are
-    # not supported in the old revision.
-    # When testing a PR, use the test files from the PR so that we can test their
-    # changes.
-    test_prefix=$([ "$PR_TO_TEST" == "0" ] && echo left || echo right)/performance
-
-    for x in {test-times,skipped-tests,wall-clock-times,report-thresholds,client-times}.tsv
-    do
-        rm -v "$x" ||:
-        touch "$x"
-    done
-
-
-    # FIXME a quick crutch to bring the run time down for the unstable tests --
-    # if some performance tests xmls were changed in a PR, run only these ones.
-    if [ "$PR_TO_TEST" != "0" ]
+    # Find the directory with test files.
+    if [ -v CHPC_TEST_PATH ]
     then
-        # changed-test.txt prepared in entrypoint.sh from git diffs, because it
-        # has the cloned repo. Used to use rsync for that but it was really ugly
-        # and not always correct (e.g. when the reference SHA is really old and
-        # has some other differences to the tested SHA, besides the one introduced
-        # by the PR).
-        test_files_override=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-tests.txt)
-        if [ "$test_files_override" != "" ]
-        then
-            test_files=$test_files_override
-        fi
+        # Use the explicitly set path to directory with test files.
+        test_prefix="$CHPC_TEST_PATH"
+    elif [ "$PR_TO_TEST" = "0" ]
+    then
+        # When testing commits from master, use the older test files. This
+        # allows the tests to pass even when we add new functions and tests for
+        # them, that are not supported in the old revision.
+        test_prefix=left/performance
+    elif [ "$PR_TO_TEST" != "" ] && [ "$PR_TO_TEST" != "0" ]
+    then
+        # For PRs, use newer test files so we can test these changes.
+        test_prefix=right/performance
+
+        # If some tests were changed in the PR, we may want to run only these
+        # ones. The list of changed tests in changed-test.txt is prepared in
+        # entrypoint.sh from git diffs, because it has the cloned repo.  Used
+        # to use rsync for that but it was really ugly and not always correct
+        # (e.g. when the reference SHA is really old and has some other
+        # differences to the tested SHA, besides the one introduced by the PR).
+        changed_test_files=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-tests.txt)
     fi
 
-    # Run only explicitly specified tests, if any
+    # Determine which tests to run.
     if [ -v CHPC_TEST_GREP ]
     then
+        # Run only explicitly specified tests, if any.
         test_files=$(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}")
-    fi
-
-    if [ "$test_files" == "" ]
+    elif [ "$changed_test_files" != "" ]
     then
-        # FIXME remove some broken long tests
-        for test_name in {IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format,arithmetic,cryptographic_hashes,logical_functions_{medium,small}}
-        do
-            printf "%s\tMarked as broken (see compare.sh)\n" "$test_name">> skipped-tests.tsv
-            rm "$test_prefix/$test_name.xml" ||:
-        done
+        # Use test files that changed in the PR.
+        test_files="$changed_test_files"
+    else
+        # The default -- run all tests found in the test dir.
         test_files=$(ls "$test_prefix"/*.xml)
     fi
 
+    # Delete old report files.
+    for x in {test-times,skipped-tests,wall-clock-times,report-thresholds,client-times}.tsv
+    do
+        rm -v "$x" ||:
+        touch "$x"
+    done
+
     # Run the tests.
     test_name="<none>"
     for test in $test_files
@@ -554,11 +553,10 @@ case "$stage" in
     echo Servers stopped.
     ;&
 "analyze_queries")
-    # FIXME grep for set_index fails -- argument list too long.
     time analyze_queries ||:
     ;&
 "report")
-    time report
+    time report ||:
 
     time "$script_dir/report.py" --report=all-queries > all-queries.html 2> >(tee -a report/errors.log 1>&2) ||:
     time "$script_dir/report.py" > report.html

docker/test/performance-comparison/performance_comparison.md

@@ -54,7 +54,27 @@ docker/test/performance-comparison/perf.py --host=localhost --port=9000 --runs=1
 ```
 
 #### Run all tests on some custom configuration
-Technically possible, but inconvenient -- requires some scripting and setting up the directory structure. See `manual-run.sh` for inspiration. `compare.sh` has some stages which you can skip, specified by the `stage` environment variable.
+Start two servers manually on ports `9001` (old) and `9002` (new). Change to a
+new directory to be used as workspace for tests, and try something like this:
+```
+$ PATH=$PATH:~/ch4/build-gcc9-rel/programs \
+    CHPC_TEST_PATH=~/ch3/ch/tests/performance \
+    CHPC_TEST_GREP=visit_param \
+    stage=run_tests \
+    ~/ch3/ch/docker/test/performance-comparison/compare.sh
+```
+* `PATH` must contain `clickhouse-local` and `clickhouse-client`.
+* `CHPC_TEST_PATH` -- path to performance test cases, e.g. `tests/performance`.
+* `CHPC_TEST_GREP` -- a filter for which tests to run, as a grep pattern.
+* `stage` -- from which execution stage to start. To run the tests, use
+  `run_tests` stage.
+
+The tests will run, and the `report.html` will be generated in the current
+directory.
+
+More complex setup is possible, but inconvenient and requires some scripting.
+See `manual-run.sh` for inspiration.
+
 
 #### Statistical considerations
 Generating randomization distribution for medians is tricky. Suppose we have N

docker/test/performance-comparison/report.py

@@ -143,13 +143,42 @@ def printSimpleTable(caption, columns, rows):
         print(tableRow(row))
     print(tableEnd())
 
+def print_tested_commits():
+    global report_errors
+    try:
+        printSimpleTable('Tested commits', ['Old', 'New'],
+            [['<pre>{}</pre>'.format(x) for x in
+                [open('left-commit.txt').read(),
+                 open('right-commit.txt').read()]]])
+    except:
+        # Don't fail if no commit info -- maybe it's a manual run.
+        report_errors.append(
+            traceback.format_exception_only(
+                *sys.exc_info()[:2])[-1])
+        pass
+
+def print_report_errors():
+    global report_errors
+    # Add the errors reported by various steps of comparison script
+    try:
+        report_errors += [l.strip() for l in open('report/errors.log')]
+    except:
+        report_errors.append(
+            traceback.format_exception_only(
+                *sys.exc_info()[:2])[-1])
+        pass
+
+    if len(report_errors):
+        print(tableStart('Errors while building the report'))
+        print(tableHeader(['Error']))
+        for x in report_errors:
+            print(tableRow([x]))
+        print(tableEnd())
+
 if args.report == 'main':
     print(header_template.format())
 
-    printSimpleTable('Tested commits', ['Old', 'New'],
-        [['<pre>{}</pre>'.format(x) for x in
-            [open('left-commit.txt').read(),
-             open('right-commit.txt').read()]]])
+    print_tested_commits()
 
     def print_changes():
         rows = tsvRows('report/changed-perf.tsv')
@@ -288,15 +317,7 @@ if args.report == 'main':
 
     print_test_times()
 
-    # Add the errors reported by various steps of comparison script
-    report_errors += [l.strip() for l in open('report/errors.log')]
-    if len(report_errors):
-        print(tableStart('Errors while building the report'))
-        print(tableHeader(['Error']))
-        for x in report_errors:
-            print(tableRow([x]))
-        print(tableEnd())
-
+    print_report_errors()
 
     print("""
     <p class="links">
@@ -350,10 +371,7 @@ elif args.report == 'all-queries':
 
     print(header_template.format())
 
-    printSimpleTable('Tested commits', ['Old', 'New'],
-        [['<pre>{}</pre>'.format(x) for x in
-            [open('left-commit.txt').read(),
-             open('right-commit.txt').read()]]])
+    print_tested_commits()
 
     def print_all_queries():
         rows = tsvRows('report/all-queries.tsv')
@@ -405,6 +423,8 @@ elif args.report == 'all-queries':
 
     print_all_queries()
 
+    print_report_errors()
+
     print("""
     <p class="links">
     <a href="output.7z">Test output</a>