Merge branch 'master' into initial-explain

docker/images.json

@@ -79,5 +79,9 @@
     "docker/test/integration/runner": {
         "name": "yandex/clickhouse-integration-tests-runner",
         "dependent": []
+    },
+    "docker/test/testflows/runner": {
+        "name": "yandex/clickhouse-testflows-runner",
+        "dependent": []
     }
 }

docker/test/testflows/runner/Dockerfile

+# docker build -t yandex/clickhouse-testflows-runner .
+FROM ubuntu:20.04
+
+RUN apt-get update \
+    && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+    ca-certificates \
+    bash \
+    btrfs-progs \
+    e2fsprogs \
+    iptables \
+    xfsprogs \
+    tar \
+    pigz \
+    wget \
+    git \
+    iproute2 \
+    cgroupfs-mount \
+    python3-pip \
+    tzdata \
+    libreadline-dev \
+    libicu-dev \
+    bsdutils \
+    curl \
+    liblua5.1-dev \
+    luajit \
+    libssl-dev \
+    libcurl4-openssl-dev \
+    gdb \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /var/cache/debconf \
+        /tmp/* \
+    && apt-get clean
+
+ENV TZ=Europe/Moscow
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN pip3 install urllib3 testflows==1.6.24 docker-compose docker dicttoxml kazoo tzlocal
+
+ENV DOCKER_CHANNEL stable
+ENV DOCKER_VERSION 17.09.1-ce
+
+RUN set -eux; \
+  \
+# this "case" statement is generated via "update.sh"
+  \
+  if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \
+    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \
+    exit 1; \
+  fi; \
+  \
+  tar --extract \
+    --file docker.tgz \
+    --strip-components 1 \
+    --directory /usr/local/bin/ \
+  ; \
+  rm docker.tgz; \
+  \
+  dockerd --version; \
+  docker --version
+
+COPY modprobe.sh /usr/local/bin/modprobe
+COPY dockerd-entrypoint.sh /usr/local/bin/
+
+RUN set -x \
+  && addgroup --system dockremap \
+    && adduser --system dockremap \
+  && adduser dockremap dockremap \
+  && echo 'dockremap:165536:65536' >> /etc/subuid \
+    && echo 'dockremap:165536:65536' >> /etc/subgid
+
+VOLUME /var/lib/docker
+EXPOSE 2375
+ENTRYPOINT ["dockerd-entrypoint.sh"]
+CMD ["sh", "-c", "python3 regression.py --no-color --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS} && cat test.log | tfs report results --format json > results.json"]
+

docker/test/testflows/runner/dockerd-entrypoint.sh

+#!/bin/bash
+set -e
+
+dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &>/var/log/somefile &
+
+set +e
+reties=0
+while true; do
+    docker info &>/dev/null && break
+    reties=$[$reties+1]
+    if [[ $reties -ge 100 ]]; then # 10 sec max
+        echo "Can't start docker daemon, timeout exceeded." >&2
+        exit 1;
+    fi
+    sleep 0.1
+done
+set -e
+
+echo "Start tests"
+export CLICKHOUSE_TESTS_SERVER_BIN_PATH=/clickhouse
+export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse
+export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
+export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
+
+cd /ClickHouse/tests/testflows
+exec "$@"

docker/test/testflows/runner/modprobe.sh

+#!/bin/sh
+set -eu
+
+# "modprobe" without modprobe
+# https://twitter.com/lucabruno/status/902934379835662336
+
+# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
+
+# Docker often uses "modprobe -va foo bar baz"
+# so we ignore modules that start with "-"
+for module; do
+    if [ "${module#-}" = "$module" ]; then
+        ip link show "$module" || true
+        lsmod | grep "$module" || true
+    fi
+done
+
+# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
+export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
+exec modprobe "$@"

src/Functions/FunctionsLogical.cpp

@@ -42,7 +42,7 @@ using UInt8Container = ColumnUInt8::Container;
 using UInt8ColumnPtrs = std::vector<const ColumnUInt8 *>;
 
 
-MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable)
+MutableColumnPtr buildColumnFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable)
 {
     const size_t rows_count = ternary_data.size();
 
@@ -63,7 +63,7 @@ MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, con
 }
 
 template <typename T>
-bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res)
+bool tryConvertColumnToBool(const IColumn * column, UInt8Container & res)
 {
     const auto col = checkAndGetColumn<ColumnVector<T>>(column);
     if (!col)
@@ -71,22 +71,22 @@ bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res)
 
     std::transform(
             col->getData().cbegin(), col->getData().cend(), res.begin(),
-            [](const auto x) { return x != 0; });
+            [](const auto x) { return !!x; });
 
     return true;
 }
 
-void convertColumnToUInt8(const IColumn * column, UInt8Container & res)
+void convertAnyColumnToBool(const IColumn * column, UInt8Container & res)
 {
-    if (!tryConvertColumnToUInt8<Int8>(column, res) &&
-        !tryConvertColumnToUInt8<Int16>(column, res) &&
-        !tryConvertColumnToUInt8<Int32>(column, res) &&
-        !tryConvertColumnToUInt8<Int64>(column, res) &&
-        !tryConvertColumnToUInt8<UInt16>(column, res) &&
-        !tryConvertColumnToUInt8<UInt32>(column, res) &&
-        !tryConvertColumnToUInt8<UInt64>(column, res) &&
-        !tryConvertColumnToUInt8<Float32>(column, res) &&
-        !tryConvertColumnToUInt8<Float64>(column, res))
+    if (!tryConvertColumnToBool<Int8>(column, res) &&
+        !tryConvertColumnToBool<Int16>(column, res) &&
+        !tryConvertColumnToBool<Int32>(column, res) &&
+        !tryConvertColumnToBool<Int64>(column, res) &&
+        !tryConvertColumnToBool<UInt16>(column, res) &&
+        !tryConvertColumnToBool<UInt32>(column, res) &&
+        !tryConvertColumnToBool<UInt64>(column, res) &&
+        !tryConvertColumnToBool<Float32>(column, res) &&
+        !tryConvertColumnToBool<Float64>(column, res))
         throw Exception("Unexpected type of column: " + column->getName(), ErrorCodes::ILLEGAL_COLUMN);
 }
 
@@ -119,7 +119,7 @@ static bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res, Func && func)
 }
 
 template <class Op>
-inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res)
+inline bool extractConstColumnsAsBool(ColumnRawPtrs & in, UInt8 & res)
 {
     return extractConstColumns<Op>(
         in, res,
@@ -131,7 +131,7 @@ inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res)
 }
 
 template <class Op>
-inline bool extractConstColumnsTernary(ColumnRawPtrs & in, UInt8 & res_3v)
+inline bool extractConstColumnsAsTernary(ColumnRawPtrs & in, UInt8 & res_3v)
 {
     return extractConstColumns<Op>(
         in, res_3v,
@@ -145,6 +145,7 @@ inline bool extractConstColumnsTernary(ColumnRawPtrs & in, UInt8 & res_3v)
 }
 
 
+/// N.B. This class calculates result only for non-nullable types
 template <typename Op, size_t N>
 class AssociativeApplierImpl
 {
@@ -158,7 +159,7 @@ public:
     /// Returns a combination of values in the i-th row of all columns stored in the constructor.
     inline ResultValueType apply(const size_t i) const
     {
-        const auto & a = vec[i];
+        const auto a = !!vec[i];
         if constexpr (Op::isSaturable())
             return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i));
         else
@@ -179,7 +180,7 @@ public:
     explicit AssociativeApplierImpl(const UInt8ColumnPtrs & in)
         : vec(in[in.size() - 1]->getData()) {}
 
-    inline ResultValueType apply(const size_t i) const { return vec[i]; }
+    inline ResultValueType apply(const size_t i) const { return !!vec[i]; }
 
 private:
     const UInt8Container & vec;
@@ -188,7 +189,7 @@ private:
 
 /// A helper class used by AssociativeGenericApplierImpl
 /// Allows for on-the-fly conversion of any data type into intermediate ternary representation
-using ValueGetter = std::function<Ternary::ResultType (size_t)>;
+using TernaryValueGetter = std::function<Ternary::ResultType (size_t)>;
 
 template <typename ... Types>
 struct ValueGetterBuilderImpl;
@@ -196,7 +197,7 @@ struct ValueGetterBuilderImpl;
 template <typename Type, typename ...Types>
 struct ValueGetterBuilderImpl<Type, Types...>
 {
-    static ValueGetter build(const IColumn * x)
+    static TernaryValueGetter build(const IColumn * x)
     {
         if (const auto * nullable_column = typeid_cast<const ColumnNullable *>(x))
         {
@@ -218,7 +219,7 @@ struct ValueGetterBuilderImpl<Type, Types...>
 template <>
 struct ValueGetterBuilderImpl<>
 {
-    static ValueGetter build(const IColumn * x)
+    static TernaryValueGetter build(const IColumn * x)
     {
         throw Exception(
                 std::string("Unknown numeric column of type: ") + demangle(typeid(x).name()),
@@ -247,13 +248,13 @@ public:
     {
         const auto a = val_getter(i);
         if constexpr (Op::isSaturable())
-            return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i));
+            return Op::isSaturatedValueTernary(a) ? a : Op::apply(a, next.apply(i));
         else
             return Op::apply(a, next.apply(i));
     }
 
 private:
-    const ValueGetter val_getter;
+    const TernaryValueGetter val_getter;
     const AssociativeGenericApplierImpl<Op, N - 1> next;
 };
 
@@ -271,7 +272,7 @@ public:
     inline ResultValueType apply(const size_t i) const { return val_getter(i); }
 
 private:
-    const ValueGetter val_getter;
+    const TernaryValueGetter val_getter;
 };
 
 
@@ -332,13 +333,13 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn
 {
     /// Combine all constant columns into a single constant value.
     UInt8 const_3v_value = 0;
-    const bool has_consts = extractConstColumnsTernary<Op>(arguments, const_3v_value);
+    const bool has_consts = extractConstColumnsAsTernary<Op>(arguments, const_3v_value);
 
     /// If the constant value uniquely determines the result, return it.
     if (has_consts && (arguments.empty() || Op::isSaturatedValue(const_3v_value)))
     {
         result_info.column = ColumnConst::create(
-            convertFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()),
+            buildColumnFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()),
             input_rows_count
         );
         return;
@@ -349,7 +350,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn
 
     OperationApplier<Op, AssociativeGenericApplierImpl>::apply(arguments, result_column->getData(), has_consts);
 
-    result_info.column = convertFromTernaryData(result_column->getData(), result_info.type->isNullable());
+    result_info.column = buildColumnFromTernaryData(result_column->getData(), result_info.type->isNullable());
 }
 
 
@@ -402,12 +403,13 @@ struct TypedExecutorInvoker<Op>
 };
 
 
+/// Types of all of the arguments are guaranteed to be non-nullable here
 template <class Op>
 static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & result_info, size_t input_rows_count)
 {
     /// Combine all constant columns into a single constant value.
     UInt8 const_val = 0;
-    bool has_consts = extractConstColumns<Op>(arguments, const_val);
+    bool has_consts = extractConstColumnsAsBool<Op>(arguments, const_val);
 
     /// If the constant value uniquely determines the result, return it.
     if (has_consts && (arguments.empty() || Op::apply(const_val, 0) == Op::apply(const_val, 1)))
@@ -447,7 +449,7 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re
         else
         {
             auto converted_column = ColumnUInt8::create(input_rows_count);
-            convertColumnToUInt8(column, converted_column->getData());
+            convertAnyColumnToBool(column, converted_column->getData());
             uint8_args.push_back(converted_column.get());
             converted_columns_holder.emplace_back(std::move(converted_column));
         }

src/Functions/FunctionsLogical.h

@@ -61,7 +61,8 @@ struct AndImpl
     using ResultType = UInt8;
 
     static inline constexpr bool isSaturable() { return true; }
-    static inline constexpr bool isSaturatedValue(UInt8 a) { return a == Ternary::False; }
+    static inline constexpr bool isSaturatedValue(bool a) { return !a; }
+    static inline constexpr bool isSaturatedValueTernary(UInt8 a) { return a == Ternary::False; }
     static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a & b; }
     static inline constexpr bool specialImplementationForNulls() { return true; }
 };
@@ -71,7 +72,8 @@ struct OrImpl
     using ResultType = UInt8;
 
     static inline constexpr bool isSaturable() { return true; }
-    static inline constexpr bool isSaturatedValue(UInt8 a) { return a == Ternary::True; }
+    static inline constexpr bool isSaturatedValue(bool a) { return a; }
+    static inline constexpr bool isSaturatedValueTernary(UInt8 a) { return a == Ternary::True; }
     static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a | b; }
     static inline constexpr bool specialImplementationForNulls() { return true; }
 };
@@ -82,7 +84,8 @@ struct XorImpl
 
     static inline constexpr bool isSaturable() { return false; }
     static inline constexpr bool isSaturatedValue(bool) { return false; }
-    static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return !!a != !!b; }
+    static inline constexpr bool isSaturatedValueTernary(UInt8) { return false; }
+    static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a != b; }
     static inline constexpr bool specialImplementationForNulls() { return false; }
 
 #if USE_EMBEDDED_COMPILER

src/Functions/array/arrayIntersect.cpp

@@ -251,7 +251,6 @@ FunctionArrayIntersect::CastArgumentsResult FunctionArrayIntersect::castColumns(
             }
             else
             {
-
                 if (!arg.type->equals(*return_type) && !arg.type->equals(*nullable_return_type))
                 {
                     /// If result has array type Array(T) still cast Array(Nullable(U)) to Array(Nullable(T))

tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference

+1	1	1	1	1	1
+0	0	0	0	0
+0	0	1	0	1
+0	2	0	0	1
+0	2	1	0	1
+4	0	0	0	1
+4	0	1	0	1
+4	2	0	0	1
+4	2	1	1	1

tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql

+
+-- Test that UInt8 type is processed correctly as bool
+
+SELECT
+    1 AND 2,
+    2 AND 4,
+    1 AND 2 AND 4,
+    1 OR 2,
+    2 OR 4,
+    1 OR 2 OR 4
+;
+
+SELECT 
+    toUInt8(bitAnd(number, 4)) AS a,
+    toUInt8(bitAnd(number, 2)) AS b,
+    toUInt8(bitAnd(number, 1)) AS c,
+    a AND b AND c AS AND,
+    a OR b OR c AS OR
+FROM numbers(8)
+;

tests/testflows/README.md

+## ClickHouse Tests in [TestFlows]
+
+This directory contains integration tests written using [TestFlows] 
+that involves several ClickHouse instances, custom configs, ZooKeeper, etc.
+
+## Supported environment
+
+* [Ubuntu] 18.04
+* [Python 3] >= 3.6
+
+## Prerequisites
+
+* [Docker] [install](https://docs.docker.com/compose/install/)
+* [Docker Compose] [install](https://docs.docker.com/engine/install/)
+* [TestFlows] [install](https://testflows.com/handbook/#Installation) 
+
+## Running tests locally
+
+You can run tests locally by passing `--local` and `--clickhouse-binary-path` to the `regression.py`.
+
+* `--local` specifies that regression will be run locally
+* `--clickhouse-binary-path` specifies the path to the ClickHouse binary that will be used during the regression run
+
+> Note: you can pass `-h` or `--help` argument to the `regression.py` to see a help message.
+>
+> ```bash
+> python3 regression.py -h
+> ```
+
+> Note: make sure that the ClickHouse binary has correct permissions. 
+> If you are using `/usr/bin/clickhouse` its owner and group is set to `root:root` by default 
+> and it needs to be changed to `clickhouse:clickhouse`. You can change the owner and the group 
+> using the following command.
+> 
+> ```bash
+> sudo chown clickhouse:clickhouse /usr/bin/clickhouse
+> ```
+
+Using the default ClickHouse installation and its server binary at `/usr/bin/clickhouse`, you can run 
+regressions locally using the following command.
+
+```bash
+python3 regression.py --local --clickhouse-binary-path "/usr/bin/clickhouse"
+```
+
+## Output Verbosity
+
+You can control verbosity of the output by specifying the output format with `-o` or `--output` option.
+See `--help` for more details.
+
+[Python 3]: https://www.python.org/
+[Ubuntu]: https://ubuntu.com/ 
+[TestFlows]: https://testflows.com
+[Docker]: https://www.docker.com/
+[Docker Compose]: https://docs.docker.com/compose/

tests/testflows/example/configs/clickhouse/common.xml

+<yandex>
+    <timezone>Europe/Moscow</timezone>
+    <listen_host replace="replace">0.0.0.0</listen_host>
+    <path>/var/lib/clickhouse/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+</yandex>

tests/testflows/example/configs/clickhouse/config.d/logs.xml

+<yandex>
+    <shutdown_wait_unfinished>3</shutdown_wait_unfinished>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-server/log.log</log>
+        <errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
+        <size>1000M</size>
+        <count>10</count>
+        <stderr>/var/log/clickhouse-server/stderr.log</stderr>
+        <stdout>/var/log/clickhouse-server/stdout.log</stdout>
+    </logger>
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+        <flush_interval_milliseconds>500</flush_interval_milliseconds>
+    </part_log>
+</yandex>

tests/testflows/example/configs/clickhouse/config.d/ports.xml

+<?xml version="1.0"?>
+<yandex>
+    <https_port>8443</https_port>
+    <tcp_port_secure>9440</tcp_port_secure>
+</yandex>
\ No newline at end of file

tests/testflows/example/configs/clickhouse/config.d/remote.xml

+<?xml version="1.0"?>
+<yandex>
+    <remote_servers>
+        <replicated_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </replicated_cluster>
+        <!--
+        <replicated_cluster_readonly>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                    <user>readonly</user>
+                </replica>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9000</port>
+                    <user>readonly</user>
+                </replica>
+                <replica>
+                    <host>clickhouse3</host>
+                    <port>9000</port>
+                    <user>readonly</user>
+                </replica>
+            </shard>
+        </replicated_cluster_readonly>
+        -->
+        <replicated_cluster_secure>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+                <replica>
+                    <host>clickhouse3</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </replicated_cluster_secure>
+        <sharded_cluster>
+            <shard>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </sharded_cluster>
+        <sharded_cluster_secure>
+            <shard>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse3</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </sharded_cluster_secure>
+    </remote_servers>
+</yandex>

tests/testflows/example/configs/clickhouse/config.d/ssl.xml

+<yandex>
+    <openSSL>
+        <server>
+            <certificateFile>/etc/clickhouse-server/ssl/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/ssl/server.key</privateKeyFile>
+            <verificationMode>none</verificationMode>
+            <cacheSessions>true</cacheSessions>
+        </server>
+        <client>
+            <cacheSessions>true</cacheSessions>
+            <verificationMode>none</verificationMode>
+            <invalidCertificateHandler>
+                <name>AcceptCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</yandex>

tests/testflows/example/configs/clickhouse/config.d/storage.xml

+<yandex>
+
+<storage_configuration>
+    <disks>
+        <default>
+            <keep_free_space_bytes>1024</keep_free_space_bytes>
+        </default>
+    </disks>
+    <policies>
+        <default>
+            <volumes>
+                <default>
+                    <disk>default</disk>
+                </default>
+            </volumes>
+        </default>
+    </policies>
+</storage_configuration>
+
+</yandex>

tests/testflows/example/configs/clickhouse/config.d/zookeeper.xml

+<?xml version="1.0"?>
+<yandex>
+    <zookeeper>
+	    <node index="1">
+		    <host>zookeeper</host>
+		    <port>2181</port>
+	    </node>
+        <session_timeout_ms>15000</session_timeout_ms>
+    </zookeeper>
+</yandex>

tests/testflows/example/configs/clickhouse/config.xml

+<?xml version="1.0"?>
+<!--
+  NOTE: User and query level settings are set up in "users.xml" file.
+-->
+<yandex>
+    <logger>
+        <!-- Possible levels: https://github.com/pocoproject/poco/blob/develop/Foundation/include/Poco/Logger.h#L105 -->
+        <level>trace</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>10</count>
+        <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
+    </logger>
+    <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+
+    <!-- For HTTPS and SSL over native protocol. -->
+    <!--
+    <https_port>8443</https_port>
+    <tcp_port_secure>9440</tcp_port_secure>
+    -->
+
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <openSSL>
+        <server> <!-- Used for https server AND secure tcp port -->
+            <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
+            <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+            <!-- openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 -->
+            <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+        </server>
+
+        <client> <!-- Used for connecting to https dictionary source -->
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <!-- Use for self-signed: <verificationMode>none</verificationMode> -->
+            <invalidCertificateHandler>
+                <!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+
+    <!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -->
+    <!--
+    <http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response>
+    -->
+
+    <!-- Port for communication between replicas. Used for data exchange. -->
+    <interserver_http_port>9009</interserver_http_port>
+
+    <!-- Hostname that is used by other replicas to request this server.
+         If not specified, than it is determined analoguous to 'hostname -f' command.
+         This setting could be used to switch replication to another network interface.
+      -->
+    <!--
+    <interserver_http_host>example.yandex.ru</interserver_http_host>
+    -->
+
+    <!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -->
+    <!-- <listen_host>::</listen_host> -->
+    <!-- Same for hosts with disabled ipv6: -->
+    <!-- <listen_host>0.0.0.0</listen_host> -->
+
+    <!-- Default values - try listen localhost on ipv4 and ipv6: -->
+    <!--
+    <listen_host>::1</listen_host>
+    <listen_host>127.0.0.1</listen_host>
+    -->
+    <!-- Don't exit if ipv6 or ipv4 unavailable, but listen_host with this protocol specified -->
+    <!-- <listen_try>0</listen_try> -->
+
+    <!-- Allow listen on same address:port -->
+    <!-- <listen_reuse_port>0</listen_reuse_port> -->
+
+    <!-- <listen_backlog>64</listen_backlog> -->
+
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+
+    <!-- Maximum number of concurrent queries. -->
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+    <!-- Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve
+         correct maximum value. -->
+    <!-- <max_open_files>262144</max_open_files> -->
+
+    <!-- Size of cache of uncompressed blocks of data, used in tables of MergeTree family.
+         In bytes. Cache is single for server. Memory is allocated only on demand.
+         Cache is used when 'use_uncompressed_cache' user setting turned on (off by default).
+         Uncompressed cache is advantageous only for very short queries and in rare cases.
+      -->
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+
+    <!-- Approximate size of mark cache, used in tables of MergeTree family.
+         In bytes. Cache is single for server. Memory is allocated only on demand.
+         You should not lower this value.
+      -->
+    <mark_cache_size>5368709120</mark_cache_size>
+
+
+    <!-- Path to data directory, with trailing slash. -->
+    <path>/var/lib/clickhouse/</path>
+
+    <!-- Path to temporary data for processing hard queries. -->
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+
+    <!-- Directory with user provided files that are accessible by 'file' table function. -->
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+
+    <!-- Path to folder where users and roles created by SQL commands are stored. -->
+    <access_control_path>/var/lib/clickhouse/access/</access_control_path>
+
+    <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
+    <users_config>users.xml</users_config>
+
+    <!-- Default profile of settings. -->
+    <default_profile>default</default_profile>
+
+    <!-- System profile of settings. This settings are used by internal processes (Buffer storage, Distibuted DDL worker and so on). -->
+    <!-- <system_profile>default</system_profile> -->
+
+    <!-- Default database. -->
+    <default_database>default</default_database>
+
+    <!-- Server time zone could be set here.
+
+         Time zone is used when converting between String and DateTime types,
+          when printing DateTime in text formats and parsing DateTime from text,
+          it is used in date and time related functions, if specific time zone was not passed as an argument.
+
+         Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan.
+         If not specified, system time zone at server startup is used.
+
+         Please note, that server could display time zone alias instead of specified name.
+         Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC.
+    -->
+    <!-- <timezone>Europe/Moscow</timezone> -->
+
+    <!-- You can specify umask here (see "man umask"). Server will apply it on startup.
+         Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read).
+    -->
+    <!-- <umask>022</umask> -->
+
+    <!-- Perform mlockall after startup to lower first queries latency
+          and to prevent clickhouse executable from being paged out under high IO load.
+         Enabling this option is recommended but will lead to increased startup time for up to a few seconds.
+    -->
+    <mlock_executable>false</mlock_executable>
+
+    <!-- Configuration of clusters that could be used in Distributed tables.
+         https://clickhouse.yandex/docs/en/table_engines/distributed/
+      -->
+    <remote_servers incl="remote" >
+        <!-- Test only shard config for testing distributed storage -->
+        <test_shard_localhost>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_shard_localhost>
+        <test_cluster_two_shards_localhost>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+         </test_cluster_two_shards_localhost>
+        <test_shard_localhost_secure>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </test_shard_localhost_secure>
+        <test_unavailable_shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>1</port>
+                </replica>
+            </shard>
+        </test_unavailable_shard>
+    </remote_servers>
+
+
+    <!-- If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file.
+         By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element.
+         Values for substitutions are specified in /yandex/name_of_substitution elements in that file.
+      -->
+
+    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+         Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.yandex/docs/en/table_engines/replication/
+      -->
+    <zookeeper incl="zookeeper" optional="true" />
+
+    <!-- Substitutions for parameters of replicated tables.
+          Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.yandex/docs/en/table_engines/replication/#creating-replicated-tables
+      -->
+    <macros incl="macros" optional="true" />
+
+
+    <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->
+    <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
+
+
+    <!-- Maximum session timeout, in seconds. Default: 3600. -->
+    <max_session_timeout>3600</max_session_timeout>
+
+    <!-- Default session timeout, in seconds. Default: 60. -->
+    <default_session_timeout>60</default_session_timeout>
+
+    <!-- Sending data to Graphite for monitoring. Several sections can be defined. -->
+    <!--
+        interval - send every X second
+        root_path - prefix for keys
+        hostname_in_path - append hostname to root_path (default = true)
+        metrics - send data from table system.metrics
+        events - send data from table system.events
+        asynchronous_metrics - send data from table system.asynchronous_metrics
+    -->
+    <!--
+    <graphite>
+        <host>localhost</host>
+        <port>42000</port>
+        <timeout>0.1</timeout>
+        <interval>60</interval>
+        <root_path>one_min</root_path>
+        <hostname_in_path>true</hostname_in_path>
+
+        <metrics>true</metrics>
+        <events>true</events>
+        <asynchronous_metrics>true</asynchronous_metrics>
+    </graphite>
+    <graphite>
+        <host>localhost</host>
+        <port>42000</port>
+        <timeout>0.1</timeout>
+        <interval>1</interval>
+        <root_path>one_sec</root_path>
+
+        <metrics>true</metrics>
+        <events>true</events>
+        <asynchronous_metrics>false</asynchronous_metrics>
+    </graphite>
+    -->
+
+
+    <!-- Query log. Used only for queries with setting log_queries = 1. -->
+    <query_log>
+        <!-- What table to insert data. If table is not exist, it will be created.
+             When query log structure is changed after system update,
+              then old table will be renamed and new table will be created automatically.
+        -->
+        <database>system</database>
+        <table>query_log</table>
+        <!--
+            PARTITION BY expr https://clickhouse.yandex/docs/en/table_engines/custom_partitioning_key/
+            Example:
+                event_date
+                toMonday(event_date)
+                toYYYYMM(event_date)
+                toStartOfHour(event_time)
+        -->
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <!-- Interval of flushing data. -->
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+
+    <!-- Trace log. Stores stack traces collected by query profilers.
+         See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings. -->
+    <trace_log>
+        <database>system</database>
+        <table>trace_log</table>
+
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </trace_log>
+
+    <!-- Query thread log. Has information about all threads participated in query execution.
+         Used only for queries with setting log_query_threads = 1. -->
+    <query_thread_log>
+        <database>system</database>
+        <table>query_thread_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_thread_log>
+
+    <!-- Uncomment if use part log.
+         Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads).
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </part_log>
+    -->
+
+    <!-- Uncomment to write text log into table.
+         Text log contains all information from usual server log but stores it in structured and efficient way.
+    <text_log>
+        <database>system</database>
+        <table>text_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </text_log>
+    -->
+
+    <!-- Parameters for embedded dictionaries, used in Yandex.Metrica.
+         See https://clickhouse.yandex/docs/en/dicts/internal_dicts/
+    -->
+
+    <!-- Path to file with region hierarchy. -->
+    <!-- <path_to_regions_hierarchy_file>/opt/geo/regions_hierarchy.txt</path_to_regions_hierarchy_file> -->
+
+    <!-- Path to directory with files containing names of regions -->
+    <!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->
+
+
+    <!-- Configuration of external dictionaries. See:
+         https://clickhouse.yandex/docs/en/dicts/external_dicts/
+    -->
+    <dictionaries_config>*_dictionary.xml</dictionaries_config>
+
+    <!-- Uncomment if you want data to be compressed 30-100% better.
+         Don't do that if you just started using ClickHouse.
+      -->
+    <compression incl="compression">
+    <!--
+        <!- - Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. - ->
+        <case>
+
+            <!- - Conditions. All must be satisfied. Some conditions may be omitted. - ->
+            <min_part_size>10000000000</min_part_size>        <!- - Min part size in bytes. - ->
+            <min_part_size_ratio>0.01</min_part_size_ratio>   <!- - Min size of part relative to whole table size. - ->
+
+            <!- - What compression method to use. - ->
+            <method>zstd</method>
+        </case>
+    -->
+    </compression>
+
+    <!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
+         Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
+    <distributed_ddl>
+        <!-- Path in ZooKeeper to queue with DDL queries -->
+        <path>/clickhouse/task_queue/ddl</path>
+
+        <!-- Settings from this profile will be used to execute DDL queries -->
+        <!-- <profile>default</profile> -->
+    </distributed_ddl>
+
+    <!-- Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -->
+    <!--
+    <merge_tree>
+        <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
+    </merge_tree>
+    -->
+
+    <!-- Protection from accidental DROP.
+         If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
+         If you want do delete one table and don't want to restart clickhouse-server, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
+         By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables.
+         The same for max_partition_size_to_drop.
+         Uncomment to disable protection.
+    -->
+    <!-- <max_table_size_to_drop>0</max_table_size_to_drop> -->
+    <!-- <max_partition_size_to_drop>0</max_partition_size_to_drop> -->
+
+    <!-- Example of parameters for GraphiteMergeTree table engine -->
+    <graphite_rollup_example>
+        <pattern>
+            <regexp>click_cost</regexp>
+            <function>any</function>
+            <retention>
+                <age>0</age>
+                <precision>3600</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>60</precision>
+            </retention>
+        </pattern>
+        <default>
+            <function>max</function>
+            <retention>
+                <age>0</age>
+                <precision>60</precision>
+            </retention>
+            <retention>
+                <age>3600</age>
+                <precision>300</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>3600</precision>
+            </retention>
+        </default>
+    </graphite_rollup_example>
+
+    <!-- Directory in <clickhouse-path> containing schema files for various input formats.
+         The directory will be created if it doesn't exist.
+      -->
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <!-- Uncomment to disable ClickHouse internal DNS caching. -->
+    <!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
+</yandex>

tests/testflows/example/configs/clickhouse/ssl/dhparam.pem

+-----BEGIN DH PARAMETERS-----
+MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR
+XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5
+Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb
+9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf
+slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb
+dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg==
+-----END DH PARAMETERS-----

tests/testflows/example/configs/clickhouse/ssl/server.crt

+-----BEGIN CERTIFICATE-----
+MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow
+FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI
+4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T
+4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU
+7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj
+sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg
+pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC
+kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j
+4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr
+85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C
+L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD
++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L
+P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp
+0g==
+-----END CERTIFICATE-----

tests/testflows/example/configs/clickhouse/ssl/server.key

+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn
+DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L
+OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus
+TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7
+JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F
+KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6
+ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP
+TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f
+7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN
+MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5
+212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu
+plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr
+/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu
+Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa
+CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g
+1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5
+bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq
+cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez
+on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K
+GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54
+nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU
+2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh
+vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa
+dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr
+vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1
+KCBtmIrQLqzMpnBpLNuSY+Q=
+-----END PRIVATE KEY-----

tests/testflows/example/configs/clickhouse/users.xml

+<?xml version="1.0"?>
+<yandex>
+    <!-- Profiles of settings. -->
+    <profiles>
+        <!-- Default settings. -->
+        <default>
+            <!-- Maximum memory usage for processing single query, in bytes. -->
+            <max_memory_usage>10000000000</max_memory_usage>
+
+            <!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. -->
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+
+            <!-- How to choose between replicas during distributed query processing.
+                 random - choose random replica from set of replicas with minimum number of errors
+                 nearest_hostname - from set of replicas with minimum number of errors, choose replica
+                  with minimum number of different symbols between replica's hostname and local hostname
+                  (Hamming distance).
+                 in_order - first live replica is chosen in specified order.
+                 first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
+            -->
+            <load_balancing>random</load_balancing>
+        </default>
+
+        <!-- Profile that allows only read queries. -->
+        <readonly>
+            <readonly>1</readonly>
+        </readonly>
+    </profiles>
+
+    <!-- Users and ACL. -->
+    <users>
+        <!-- If user name was not specified, 'default' user is used. -->
+        <default>
+            <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+
+                 If you want to specify password in plaintext (not recommended), place it in 'password' element.
+                 Example: <password>qwerty</password>.
+                 Password could be empty.
+
+                 If you want to specify SHA256, place it in 'password_sha256_hex' element.
+                 Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
+                 Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
+
+                 If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
+                 Example: <password_double_sha1_hex>e395796d6546b1b65db9d665cd43f0e858dd4303</password_double_sha1_hex>
+
+                 How to generate decent password:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+                 In first line will be password and in second - corresponding SHA256.
+
+                 How to generate double SHA1:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | openssl dgst -sha1 -binary | openssl dgst -sha1
+                 In first line will be password and in second - corresponding double SHA1.
+            -->
+            <password></password>
+
+            <!-- List of networks with open access.
+
+                 To open access from everywhere, specify:
+                    <ip>::/0</ip>
+
+                 To open access only from localhost, specify:
+                    <ip>::1</ip>
+                    <ip>127.0.0.1</ip>
+
+                 Each element of list has one of the following forms:
+                 <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
+                     2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+                 <host> Hostname. Example: server01.yandex.ru.
+                     To check access, DNS query is performed, and all received addresses compared to peer address.
+                 <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
+                     To check access, DNS PTR query is performed for peer address and then regexp is applied.
+                     Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
+                     Strongly recommended that regexp is ends with $
+                 All results of DNS requests are cached till server restart.
+            -->
+            <networks incl="networks" replace="replace">
+                <ip>::/0</ip>
+            </networks>
+
+            <!-- Settings profile for user. -->
+            <profile>default</profile>
+
+            <!-- Quota for user. -->
+            <quota>default</quota>
+
+            <!-- Allow access management -->
+            <access_management>1</access_management>
+
+            <!-- Example of row level security policy. -->
+            <!-- <databases>
+                <test>
+                    <filtered_table1>
+                        <filter>a = 1</filter>
+                    </filtered_table1>
+                    <filtered_table2>
+                        <filter>a + b &lt; 1 or c - d &gt; 5</filter>
+                    </filtered_table2>
+                </test>
+            </databases> -->
+        </default>
+
+        <!-- Example of user with readonly access. -->
+        <!-- <readonly>
+            <password></password>
+            <networks incl="networks" replace="replace">
+                <ip>::1</ip>
+                <ip>127.0.0.1</ip>
+            </networks>
+            <profile>readonly</profile>
+            <quota>default</quota>
+        </readonly> -->
+    </users>
+
+    <!-- Quotas. -->
+    <quotas>
+        <!-- Name of quota. -->
+        <default>
+            <!-- Limits for time interval. You could specify many intervals with different limits. -->
+            <interval>
+                <!-- Length of interval. -->
+                <duration>3600</duration>
+
+                <!-- No limits. Just calculate resource usage for time interval. -->
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</yandex>

tests/testflows/example/configs/clickhouse1/config.d/macros.xml

+<?xml version="1.0"?>
+<yandex>
+    <macros>
+        <replica>clickhouse1</replica>
+        <shard>01</shard>
+        <shard2>01</shard2>
+    </macros>
+</yandex>

tests/testflows/example/docker-compose/clickhouse-service.yml

+version: '2.3'
+
+services:
+  clickhouse:
+    image: yandex/clickhouse-integration-test
+    expose:
+      - "9000"
+      - "9009"
+      - "8123"
+    volumes:
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml"
+      - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse"
+      - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge"
+    entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log"
+    healthcheck:
+      test: clickhouse client --query='select 1'
+      interval: 3s
+      timeout: 5s
+      retries: 40
+      start_period: 2s
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - label:disable

tests/testflows/example/docker-compose/docker-compose.yml

+version: '2.3'
+
+services:
+  zookeeper:
+    extends:
+      file: zookeeper-service.yml
+      service: zookeeper
+
+  clickhouse1:
+    extends:
+      file: clickhouse-service.yml
+      service: clickhouse
+    hostname: clickhouse1
+    volumes:
+      - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/"
+      - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d"
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+
+  # dummy service which does nothing, but allows to postpone 
+  # 'docker-compose up -d' till all dependecies will go healthy
+  all_services_ready:
+    image: hello-world
+    depends_on:
+      clickhouse1:
+        condition: service_healthy
+      zookeeper:
+        condition: service_healthy

tests/testflows/example/docker-compose/zookeeper-service.yml

+version: '2.3'
+
+services:
+  zookeeper:
+    image: zookeeper:3.4.12
+    expose:
+      - "2181"
+    environment:
+      ZOO_TICK_TIME: 500
+      ZOO_MY_ID: 1
+    healthcheck:
+      test: echo stat | nc localhost 2181
+      interval: 3s
+      timeout: 2s
+      retries: 5
+      start_period: 2s
+    security_opt:
+      - label:disable

tests/testflows/example/regression.py

+#!/usr/bin/env python3
+import sys
+from testflows.core import *
+
+append_path(sys.path, "..") 
+
+from helpers.cluster import Cluster
+from helpers.argparser import argparser
+
+@TestFeature
+@Name("example")
+@ArgumentParser(argparser)
+def regression(self, local, clickhouse_binary_path):
+    """Simple example of how you can use TestFlows to test ClickHouse.
+    """
+    nodes = {
+        "clickhouse": ("clickhouse1",),
+    }
+ 
+    with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster:
+        self.context.cluster = cluster
+
+        Scenario(run=load("example.tests.example", "scenario"))
+
+if main():
+    regression()

tests/testflows/example/requirements/CH_SRS001_ClickHouse.md

+# CH-SRS001 ClickHouse Software Requirements Specification Template
+
+**Author:** [name of the author]
+
+**Date:** [date]
+
+## Table of Contents
+
+* 1 [Revision History](#revision-history)
+* 2 [Introduction](#introduction)
+  * 2.1 [Table of Contents](#table-of-contents)
+  * 2.2 [Generating HTML version](#generating-html-version)
+  * 2.3 [Generating Python Requirements](#generating-python-requirements)
+* 3 [Terminology](#terminology)
+  * 3.1 [SRS](#srs)
+  * 3.2 [Some term that you will use](#some-term-that-you-will-use)
+* 4 [Requirements](#requirements)
+  * 4.1 [RQ.CH-SRS001.Example](#rqch-srs001example)
+  * 4.2 [RQ.CH-SRS001.Example.Subgroup](#rqch-srs001examplesubgroup)
+  * 4.3 [RQ.CH-SRS001.Example.Select.1](#rqch-srs001exampleselect1)
+* 5 [References](#references)
+
+## Revision History
+
+This document is stored in an electronic form using [Git] source control management software.
+
+## Introduction
+
+This section provides an introduction to the project or the feature.
+All [SRS] documents must be uniquely identified by a number. In this
+case this document is identified by the number
+
+    CH-SRS001
+
+The document number must always be used as a prefix to the document title. For example,
+
+    CH-SRSxxx Name of the document
+
+All the requirements must be specified in the [Requirements](#requirements) section.
+
+### Table of Contents
+
+Note that currently the table of contents is generated manually using 
+
+```bash
+cat CH_SRS001_ClickHouse.md | tfs document toc
+```
+
+command and needs to be updated any time requirement name is changed
+or a new requirement is added.
+
+### Generating HTML version
+
+You can easily generate a pretty HTML version of this document using the command.
+
+```bash
+cat CH_SRS001_ClickHouse.md | tfs document convert > CH_SRS001_ClickHouse.html
+```
+
+### Generating Python Requirements
+
+You can convert this [SRS] into the `requirements.py` by using the command.
+
+```bash
+cat CH_SRS001_ClickHouse.md | tfs requirements generate > requirements.py
+```
+
+## Terminology
+
+You can define terminolgy using the examples below and you can make them
+linkable as [SRS] by defining the links in the [References](#References) section.
+
+### SRS
+
+Software Requirements Specification
+
+### Some term that you will use
+
+Some description of the term that you would like to use.
+
+## Requirements
+
+This section includes all the requirements. This section can be structured in any way one sees fit. 
+
+Each requirement is defined by the section that starts with
+the following prefix:
+
+    RQ.[document id].[requirement name]
+
+then immediately followed by a one-line block that contains the 
+the `version` of the requirement.
+
+### RQ.CH-SRS001.Example
+version: 1.0
+
+This is a long description of the requirement that can include any
+relevant information. 
+
+The one-line block that follows the requirement defines the `version` 
+of the requirement. The version is controlled manually and is used
+to indicate material changes to the requirement that would 
+require tests that cover this requirement to be updated.
+
+It is a good practice to use requirement names that are broken
+up into groups. It is not recommended to use only numbers
+because if the requirement must be moved the numbering will not match.
+Therefore, the requirement name should start with the group
+name which is then followed by a number if any. For example,
+
+    RQ.SRS001.Group.Subgroup.1
+
+To keep names short, try to use abbreviations for the requirement's group name.
+
+### RQ.CH-SRS001.Example.Subgroup
+version: 1.0
+
+This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example).
+
+### RQ.CH-SRS001.Example.Select.1
+version: 1.0
+
+[ClickHouse] SHALL return `1` when user executes query
+
+```sql
+SELECT 1
+```
+
+## References
+
+* **ClickHouse:** https://clickhouse.tech
+
+[SRS]: #SRS
+[Some term that you will use]: #Sometermthatyouwilluse
+[ClickHouse]: https://clickhouse.tech
+[Git]: https://git-scm.com/

tests/testflows/example/requirements/__init__.py

+from .requirements import *

tests/testflows/example/requirements/requirements.py

+# These requirements were auto generated
+# from software requirements specification (SRS)
+# document by TestFlows v1.6.200627.1211752.
+# Do not edit by hand but re-generate instead
+# using 'tfs requirements generate' command.
+from testflows.core import Requirement
+
+RQ_CH_SRS001_Example = Requirement(
+        name='RQ.CH-SRS001.Example',
+        version='1.0',
+        priority=None,
+        group=None,
+        type=None,
+        uid=None,
+        description=(
+        'This is a long description of the requirement that can include any\n'
+        'relevant information. \n'
+        '\n'
+        'The one-line block that follows the requirement defines the `version` \n'
+        'of the requirement. The version is controlled manually and is used\n'
+        'to indicate material changes to the requirement that would \n'
+        'require tests that cover this requirement to be updated.\n'
+        '\n'
+        'It is a good practice to use requirement names that are broken\n'
+        'up into groups. It is not recommended to use only numbers\n'
+        'because if the requirement must be moved the numbering will not match.\n'
+        'Therefore, the requirement name should start with the group\n'
+        'name which is then followed by a number if any. For example,\n'
+        '\n'
+        '    RQ.SRS001.Group.Subgroup.1\n'
+        '\n'
+        "To keep names short, try to use abbreviations for the requirement's group name.\n"
+        ),
+        link=None
+    )
+
+RQ_CH_SRS001_Example_Subgroup = Requirement(
+        name='RQ.CH-SRS001.Example.Subgroup',
+        version='1.0',
+        priority=None,
+        group=None,
+        type=None,
+        uid=None,
+        description=(
+        'This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example).\n'
+        ),
+        link=None
+    )
+
+RQ_CH_SRS001_Example_Select_1 = Requirement(
+        name='RQ.CH-SRS001.Example.Select.1',
+        version='1.0',
+        priority=None,
+        group=None,
+        type=None,
+        uid=None,
+        description=(
+        '[ClickHouse] SHALL return `1` when user executes query\n'
+        '\n'
+        '```sql\n'
+        'SELECT 1\n'
+        '```\n'
+        ),
+        link=None
+    )

tests/testflows/example/tests/example.py

+from testflows.core import *
+from testflows.asserts import error
+
+from example.requirements import *
+
+@TestScenario
+@Name("select 1")
+@Requirements(
+    RQ_CH_SRS001_Example_Select_1("1.0")
+)
+def scenario(self, node="clickhouse1"):
+    """Check that ClickHouse returns 1 when user executes `SELECT 1` query.
+    """
+    node = self.context.cluster.node(node)
+
+    with When("I execute query select 1"):
+        r = node.query("SELECT 1").output.strip()
+
+    with Then("the result should be 1"):
+        assert r == "1", error()

tests/testflows/helpers/argparser.py

+import os
+
+def argparser(parser):
+    """Default argument parser for regressions.
+    """
+    parser.add_argument("--local", 
+        action="store_true",
+        help="run regression in local mode", default=False)
+    
+    parser.add_argument("--clickhouse-binary-path", 
+        type=str, dest="clickhouse_binary_path",
+        help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path",
+        default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse"))

tests/testflows/helpers/cluster.py

+import os
+import time
+import inspect
+import threading
+import tempfile
+
+from testflows.core import *
+from testflows.asserts import error
+from testflows.connect import Shell
+
+class QueryRuntimeException(Exception):
+    """Exception during query execution on the server.
+    """
+    pass
+
+class Node(object):
+    """Generic cluster node.
+    """
+    config_d_dir = "/etc/clickhouse-server/config.d/"
+
+    def __init__(self, cluster, name):
+        self.cluster = cluster
+        self.name = name
+
+    def repr(self):
+        return f"Node(name='{self.name}')"
+
+    def restart(self, timeout=120, safe=True):
+        """Restart node.
+        """
+        with self.cluster.lock:
+            for key in list(self.cluster._bash.keys()):
+                if key.endswith(f"-{self.name}"):
+                    shell = self.cluster._bash.pop(key)
+                    shell.__exit__(None, None, None)
+
+        self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout)
+
+    def command(self, *args, **kwargs):
+        return self.cluster.command(self.name, *args, **kwargs)
+
+class ClickHouseNode(Node):
+    """Node with ClickHouse server.
+    """
+    def wait_healthy(self, timeout=120):
+        with By(f"waiting until container {self.name} is healthy"):
+            start_time = time.time()
+            while True:
+                if self.query("select 1", no_checks=1, timeout=120, steps=False).exitcode == 0:
+                    break
+                if time.time() - start_time < timeout:
+                    time.sleep(2)
+                    continue
+                assert False, "container is not healthy"
+
+    def restart(self, timeout=120, safe=True):
+        """Restart node.
+        """
+        if safe:
+            self.query("SYSTEM STOP MOVES")
+            self.query("SYSTEM STOP MERGES")
+            self.query("SYSTEM FLUSH LOGS")
+            with By("waiting for 5 sec for moves and merges to stop"):
+                time.sleep(5)
+            with And("forcing to sync everything to disk"):
+                self.command("sync", timeout=30)
+
+        with self.cluster.lock:
+            for key in list(self.cluster._bash.keys()):
+                if key.endswith(f"-{self.name}"):
+                    shell = self.cluster._bash.pop(key)
+                    shell.__exit__(None, None, None)
+
+        self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout)
+
+        self.wait_healthy(timeout)
+
+    def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False,
+              raise_on_exception=False, step=By, settings=None, *args, **kwargs):
+        """Execute and check query.
+
+        :param sql: sql query
+        :param message: expected message that should be in the output, default: None
+        :param exitcode: expected exitcode, default: None
+        """
+        if len(sql) > 1024:
+            with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query:
+                query.write(sql)
+                query.flush()
+                command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n"
+                for setting in settings or []:
+                    name, value = setting
+                    command += f" --{name} \"{value}\""
+                description = f"""
+                    echo -e \"{sql[:100]}...\" > {query.name} 
+                    {command}
+                """
+                with step("executing command", description=description) if steps else NullStep():
+                    r = self.cluster.bash(None)(command, *args, **kwargs)
+        else:
+            command = f"echo -e \"{sql}\" | clickhouse client -n"
+            for setting in settings or []:
+                name, value = setting
+                command += f" --{name} \"{value}\""
+            with step("executing command", description=command) if steps else NullStep():
+                r = self.cluster.bash(self.name)(command, *args, **kwargs)
+
+        if no_checks:
+            return r
+
+        if exitcode is not None:
+            with Then(f"exitcode should be {exitcode}") if steps else NullStep():
+                assert r.exitcode == exitcode, error(r.output)
+
+        if message is not None:
+            with Then(f"output should contain message", description=message) if steps else NullStep():
+                assert message in r.output, error(r.output)
+
+        if message is None or "Exception:" not in message:
+            with Then("check if output has exception") if steps else NullStep():
+                if "Exception:" in r.output:
+                    if raise_on_exception:
+                        raise QueryRuntimeException(r.output)
+                    assert False, error(r.output)
+
+        return r
+
+class Cluster(object):
+    """Simple object around docker-compose cluster.
+    """
+    def __init__(self, local=False, 
+            clickhouse_binary_path=None, configs_dir=None, 
+            nodes=None,
+            docker_compose="docker-compose", docker_compose_project_dir=None, 
+            docker_compose_file="docker-compose.yml"):
+        
+        self._bash = {}
+        self.clickhouse_binary_path = clickhouse_binary_path
+        self.configs_dir = configs_dir
+        self.local = local
+        self.nodes = nodes or {}
+        self.docker_compose = docker_compose
+
+        frame = inspect.currentframe().f_back
+        caller_dir = os.path.dirname(os.path.abspath(frame.f_globals["__file__"]))
+
+        # auto set configs directory
+        if self.configs_dir is None:
+            caller_configs_dir = caller_dir
+            if os.path.exists(caller_configs_dir):
+                self.configs_dir = caller_configs_dir
+
+        if not os.path.exists(self.configs_dir):
+            raise TypeError("configs directory '{self.configs_dir}' does not exist")
+
+        # auto set docker-compose project directory
+        if docker_compose_project_dir is None:
+            caller_project_dir = os.path.join(caller_dir, "docker-compose")
+            if os.path.exists(caller_project_dir):
+                docker_compose_project_dir = caller_project_dir
+
+        docker_compose_file_path = os.path.join(docker_compose_project_dir or "", docker_compose_file) 
+
+        if not os.path.exists(docker_compose_file_path):
+            raise TypeError("docker compose file '{docker_compose_file_path}' does not exist")
+
+        self.docker_compose += f" --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\""
+        self.lock = threading.Lock()
+
+    def bash(self, node, timeout=60):
+        """Returns thread-local bash terminal
+        to a specific node.
+
+        :param node: name of the service
+        """
+        current_thread = threading.current_thread()
+        id = f"{current_thread.ident}-{node}"
+        with self.lock:
+            if self._bash.get(id) is None:
+                if node is None:
+                    self._bash[id] = Shell().__enter__()
+                else:
+                    self._bash[id] = Shell(command=[
+                        "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting"
+                    ], name=node).__enter__()
+                self._bash[id].timeout = timeout
+            return self._bash[id]
+
+    def __enter__(self):
+        with Given("docker-compose cluster"):
+            self.up()
+        return self
+
+    def __exit__(self, type, value, traceback):
+        try:
+            with Finally("I clean up"):
+                self.down()
+        finally:
+            with self.lock:
+                for shell in self._bash.values():
+                    shell.__exit__(type, value, traceback)
+
+    def node(self, name):
+        """Get object with node bound methods.
+
+        :param name: name of service name
+        """
+        if name.startswith("clickhouse"):
+            return ClickHouseNode(self, name)
+        return Node(self, name)
+
+    def down(self, timeout=120):
+        """Bring cluster down by executing docker-compose down."""
+        try:
+            bash = self.bash(None)
+            with self.lock:
+                # remove and close all not None node terminals
+                for id in list(self._bash.keys()):
+                    shell = self._bash.pop(id)
+                    if shell is not bash:
+                        shell.__exit__(None, None, None)
+                    else:
+                        self._bash[id] = shell
+        finally:
+            return self.command(None, f"{self.docker_compose} down", timeout=timeout)
+
+    def up(self):
+        if self.local:
+            with Given("I am running in local mode"):
+                with Then("check --clickhouse-binary-path is specified"):
+                    assert self.clickhouse_binary_path, "when running in local mode then --clickhouse-binary-path must be specified"
+                with And("path should exist"):
+                    assert os.path.exists(self.clickhouse_binary_path)
+
+            os.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = self.clickhouse_binary_path
+            os.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = os.path.join(os.path.dirname(self.clickhouse_binary_path),
+                                                                               "clickhouse-odbc-bridge")
+            os.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir
+
+            with Given("docker-compose"):
+                self.command(None, "env | grep CLICKHOUSE")
+                cmd = self.command(None, f'{self.docker_compose} up -d 2>&1 | tee', timeout=30 * 60)
+        else:
+            with Given("docker-compose"):
+                cmd = self.command(None, f'{self.docker_compose} up -d --no-recreate 2>&1 | tee')
+
+        with Then("check there are no unhealthy containers"):
+            assert "is unhealthy" not in cmd.output, error()
+
+        with Then("wait all nodes report healhy"):
+            for name in self.nodes["clickhouse"]:
+                self.node(name).wait_healthy()
+
+    def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs):
+        """Execute and check command.
+
+        :param node: name of the service
+        :param command: command
+        :param message: expected message that should be in the output, default: None
+        :param exitcode: expected exitcode, default: None
+        :param steps: don't break command into steps, default: True
+        """
+        debug(f"command() {node}, {command}")
+        with By("executing command", description=command) if steps else NullStep():
+            r = self.bash(node)(command, *args, **kwargs)
+        if exitcode is not None:
+            with Then(f"exitcode should be {exitcode}") if steps else NullStep():
+                assert r.exitcode == exitcode, error(r.output)
+        if message is not None:
+            with Then(f"output should contain message", description=message) if steps else NullStep():
+                assert message in r.output, error(r.output)
+        return r

tests/testflows/regression.py

+#!/usr/bin/env python3
+import sys
+from testflows.core import *
+
+append_path(sys.path, "."), 
+
+from helpers.argparser import argparser
+
+@TestModule
+@Name("clickhouse")
+@ArgumentParser(argparser)
+def regression(self, local, clickhouse_binary_path):
+    """ClickHouse regression.
+    """
+    Feature(test=load("example.regression", "regression"))(
+        local=local, clickhouse_binary_path=clickhouse_binary_path)
+
+if main():
+    regression()

tests/testflows/runner

+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+import subprocess
+import os
+import getpass
+import argparse
+import logging
+import signal
+import subprocess
+import sys
+
+CUR_FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+DEFAULT_CLICKHOUSE_ROOT = os.path.abspath(os.path.join(CUR_FILE_DIR, "../../"))
+CURRENT_WORK_DIR = os.getcwd()
+CONTAINER_NAME = "clickhouse_testflows_tests"
+
+DIND_TESTFLOWS_TESTS_IMAGE_NAME = "yandex/clickhouse-testflows-runner"
+
+def check_args_and_update_paths(args):
+    if not os.path.isabs(args.binary):
+        args.binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.binary))
+
+    if not args.bridge_binary:
+        args.bridge_binary = os.path.join(os.path.dirname(args.binary), 'clickhouse-odbc-bridge')
+    elif not os.path.isabs(args.bridge_binary):
+        args.bridge_binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.bridge_binary))
+
+    if not os.path.isabs(args.configs_dir):
+        args.configs_dir = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.configs_dir))
+
+    if not os.path.isabs(args.clickhouse_root):
+        args.clickhouse_root = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.clickhouse_root))
+
+    for path in [args.binary, args.configs_dir, args.clickhouse_root]:
+        if not os.path.exists(path):
+            raise Exception("Path {} doesn't exists".format(path))
+
+def docker_kill_handler_handler(signum, frame):
+    subprocess.check_call('docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format(name=CONTAINER_NAME), shell=True)
+    raise KeyboardInterrupt("Killed by Ctrl+C")
+
+signal.signal(signal.SIGINT, docker_kill_handler_handler)
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    parser = argparse.ArgumentParser(description="ClickHouse testflows runner")
+
+    parser.add_argument(
+        "--binary",
+        default=os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH", os.environ.get("CLICKHOUSE_TESTS_CLIENT_BIN_PATH", "/usr/bin/clickhouse")),
+        help="Path to clickhouse binary")
+
+    parser.add_argument(
+        "--bridge-binary",
+        default=os.environ.get("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH", ""),
+        help="Path to clickhouse-odbc-bridge binary. Defaults to clickhouse-odbc-bridge in the same dir as clickhouse.")
+
+    parser.add_argument(
+        "--configs-dir",
+        default=os.environ.get("CLICKHOUSE_TESTS_BASE_CONFIG_DIR", os.path.join(DEFAULT_CLICKHOUSE_ROOT, "programs/server")),
+        help="Path to clickhouse configs directory")
+
+    parser.add_argument(
+        "--clickhouse-root",
+        default=DEFAULT_CLICKHOUSE_ROOT,
+        help="Path to repository root folder")
+
+    parser.add_argument(
+        "--command",
+        default='',
+        help="Set it to run some other command in container (for example bash)")
+
+    parser.add_argument(
+        "--disable-net-host",
+        action='store_true',
+        default=False,
+        help="Don't use net host in parent docker container")
+
+    parser.add_argument(
+        "--docker-image-version",
+        default="latest",
+        help="Version of docker image which runner will use to run tests")
+
+
+    parser.add_argument('testflows_args', nargs='*', help="args for testflows command")
+
+    args = parser.parse_args()
+
+    check_args_and_update_paths(args)
+
+    net = ""
+    if not args.disable_net_host:
+        net = "--net=host"
+
+    # create named volume which will be used inside to store images and other docker related files,
+    # to avoid redownloading it every time
+    #
+    # should be removed manually when not needed
+    subprocess.check_call('docker volume create {name}_volume'.format(name=CONTAINER_NAME), shell=True)
+
+    # enable tty mode & interactive for docker if we have real tty
+    tty = ""
+    if sys.stdout.isatty() and sys.stdin.isatty():
+        tty = "-it"
+
+    cmd = "docker run {net} {tty} --rm --name {name} --privileged --volume={bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \
+        --volume={cfg}:/clickhouse-config --volume={pth}:/ClickHouse --volume={name}_volume:/var/lib/docker -e TESTFLOWS_OPTS='{opts}' {img} {command}".format(
+        net=net,
+        tty=tty,
+        bin=args.binary,
+        bridge_bin=args.bridge_binary,
+        cfg=args.configs_dir,
+        pth=args.clickhouse_root,
+        opts=' '.join(args.testflows_args),
+        img=DIND_TESTFLOWS_TESTS_IMAGE_NAME + ":" + args.docker_image_version,
+        name=CONTAINER_NAME,
+        command=args.command
+    )
+
+    print("Running testflows container as: '" + cmd + "'.")
+    subprocess.check_call(cmd, shell=True)