Draft.

docker/test/integration/base/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get update \
         odbc-postgresql \
         sqlite3 \
         curl \
+        bind9-host \
         tar
 RUN rm -rf \
         /var/lib/apt/lists/* \

docker/test/stateless_unbundled/Dockerfile

 # docker build -t yandex/clickhouse-stateless-unbundled-test .
-FROM yandex/clickhouse-test-base
+FROM ubuntu:20.04
 
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
 
-RUN apt-get --allow-unauthenticated update -y \
-    && env DEBIAN_FRONTEND=noninteractive \
-        apt-get --allow-unauthenticated install --yes --no-install-recommends \
-            alien \
+ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10
+
+RUN apt-get update \
+    && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
+        --yes --no-install-recommends --verbose-versions \
+    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
+    && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
+    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
+    && apt-key add /tmp/llvm-snapshot.gpg.key \
+    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
+    && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
+        /etc/apt/sources.list
+
+# initial packages
+RUN apt-get update \
+    && apt-get install \
+        bash \
+        fakeroot \
+        ccache \
+        curl \
+        software-properties-common \
+        --yes --no-install-recommends
+
+# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
+# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
+# Significantly increase deb packaging speed and compatible with old systems
+RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \
+    && chmod +x dpkg-deb \
+    && cp dpkg-deb /usr/bin
+
+
+RUN apt-get update \
+    && apt-get install \
+        clang-${LLVM_VERSION} \
+        debhelper \
+        devscripts \
+        gdb  \
+        git \
+        gperf \
+        lcov \
+        llvm-${LLVM_VERSION} \
+        moreutils \
+        perl \
+        perl \
+        pigz \
+        pkg-config \
+        tzdata \
+                    alien \
             brotli \
             cmake \
             devscripts \
@@ -56,8 +100,6 @@ RUN apt-get --allow-unauthenticated update -y \
             pkg-config \
             python \
             python-lxml \
-            python-requests \
-            python-termcolor \
             qemu-user-static \
             sudo \
             telnet \
@@ -68,7 +110,10 @@ RUN apt-get --allow-unauthenticated update -y \
             wget \
             zlib1g-dev \
             zookeeper \
-            zookeeperd
+            zookeeperd \
+            --yes --no-install-recommends
+
+
 
 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
    && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
@@ -77,6 +122,13 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \
    && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
    && rm -rf /tmp/clickhouse-odbc-tmp
 
+# Sanitizer options
+RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \
+  echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
+  echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \
+  ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer;
+
+
 ENV TZ=Europe/Moscow
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 

tests/integration/helpers/cluster.py

@@ -111,6 +111,7 @@ class ClickHouseCluster:
 
         custom_dockerd_host = custom_dockerd_host or os.environ.get('CLICKHOUSE_TESTS_DOCKERD_HOST')
         self.docker_api_version = os.environ.get("DOCKER_API_VERSION")
+        self.docker_base_tag = os.environ.get("DOCKER_BASE_TAG")
 
         self.base_cmd = ['docker-compose']
         if custom_dockerd_host:
@@ -165,7 +166,7 @@ class ClickHouseCluster:
                      with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None,
                      with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False,
                      with_redis=False, with_minio=False, with_cassandra=False,
-                     hostname=None, env_variables=None, image="yandex/clickhouse-integration-test",
+                     hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None,
                      stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None,
                      zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None):
         """Add an instance to the cluster.
@@ -183,13 +184,16 @@ class ClickHouseCluster:
         if name in self.instances:
             raise Exception("Can\'t add instance `%s': there is already an instance with the same name!" % name)
 
+        if tag is None:
+            tag = self.docker_base_tag
+
         instance = ClickHouseInstance(
             self, self.base_dir, name, config_dir, main_configs or [], user_configs or [], macros or {},
             with_zookeeper,
             self.zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra,
             self.base_configs_dir, self.server_bin_path,
             self.odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=hostname,
-            env_variables=env_variables or {}, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address,
+            env_variables=env_variables or {}, image=image, tag=tag, stay_alive=stay_alive, ipv4_address=ipv4_address,
             ipv6_address=ipv6_address,
             with_installed_binary=with_installed_binary, tmpfs=tmpfs or [])
 
@@ -704,7 +708,7 @@ DOCKER_COMPOSE_TEMPLATE = '''
 version: '2.3'
 services:
     {name}:
-        image: {image}
+        image: {image}:{tag}
         hostname: {hostname}
         volumes:
             - {configs_dir}:/etc/clickhouse-server/
@@ -739,7 +743,7 @@ class ClickHouseInstance:
             with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra,
             base_configs_dir, server_bin_path, odbc_bridge_bin_path,
             clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None,
-            image="yandex/clickhouse-integration-test",
+            image="yandex/clickhouse-integration-test", tag="latest",
             stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None):
 
         self.name = name
@@ -783,6 +787,7 @@ class ClickHouseInstance:
         self.client = None
         self.default_timeout = 20.0  # 20 sec
         self.image = image
+        self.tag = tag
         self.stay_alive = stay_alive
         self.ipv4_address = ipv4_address
         self.ipv6_address = ipv6_address
@@ -1160,6 +1165,7 @@ class ClickHouseInstance:
         with open(self.docker_compose_path, 'w') as docker_compose:
             docker_compose.write(DOCKER_COMPOSE_TEMPLATE.format(
                 image=self.image,
+                tag=self.tag,
                 name=self.name,
                 hostname=self.hostname,
                 binary_volume=binary_volume,

tests/integration/runner

@@ -154,6 +154,8 @@ if __name__ == "__main__":
                 env_tags += "-e {}={} ".format("DOCKER_MYSQL_PHP_CLIENT_TAG", tag)
             elif image == "yandex/clickhouse-postgresql-java-client":
                 env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag)
+            elif image == "yandex/clickhouse-integration-test":
+                env_tags += "-e {}={}".format("DOCKER_BASE_TAG", tag)
             else:
                 raise Exception("Unknown image {}".format(image))
 

tests/integration/test_host_ip_change/configs/dns_update_short.xml

 <yandex>
-    <dns_cache_update_period>2</dns_cache_update_period>
+    <dns_cache_update_period>1</dns_cache_update_period>
 </yandex>

tests/integration/test_host_ip_change/test.py

@@ -106,11 +106,16 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
 
     # Put some data to source node3
     node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)")
+
+
     # Check that data is placed on node3
     assert node3.query("SELECT count(*) from test_table_update") == "6\n"
 
+    result = node4.exec_in_container(["bash", "-c", "/usr/bin/host node3"])
+    print("HOST RESULT %s", result)
+
     # Because of DNS cache update, ip of node3 would be updated
-    assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "6")
+    assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "6", sleep_time=3)
 
     # Just to be sure check one more time
     node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 8)")