mysql/postgresql: move Dockerfiles and docker_compose to docker/test (#13448)

* mysql/postgresql: move Dockerfiles and docker_compose to docker/test
* Build client containers in CI
* pass tag to docker_compose

docker/images.json

@@ -103,5 +103,25 @@
     "docker/test/integration/helper_container": {
         "name": "yandex/clickhouse-integration-helper",
         "dependent": []
+    },
+    "docker/test/integration/mysql_golang_client": {
+        "name": "yandex/clickhouse-mysql-golang-client",
+        "dependent": []
+    },
+    "docker/test/integration/mysql_java_client": {
+        "name": "yandex/clickhouse-mysql-java-client",
+        "dependent": []
+    },
+    "docker/test/integration/mysql_js_client": {
+        "name": "yandex/clickhouse-mysql-js-client",
+        "dependent": []
+    },
+    "docker/test/integration/mysql_php_client": {
+        "name": "yandex/clickhouse-mysql-php-client",
+        "dependent": []
+    },
+    "docker/test/integration/postgresql_java_client": {
+        "name": "yandex/clickhouse-postgresql-java-client",
+        "dependent": []
     }
 }

tests/integration/test_mysql_protocol/clients/golang/Dockerfile → docker/test/integration/mysql_golang_client/Dockerfile

+# docker build -t yandex/clickhouse-mysql-golang-client .
+# MySQL golang client docker container
+
 FROM golang:1.12.2
 
 RUN go get "github.com/go-sql-driver/mysql"

tests/integration/test_mysql_protocol/clients/java/Dockerfile → docker/test/integration/mysql_java_client/Dockerfile

+# docker build -t yandex/clickhouse-mysql-java-client .
+# MySQL Java client docker container
+
 FROM ubuntu:18.04
 
 RUN apt-get update && \

tests/integration/test_mysql_protocol/clients/mysqljs/Dockerfile → docker/test/integration/mysql_js_client/Dockerfile

+# docker build -t yandex/clickhouse-mysql-js-client .
+# MySQL JavaScript client docker container
+
 FROM node:8
 
 RUN npm install mysql

tests/integration/test_mysql_protocol/clients/php-mysqlnd/Dockerfile → docker/test/integration/mysql_php_client/Dockerfile

+# docker build -t yandex/clickhouse-mysql-php-client .
+# MySQL PHP client docker container
+
 FROM php:7.3-cli
 
 COPY ./client.crt client.crt

tests/integration/test_postgresql_protocol/clients/java/Dockerfile → docker/test/integration/postgresql_java_client/Dockerfile

+# docker build -t yandex/clickhouse-postgresql-java-client .
+# PostgreSQL Java client docker container
+
 FROM ubuntu:18.04
 
 RUN apt-get update && \

tests/integration/test_mysql_protocol/clients/golang/docker_compose.yml → docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml

 version: '2.3'
 services:
   golang1:
-    build:
-      context: ./
-      network: host
+    image: yandex/clickhouse-mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG}
     # to keep container running
     command: sleep infinity

tests/integration/test_mysql_protocol/clients/java/docker_compose.yml → docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml

 version: '2.3'
 services:
   java1:
-    build:
-      context: ./
-      network: host
+    image: yandex/clickhouse-mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG}
     # to keep container running
     command: sleep infinity

tests/integration/test_mysql_protocol/clients/mysqljs/docker_compose.yml → docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml

 version: '2.3'
 services:
   mysqljs1:
-    build:
-      context: ./
-      network: host
+    image: yandex/clickhouse-mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG}
     # to keep container running
     command: sleep infinity

tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml → docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml

 version: '2.3'
 services:
   php1:
-    build:
-      context: ./
+    image: yandex/clickhouse-mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG}
     # to keep container running
     command: sleep infinity

tests/integration/test_postgresql_protocol/clients/java/docker_compose.yml → docker/test/integration/runner/compose/docker_compose_postgesql_java_client.yml

 version: '2.2'
 services:
   java:
-    build:
-      context: ./
-      network: host
+    image: yandex/clickhouse-postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG}
     # to keep container running
     command: sleep infinity

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -22,5 +22,11 @@ export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse
 export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
 export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
 
+export DOCKER_MYSQL_GOLANG_CLIENT_TAG=${DOCKER_MYSQL_GOLANG_CLIENT_TAG:=latest}
+export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest}
+export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest}
+export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest}
+export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest}
+
 cd /ClickHouse/tests/integration
 exec "$@"

tests/integration/runner

@@ -139,6 +139,24 @@ if __name__ == "__main__":
     if not args.disable_net_host:
         net = "--net=host"
 
+    env_tags = ""
+
+    if args.docker_compose_images_tags is not None:
+        for img_tag in args.docker_compose_images_tags:
+            [image, tag] = img_tag.split(":")
+            if image == "yandex/clickhouse-mysql-golang-client":
+                env_tags += "-e {}={} ".format("DOCKER_MYSQL_GOLANG_CLIENT_TAG", tag)
+            elif image == "yandex/clickhouse-mysql-java-client":
+                env_tags += "-e {}={} ".format("DOCKER_MYSQL_JAVA_CLIENT_TAG", tag)
+            elif image == "yandex/clickhouse-mysql-js-client":
+                env_tags += "-e {}={} ".format("DOCKER_MYSQL_JS_CLIENT_TAG", tag)
+            elif image == "yandex/clickhouse-mysql-php-client":
+                env_tags += "-e {}={} ".format("DOCKER_MYSQL_PHP_CLIENT_TAG", tag)
+            elif image == "yandex/clickhouse-postgresql-java-client":
+                env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag)
+            else:
+                raise Exception("Unknown image {}".format(image))
+
     # create named volume which will be used inside to store images and other docker related files,
     # to avoid redownloading it every time
     #
@@ -152,13 +170,14 @@ if __name__ == "__main__":
 
     cmd = "docker run {net} {tty} --rm --name {name} --privileged --volume={bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \
         --volume={base_cfg}:/clickhouse-config --volume={cases_dir}:/ClickHouse/tests/integration \
-        --volume={name}_volume:/var/lib/docker -e PYTEST_OPTS='{opts}' {img} {command}".format(
+        --volume={name}_volume:/var/lib/docker {env_tags} -e PYTEST_OPTS='{opts}' {img} {command}".format(
         net=net,
         tty=tty,
         bin=args.binary,
         bridge_bin=args.bridge_binary,
         base_cfg=args.base_configs_dir,
         cases_dir=args.cases_dir,
+        env_tags=env_tags,
         opts=' '.join(args.pytest_args),
         img=DIND_INTEGRATION_TESTS_IMAGE_NAME + ":" + args.docker_image_version,
         name=CONTAINER_NAME,

tests/integration/test_mysql_protocol/test.py

@@ -11,10 +11,11 @@ import pymysql.connections
 
 from docker.models.containers import Container
 
-from helpers.cluster import ClickHouseCluster
+from helpers.cluster import ClickHouseCluster, get_docker_compose_path
 
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+DOCKER_COMPOSE_PATH = get_docker_compose_path()
 
 config_dir = os.path.join(SCRIPT_DIR, './configs')
 cluster = ClickHouseCluster(__file__)
@@ -34,7 +35,7 @@ def server_address():
 
 @pytest.fixture(scope='module')
 def mysql_client():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'mysql', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_client.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_mysql1_1')
 
@@ -60,28 +61,28 @@ def mysql_server(mysql_client):
 
 @pytest.fixture(scope='module')
 def golang_container():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'golang', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_golang_client.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_golang1_1')
 
 
 @pytest.fixture(scope='module')
 def php_container():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'php-mysqlnd', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_php_client.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_php1_1')
 
 
 @pytest.fixture(scope='module')
 def nodejs_container():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'mysqljs', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_js_client.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_mysqljs1_1')
 
 
 @pytest.fixture(scope='module')
 def java_container():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'java', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_java_client.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_java1_1')
 
@@ -328,7 +329,7 @@ def test_python_client(server_address):
 
 def test_golang_client(server_address, golang_container):
     # type: (str, Container) -> None
-    with open(os.path.join(SCRIPT_DIR, 'clients', 'golang', '0.reference')) as fp:
+    with open(os.path.join(SCRIPT_DIR,'golang.reference')) as fp:
         reference = fp.read()
 
     code, (stdout, stderr) = golang_container.exec_run('./main --host {host} --port {port} --user default --password 123 --database '
@@ -385,7 +386,7 @@ def test_mysqljs_client(server_address, nodejs_container):
 
 def test_java_client(server_address, java_container):
     # type: (str, Container) -> None
-    with open(os.path.join(SCRIPT_DIR, 'clients', 'java', '0.reference')) as fp:
+    with open(os.path.join(SCRIPT_DIR, 'java.reference')) as fp:
         reference = fp.read()
 
     # database not exists exception.

tests/integration/test_postgresql_protocol/test.py

@@ -14,12 +14,13 @@ import subprocess
 import time
 import uuid
 
-from helpers.cluster import ClickHouseCluster
+from helpers.cluster import ClickHouseCluster, get_docker_compose_path
 
 psycopg2.extras.register_uuid()
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 config_dir = os.path.join(SCRIPT_DIR, './configs')
+DOCKER_COMPOSE_PATH = get_docker_compose_path()
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance('node', config_dir=config_dir, env_variables={'UBSAN_OPTIONS': 'print_stacktrace=1'})
@@ -38,7 +39,7 @@ def server_address():
 
 @pytest.fixture(scope='module')
 def psql_client():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'psql', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_postgesql.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_psql_1')
 
@@ -61,7 +62,7 @@ def psql_server(psql_client):
 
 @pytest.fixture(scope='module')
 def java_container():
-    docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'java', 'docker_compose.yml')
+    docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_postgesql_java_client.yml')
     subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build'])
     yield docker.from_env().containers.get(cluster.project_name + '_java_1')
 
@@ -132,7 +133,7 @@ def test_python_client(server_address):
 
 
 def test_java_client(server_address, java_container):
-    with open(os.path.join(SCRIPT_DIR, 'clients', 'java', '0.reference')) as fp:
+    with open(os.path.join(SCRIPT_DIR, 'java.reference')) as fp:
         reference = fp.read()
 
     # database not exists exception.