From 190ac2d048a2d67904a217a1d2bf77ab031238c1 Mon Sep 17 00:00:00 2001 From: Ning Yu Date: Fri, 10 Jul 2020 15:52:39 +0800 Subject: [PATCH] ic-proxy: add ic-proxy ICW jobs --- concourse/pipelines/gpdb_master-generated.yml | 117 +++++++++++++++++- concourse/pipelines/templates/gpdb-tpl.yml | 96 ++++++++++++++ concourse/scripts/common.bash | 25 ++++ 3 files changed, 237 insertions(+), 1 deletion(-) diff --git a/concourse/pipelines/gpdb_master-generated.yml b/concourse/pipelines/gpdb_master-generated.yml index 62a8957313..c925aad7f6 100644 --- a/concourse/pipelines/gpdb_master-generated.yml +++ b/concourse/pipelines/gpdb_master-generated.yml @@ -12,7 +12,7 @@ ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -t prod). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2020-06-21 20:24:25.703393 +## Generated by gen_pipeline.py at: 2020-07-23 14:11:08.112228 ## Template file: gpdb-tpl.yml ## OS Types: ['centos6', 'centos7', 'ubuntu18.04', 'win'] ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions'] @@ -178,11 +178,15 @@ groups: ## -------------------------------------------------------------------- - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - gpexpand - pg_upgrade - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 - gate_icw_end ## -------------------------------------------------------------------- - interconnect @@ -251,11 +255,15 @@ groups: jobs: - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - compile_gpdb_centos7 - pg_upgrade - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 - compile_gpdb_ubuntu18.04 - gate_icw_end @@ -418,12 +426,27 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) regexp: gp-internal-artifacts/centos7/libquicklz-devel-(1\.5\.0-.*)-1.el7.x86_64.rpm +- name: libuv-centos7 + type: gcs + source: + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/centos7/libuv-(1\.38\.0.*).tar.gz + - name: libquicklz-ubuntu18.04 type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) regexp: gp-internal-artifacts/ubuntu18.04/libquicklz-dev_(1\.5\.0-.*)-1_amd64.deb + +- name: libuv-ubuntu18.04 + type: gcs + source: + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/ubuntu18.04/libuv-(1\.38\.0.*).tar.gz + - name: gpdb7-centos7-build type: docker-image source: @@ -645,6 +668,8 @@ jobs: resource: libquicklz-centos7 - get: libquicklz-devel-installer resource: libquicklz-devel-centos7 + - get: libuv-installer + resource: libuv-centos7 - task: compile_gpdb image: gpdb7-centos7-build file: gpdb_src/concourse/tasks/compile_gpdb.yml @@ -674,6 +699,8 @@ jobs: - get: gpdb7-ubuntu18.04-build - get: libquicklz-installer resource: libquicklz-ubuntu18.04 + - get: libuv-installer + resource: libuv-ubuntu18.04 - task: compile_gpdb image: gpdb7-ubuntu18.04-build file: gpdb_src/concourse/tasks/compile_gpdb.yml @@ -799,6 +826,44 @@ jobs: params: file: sqldump/dump.sql.xz +- name: icw_gporca_icproxy_centos7 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_centos7] + - get: bin_gpdb + resource: bin_gpdb_centos7 + passed: [compile_gpdb_centos7] + trigger: true + - get: gpdb7-centos7-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-centos7-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: centos + CONFIGURE_FLAGS: {{configure_flags}} + +- name: icw_planner_icproxy_centos7 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_centos7] + - get: bin_gpdb + passed: [compile_gpdb_centos7] + resource: bin_gpdb_centos7 + trigger: true + - get: gpdb7-centos7-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-centos7-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: centos + CONFIGURE_FLAGS: {{configure_flags}} + - name: icw_gporca_ictcp_centos7 plan: - in_parallel: @@ -855,6 +920,44 @@ jobs: TEST_OS: ubuntu CONFIGURE_FLAGS: {{configure_flags}} +- name: icw_gporca_icproxy_ubuntu18.04 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_ubuntu18.04] + - get: bin_gpdb + resource: bin_gpdb_ubuntu18.04 + passed: [compile_gpdb_ubuntu18.04] + trigger: true + - get: gpdb7-ubuntu18.04-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-ubuntu18.04-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: ubuntu + CONFIGURE_FLAGS: {{configure_flags}} + +- name: icw_planner_icproxy_ubuntu18.04 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_ubuntu18.04] + - get: bin_gpdb + passed: [compile_gpdb_ubuntu18.04] + resource: bin_gpdb_ubuntu18.04 + trigger: true + - get: gpdb7-ubuntu18.04-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-ubuntu18.04-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: ubuntu + CONFIGURE_FLAGS: {{configure_flags}} + - name: gate_icw_end plan: - in_parallel: @@ -863,6 +966,8 @@ jobs: passed: - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - icw_extensions_gpcloud_centos7 - get: icw_planner_centos7_dump @@ -872,6 +977,8 @@ jobs: passed: - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - icw_extensions_gpcloud_centos7 trigger: true @@ -3104,11 +3211,15 @@ jobs: - compile_gpdb_ubuntu18.04 - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - icw_extensions_gpcloud_centos7 - resource_group_centos7 - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 - test_gpdb_clients_windows - cli_cross_subnet - gpmovemirrors @@ -3154,6 +3265,8 @@ jobs: - compile_gpdb_centos7 - icw_planner_centos7 - icw_gporca_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - icw_extensions_gpcloud_centos7 - resource_group_centos7 @@ -3187,6 +3300,8 @@ jobs: - compile_gpdb_ubuntu18.04 - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 - gpmovemirrors_ubuntu18 - gppkg_ubuntu18 - analyzedb_ubuntu18 diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index 0d0af15b23..7baa81d78f 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -203,6 +203,8 @@ groups: {% if "centos7" in os_types %} - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 {% endif %} {% if "centos7" in os_types and "CLI" in test_sections %} @@ -212,6 +214,8 @@ groups: {% if "ubuntu18.04" in os_types %} - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 {% endif %} - gate_icw_end {% endif %} @@ -291,6 +295,8 @@ groups: {% if "centos7" in os_types %} - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - compile_gpdb_centos7 {% endif %} @@ -300,6 +306,8 @@ groups: {% if "ubuntu18.04" in os_types %} - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 - compile_gpdb_ubuntu18.04 {% endif %} - gate_icw_end @@ -956,6 +964,44 @@ jobs: params: file: sqldump/dump.sql.xz +- name: icw_gporca_icproxy_centos7 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_centos7] + - get: bin_gpdb + resource: bin_gpdb_centos7 + passed: [compile_gpdb_centos7] + trigger: [[ test_trigger ]] + - get: gpdb7-centos7-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-centos7-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: centos + CONFIGURE_FLAGS: {{configure_flags}} + +- name: icw_planner_icproxy_centos7 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_centos7] + - get: bin_gpdb + passed: [compile_gpdb_centos7] + resource: bin_gpdb_centos7 + trigger: [[ test_trigger ]] + - get: gpdb7-centos7-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-centos7-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: centos + CONFIGURE_FLAGS: {{configure_flags}} + - name: icw_gporca_ictcp_centos7 plan: - in_parallel: @@ -1014,6 +1060,44 @@ jobs: TEST_OS: ubuntu CONFIGURE_FLAGS: {{configure_flags}} +- name: icw_gporca_icproxy_ubuntu18.04 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_ubuntu18.04] + - get: bin_gpdb + resource: bin_gpdb_ubuntu18.04 + passed: [compile_gpdb_ubuntu18.04] + trigger: [[ test_trigger ]] + - get: gpdb7-ubuntu18.04-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-ubuntu18.04-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: ubuntu + CONFIGURE_FLAGS: {{configure_flags}} + +- name: icw_planner_icproxy_ubuntu18.04 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [compile_gpdb_ubuntu18.04] + - get: bin_gpdb + passed: [compile_gpdb_ubuntu18.04] + resource: bin_gpdb_ubuntu18.04 + trigger: [[ test_trigger ]] + - get: gpdb7-ubuntu18.04-test + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb7-ubuntu18.04-test + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world + TEST_OS: ubuntu + CONFIGURE_FLAGS: {{configure_flags}} + {% endif %} - name: gate_icw_end plan: @@ -1024,6 +1108,8 @@ jobs: passed: - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 {% if "Extensions" in test_sections %} - icw_extensions_gpcloud_centos7 @@ -1037,6 +1123,8 @@ jobs: {% if "centos7" in os_types %} - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 {% if "Extensions" in test_sections %} - icw_extensions_gpcloud_centos7 @@ -1735,11 +1823,15 @@ jobs: - compile_gpdb_ubuntu18.04 - icw_gporca_centos7 - icw_planner_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - icw_extensions_gpcloud_centos7 - resource_group_centos7 - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 - test_gpdb_clients_windows - cli_cross_subnet {% for test in CLI_BEHAVE_TESTS %} @@ -1761,6 +1853,8 @@ jobs: - compile_gpdb_centos7 - icw_planner_centos7 - icw_gporca_centos7 + - icw_gporca_icproxy_centos7 + - icw_planner_icproxy_centos7 - icw_gporca_ictcp_centos7 - icw_extensions_gpcloud_centos7 - resource_group_centos7 @@ -1782,6 +1876,8 @@ jobs: - compile_gpdb_ubuntu18.04 - icw_gporca_ubuntu18.04 - icw_planner_ubuntu18.04 + - icw_gporca_icproxy_ubuntu18.04 + - icw_planner_icproxy_ubuntu18.04 {% for test in CLI_BEHAVE_TESTS %} - [[ test.name ]]_ubuntu18 {% endfor %} diff --git a/concourse/scripts/common.bash b/concourse/scripts/common.bash index 9913c7cdf0..be541bbd4a 100755 --- a/concourse/scripts/common.bash +++ b/concourse/scripts/common.bash @@ -48,6 +48,31 @@ function make_cluster() { export STATEMENT_MEM=250MB pushd gpdb_src/gpAux/gpdemo su gpadmin -c "source /usr/local/greenplum-db-devel/greenplum_path.sh; make create-demo-cluster" + + if [[ "$MAKE_TEST_COMMAND" =~ gp_interconnect_type=proxy ]]; then + # generate the addresses for proxy mode + su gpadmin -c bash -- -e <