ic-proxy: add ic-proxy ICW jobs

190ac2d0 · Ning Yu · 87e1d9ee · 190ac2d0 · 190ac2d0 · 190ac2d0
3 changed file
--- a/concourse/pipelines/gpdb_master-generated.yml
+++ b/concourse/pipelines/gpdb_master-generated.yml
@@ -12,7 +12,7 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2020-06-21 20:24:25.703393
+## Generated by gen_pipeline.py at: 2020-07-23 14:11:08.112228
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'ubuntu18.04', 'win']
 ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions']
@@ -178,11 +178,15 @@ groups:
  ## --------------------------------------------------------------------
  - icw_gporca_centos7
  - icw_planner_centos7
+  - icw_gporca_icproxy_centos7
+  - icw_planner_icproxy_centos7
  - icw_gporca_ictcp_centos7
  - gpexpand
  - pg_upgrade
  - icw_gporca_ubuntu18.04
  - icw_planner_ubuntu18.04
+  - icw_gporca_icproxy_ubuntu18.04
+  - icw_planner_icproxy_ubuntu18.04
  - gate_icw_end
  ## --------------------------------------------------------------------
  - interconnect
@@ -251,11 +255,15 @@ groups:
  jobs:
  - icw_gporca_centos7
  - icw_planner_centos7
+  - icw_gporca_icproxy_centos7
+  - icw_planner_icproxy_centos7
  - icw_gporca_ictcp_centos7
  - compile_gpdb_centos7
  - pg_upgrade
  - icw_gporca_ubuntu18.04
  - icw_planner_ubuntu18.04
+  - icw_gporca_icproxy_ubuntu18.04
+  - icw_planner_icproxy_ubuntu18.04
  - compile_gpdb_ubuntu18.04
  - gate_icw_end

@@ -418,12 +426,27 @@ resources:
    json_key: ((concourse-gcs-resources-service-account-key))
    regexp: gp-internal-artifacts/centos7/libquicklz-devel-(1\.5\.0-.*)-1.el7.x86_64.rpm

+- name: libuv-centos7
+  type: gcs
+  source:
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: gp-internal-artifacts/centos7/libuv-(1\.38\.0.*).tar.gz
+
 - name: libquicklz-ubuntu18.04
  type: gcs
  source:
    bucket: ((gcs-bucket))
    json_key: ((concourse-gcs-resources-service-account-key))
    regexp: gp-internal-artifacts/ubuntu18.04/libquicklz-dev_(1\.5\.0-.*)-1_amd64.deb
+
+- name: libuv-ubuntu18.04
+  type: gcs
+  source:
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: gp-internal-artifacts/ubuntu18.04/libuv-(1\.38\.0.*).tar.gz
+
 - name: gpdb7-centos7-build
  type: docker-image
  source:
@@ -645,6 +668,8 @@ jobs:
        resource: libquicklz-centos7
      - get: libquicklz-devel-installer
        resource: libquicklz-devel-centos7
+      - get: libuv-installer
+        resource: libuv-centos7
  - task: compile_gpdb
    image: gpdb7-centos7-build
    file: gpdb_src/concourse/tasks/compile_gpdb.yml
@@ -674,6 +699,8 @@ jobs:
        - get: gpdb7-ubuntu18.04-build
        - get: libquicklz-installer
          resource: libquicklz-ubuntu18.04
+        - get: libuv-installer
+          resource: libuv-ubuntu18.04
    - task: compile_gpdb
      image: gpdb7-ubuntu18.04-build
      file: gpdb_src/concourse/tasks/compile_gpdb.yml
@@ -799,6 +826,44 @@ jobs:
    params:
      file: sqldump/dump.sql.xz

+- name: icw_gporca_icproxy_centos7
+  plan:
+  - in_parallel:
+      steps:
+      - get: gpdb_src
+        passed: [compile_gpdb_centos7]
+      - get: bin_gpdb
+        resource: bin_gpdb_centos7
+        passed: [compile_gpdb_centos7]
+        trigger: true
+      - get: gpdb7-centos7-test
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb.yml
+    image: gpdb7-centos7-test
+    params:
+      MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+      TEST_OS: centos
+      CONFIGURE_FLAGS: {{configure_flags}}
+
+- name: icw_planner_icproxy_centos7
+  plan:
+  - in_parallel:
+      steps:
+      - get: gpdb_src
+        passed: [compile_gpdb_centos7]
+      - get: bin_gpdb
+        passed: [compile_gpdb_centos7]
+        resource: bin_gpdb_centos7
+        trigger: true
+      - get: gpdb7-centos7-test
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb.yml
+    image: gpdb7-centos7-test
+    params:
+      MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+      TEST_OS: centos
+      CONFIGURE_FLAGS: {{configure_flags}}
+
 - name: icw_gporca_ictcp_centos7
  plan:
  - in_parallel:
@@ -855,6 +920,44 @@ jobs:
        TEST_OS: ubuntu
        CONFIGURE_FLAGS: {{configure_flags}}

+- name: icw_gporca_icproxy_ubuntu18.04
+  plan:
+    - in_parallel:
+        steps:
+        - get: gpdb_src
+          passed: [compile_gpdb_ubuntu18.04]
+        - get: bin_gpdb
+          resource: bin_gpdb_ubuntu18.04
+          passed: [compile_gpdb_ubuntu18.04]
+          trigger: true
+        - get: gpdb7-ubuntu18.04-test
+    - task: ic_gpdb
+      file: gpdb_src/concourse/tasks/ic_gpdb.yml
+      image: gpdb7-ubuntu18.04-test
+      params:
+        MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+        TEST_OS: ubuntu
+        CONFIGURE_FLAGS: {{configure_flags}}
+
+- name: icw_planner_icproxy_ubuntu18.04
+  plan:
+    - in_parallel:
+        steps:
+        - get: gpdb_src
+          passed: [compile_gpdb_ubuntu18.04]
+        - get: bin_gpdb
+          passed: [compile_gpdb_ubuntu18.04]
+          resource: bin_gpdb_ubuntu18.04
+          trigger: true
+        - get: gpdb7-ubuntu18.04-test
+    - task: ic_gpdb
+      file: gpdb_src/concourse/tasks/ic_gpdb.yml
+      image: gpdb7-ubuntu18.04-test
+      params:
+        MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+        TEST_OS: ubuntu
+        CONFIGURE_FLAGS: {{configure_flags}}
+
 - name: gate_icw_end
  plan:
  - in_parallel:
@@ -863,6 +966,8 @@ jobs:
        passed:
        - icw_gporca_centos7
        - icw_planner_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
        - icw_extensions_gpcloud_centos7
      - get: icw_planner_centos7_dump
@@ -872,6 +977,8 @@ jobs:
        passed:
        - icw_gporca_centos7
        - icw_planner_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
        - icw_extensions_gpcloud_centos7
        trigger: true
@@ -3104,11 +3211,15 @@ jobs:
        - compile_gpdb_ubuntu18.04
        - icw_gporca_centos7
        - icw_planner_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
        - icw_extensions_gpcloud_centos7
        - resource_group_centos7
        - icw_gporca_ubuntu18.04
        - icw_planner_ubuntu18.04
+        - icw_gporca_icproxy_ubuntu18.04
+        - icw_planner_icproxy_ubuntu18.04
        - test_gpdb_clients_windows
        - cli_cross_subnet
        - gpmovemirrors
@@ -3154,6 +3265,8 @@ jobs:
        - compile_gpdb_centos7
        - icw_planner_centos7
        - icw_gporca_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
        - icw_extensions_gpcloud_centos7
        - resource_group_centos7
@@ -3187,6 +3300,8 @@ jobs:
        - compile_gpdb_ubuntu18.04
        - icw_gporca_ubuntu18.04
        - icw_planner_ubuntu18.04
+        - icw_gporca_icproxy_ubuntu18.04
+        - icw_planner_icproxy_ubuntu18.04
        - gpmovemirrors_ubuntu18
        - gppkg_ubuntu18
        - analyzedb_ubuntu18

--- a/concourse/pipelines/templates/gpdb-tpl.yml
+++ b/concourse/pipelines/templates/gpdb-tpl.yml
@@ -203,6 +203,8 @@ groups:
 {% if "centos7" in os_types %}
  - icw_gporca_centos7
  - icw_planner_centos7
+  - icw_gporca_icproxy_centos7
+  - icw_planner_icproxy_centos7
  - icw_gporca_ictcp_centos7
 {% endif %}
 {% if "centos7" in os_types and "CLI" in test_sections %}
@@ -212,6 +214,8 @@ groups:
 {% if "ubuntu18.04" in os_types %}
  - icw_gporca_ubuntu18.04
  - icw_planner_ubuntu18.04
+  - icw_gporca_icproxy_ubuntu18.04
+  - icw_planner_icproxy_ubuntu18.04
 {% endif %}
  - gate_icw_end
 {% endif %}
@@ -291,6 +295,8 @@ groups:
 {% if "centos7" in os_types %}
  - icw_gporca_centos7
  - icw_planner_centos7
+  - icw_gporca_icproxy_centos7
+  - icw_planner_icproxy_centos7
  - icw_gporca_ictcp_centos7
  - compile_gpdb_centos7
 {% endif %}
@@ -300,6 +306,8 @@ groups:
 {% if "ubuntu18.04" in os_types %}
  - icw_gporca_ubuntu18.04
  - icw_planner_ubuntu18.04
+  - icw_gporca_icproxy_ubuntu18.04
+  - icw_planner_icproxy_ubuntu18.04
  - compile_gpdb_ubuntu18.04
 {% endif %}
  - gate_icw_end
@@ -956,6 +964,44 @@ jobs:
    params:
      file: sqldump/dump.sql.xz

+- name: icw_gporca_icproxy_centos7
+  plan:
+  - in_parallel:
+      steps:
+      - get: gpdb_src
+        passed: [compile_gpdb_centos7]
+      - get: bin_gpdb
+        resource: bin_gpdb_centos7
+        passed: [compile_gpdb_centos7]
+        trigger: [[ test_trigger ]]
+      - get: gpdb7-centos7-test
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb.yml
+    image: gpdb7-centos7-test
+    params:
+      MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+      TEST_OS: centos
+      CONFIGURE_FLAGS: {{configure_flags}}
+
+- name: icw_planner_icproxy_centos7
+  plan:
+  - in_parallel:
+      steps:
+      - get: gpdb_src
+        passed: [compile_gpdb_centos7]
+      - get: bin_gpdb
+        passed: [compile_gpdb_centos7]
+        resource: bin_gpdb_centos7
+        trigger: [[ test_trigger ]]
+      - get: gpdb7-centos7-test
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb.yml
+    image: gpdb7-centos7-test
+    params:
+      MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+      TEST_OS: centos
+      CONFIGURE_FLAGS: {{configure_flags}}
+
 - name: icw_gporca_ictcp_centos7
  plan:
  - in_parallel:
@@ -1014,6 +1060,44 @@ jobs:
        TEST_OS: ubuntu
        CONFIGURE_FLAGS: {{configure_flags}}

+- name: icw_gporca_icproxy_ubuntu18.04
+  plan:
+    - in_parallel:
+        steps:
+        - get: gpdb_src
+          passed: [compile_gpdb_ubuntu18.04]
+        - get: bin_gpdb
+          resource: bin_gpdb_ubuntu18.04
+          passed: [compile_gpdb_ubuntu18.04]
+          trigger: [[ test_trigger ]]
+        - get: gpdb7-ubuntu18.04-test
+    - task: ic_gpdb
+      file: gpdb_src/concourse/tasks/ic_gpdb.yml
+      image: gpdb7-ubuntu18.04-test
+      params:
+        MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=on' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+        TEST_OS: ubuntu
+        CONFIGURE_FLAGS: {{configure_flags}}
+
+- name: icw_planner_icproxy_ubuntu18.04
+  plan:
+    - in_parallel:
+        steps:
+        - get: gpdb_src
+          passed: [compile_gpdb_ubuntu18.04]
+        - get: bin_gpdb
+          passed: [compile_gpdb_ubuntu18.04]
+          resource: bin_gpdb_ubuntu18.04
+          trigger: [[ test_trigger ]]
+        - get: gpdb7-ubuntu18.04-test
+    - task: ic_gpdb
+      file: gpdb_src/concourse/tasks/ic_gpdb.yml
+      image: gpdb7-ubuntu18.04-test
+      params:
+        MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=proxy -c optimizer=off' EXTRA_REGRESS_OPTS=--exclude-tests=external_table,copydml installcheck-world
+        TEST_OS: ubuntu
+        CONFIGURE_FLAGS: {{configure_flags}}
+
 {% endif %}
 - name: gate_icw_end
  plan:
@@ -1024,6 +1108,8 @@ jobs:
        passed:
        - icw_gporca_centos7
        - icw_planner_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
 {% if "Extensions" in test_sections %}
        - icw_extensions_gpcloud_centos7
@@ -1037,6 +1123,8 @@ jobs:
 {% if "centos7" in os_types %}
        - icw_gporca_centos7
        - icw_planner_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
 {% if "Extensions" in test_sections %}
        - icw_extensions_gpcloud_centos7
@@ -1735,11 +1823,15 @@ jobs:
        - compile_gpdb_ubuntu18.04
        - icw_gporca_centos7
        - icw_planner_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
        - icw_extensions_gpcloud_centos7
        - resource_group_centos7
        - icw_gporca_ubuntu18.04
        - icw_planner_ubuntu18.04
+        - icw_gporca_icproxy_ubuntu18.04
+        - icw_planner_icproxy_ubuntu18.04
        - test_gpdb_clients_windows
        - cli_cross_subnet
 {% for test in CLI_BEHAVE_TESTS %}
@@ -1761,6 +1853,8 @@ jobs:
        - compile_gpdb_centos7
        - icw_planner_centos7
        - icw_gporca_centos7
+        - icw_gporca_icproxy_centos7
+        - icw_planner_icproxy_centos7
        - icw_gporca_ictcp_centos7
        - icw_extensions_gpcloud_centos7
        - resource_group_centos7
@@ -1782,6 +1876,8 @@ jobs:
        - compile_gpdb_ubuntu18.04
        - icw_gporca_ubuntu18.04
        - icw_planner_ubuntu18.04
+        - icw_gporca_icproxy_ubuntu18.04
+        - icw_planner_icproxy_ubuntu18.04
 {% for test in CLI_BEHAVE_TESTS %}
        - [[ test.name ]]_ubuntu18
 {% endfor %}

--- a/concourse/scripts/common.bash
+++ b/concourse/scripts/common.bash
@@ -48,6 +48,31 @@ function make_cluster() {
  export STATEMENT_MEM=250MB
  pushd gpdb_src/gpAux/gpdemo
  su gpadmin -c "source /usr/local/greenplum-db-devel/greenplum_path.sh; make create-demo-cluster"
+
+  if [[ "$MAKE_TEST_COMMAND" =~ gp_interconnect_type=proxy ]]; then
+    # generate the addresses for proxy mode
+    su gpadmin -c bash -- -e <<EOF
+      source /usr/local/greenplum-db-devel/greenplum_path.sh
+      source $PWD/gpdemo-env.sh
+
+      delta=-3000
+
+      psql -tqA -d postgres -P pager=off -F ' ' \
+          -c "select dbid, content, port+\$delta as port, address from gp_segment_configuration order by 1" \
+      | while read -r dbid content port addr; do
+          ip=127.0.0.1
+          echo "\$dbid:\$content:\$ip:\$port"
+        done \
+      | paste -sd, - \
+      | xargs -rI'{}' gpconfig --skipvalidation -c gp_interconnect_proxy_addresses -v "'{}'"
+
+      # also have to enlarge gp_interconnect_tcp_listener_backlog
+      gpconfig -c gp_interconnect_tcp_listener_backlog -v 1024
+
+      gpstop -raqi
+EOF
+  fi
+
  popd
 }