Create Extensions group on pipeline

Create an extensions group and add gpcloud as part of it. The group will
no longer be added as part of ICW, now it need to be specifically added
as a test section when calling gen_pipeline.py
Signed-off-by: NDavid Kimura <dkimura@pivotal.io>

concourse/pipelines/gen_pipeline.py

@@ -222,7 +222,7 @@ if __name__ == "__main__":
     PARSER.add_argument('-a', '--test_sections',
                         action='store',
                         dest='test_sections',
-                        choices=['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA'],
+                        choices=['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions'],
                         default=['ICW'],
                         nargs='+',
                         help='Select tests sections to run')
@@ -242,7 +242,7 @@ if __name__ == "__main__":
 
     if ARGS.pipeline_type == 'prod':
         ARGS.os_types = ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
-        ARGS.test_sections = ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA']
+        ARGS.test_sections = ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions']
 
     # if generating a dev pipeline but didn't specify an output, don't overwrite the master pipeline
     if ARGS.pipeline_type != 'prod' and os.path.basename(ARGS.output_filepath) == default_output_filename:

concourse/pipelines/gpdb_master-generated.yml

 ## ######################################################################
-##            ___   ___    _  _  ___ _____   ___ ___ ___ _____
+##            ___   ___    _  _  ___ _____   ___ ___ ___ _____ 
 ##           |   \ / _ \  | \| |/ _ \_   _| | __|   \_ _|_   _|
-##           | |) | (_) | | .` | (_) || |   | _|| |) | |  | |
-##           |___/ \___/  |_|\_|\___/ |_|   |___|___/___| |_|
-##            _____ _  _ ___ ___   ___ ___ _    ___ _
+##           | |) | (_) | | .` | (_) || |   | _|| |) | |  | |  
+##           |___/ \___/  |_|\_|\___/ |_|   |___|___/___| |_|  
+##            _____ _  _ ___ ___   ___ ___ _    ___ _ 
 ##           |_   _| || |_ _/ __| | __|_ _| |  | __| |
 ##             | | | __ || |\__ \ | _| | || |__| _||_|
 ##             |_| |_||_|___|___/ |_| |___|____|___(_)
@@ -12,10 +12,10 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2018-09-01 16:53:17.110593
+## Generated by gen_pipeline.py at: 2018-09-24 15:57:22.861130
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
-## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA']
+## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions']
 ## ======================================================================
 
 ## ======================================================================
@@ -43,7 +43,6 @@ groups:
   - icw_gporca_centos6_gpos_memory
   - icw_planner_centos6
   - icw_planner_ictcp_centos6
-  - icw_extensions_gpcloud_centos6
   - icw_gporca_centos7
   - icw_planner_centos7
   - icw_gporca_sles11
@@ -51,7 +50,6 @@ groups:
   - icw_planner_sles12
   - icw_planner_ubuntu16
   - icw_gporca_conan_ubuntu16
-  - icw_extensions_gpcloud_ubuntu16
   - gpdb_packaging_ubuntu16
   - gate_icw_end
   ## --------------------------------------------------------------------
@@ -94,6 +92,9 @@ groups:
   - MADlib_Test_planner_centos6
   - MADlib_Test_orca_centos7
   - MADlib_Test_planner_centos7
+ ## ---------------------------------------------------------------------
+  - icw_extensions_gpcloud_centos6
+  - icw_extensions_gpcloud_ubuntu16
 
 
 ## ======================================================================
@@ -132,7 +133,6 @@ groups:
   - icw_gporca_centos6_gpos_memory
   - icw_planner_centos6
   - icw_planner_ictcp_centos6
-  - icw_extensions_gpcloud_centos6
   - compile_gpdb_centos6
   - icw_gporca_centos7
   - icw_planner_centos7
@@ -145,10 +145,17 @@ groups:
   - icw_gporca_conan_ubuntu16
   - compile_gpdb_ubuntu16
   - gpdb_packaging_ubuntu16
-  - icw_extensions_gpcloud_ubuntu16
   - compile_gpdb_ubuntu16
   - gate_icw_end
 
+
+## ======================================================================
+
+- name: Extensions
+  jobs:
+  - icw_extensions_gpcloud_centos6
+  - icw_extensions_gpcloud_ubuntu16
+
 ## ======================================================================
 
 - name: Replication
@@ -1285,60 +1292,6 @@ jobs:
     params:
       DEBIAN_PACKAGE: {{deb_package_ubuntu16_versioned_file}}
 
-- name: icw_extensions_gpcloud_centos6
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [compile_gpdb_centos6]
-      trigger: true
-    - get: bin_gpdb_centos6
-      passed: [compile_gpdb_centos6]
-    - get: centos-gpdb-dev-6
-  - aggregate:
-    - task: unit_tests_gpcloud
-      file: gpdb_src/concourse/tasks/unit_tests_gpcloud.yml
-      image: centos-gpdb-dev-6
-      params:
-        TARGET_OS: centos
-    - task: regression_tests_gpcloud_centos
-      input_mapping:
-        bin_gpdb: bin_gpdb_centos6
-      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
-      image: centos-gpdb-dev-6
-      params:
-        gpcloud_access_key_id: {{gpcloud-access-key-id}}
-        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
-        TARGET_OS: centos
-    - task: gpcheckcloud_tests_gpcloud_centos
-      input_mapping:
-        bin_gpdb: bin_gpdb_centos6
-      file: gpdb_src/concourse/tasks/gpcheckcloud_tests_gpcloud.yml
-      image: centos-gpdb-dev-6
-      params:
-        gpcloud_access_key_id: {{gpcloud-access-key-id}}
-        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
-        TARGET_OS: centos
-
-- name: icw_extensions_gpcloud_ubuntu16
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [compile_gpdb_ubuntu16]
-      trigger: true
-    - get: compiled_bits_ubuntu16
-      passed: [compile_gpdb_ubuntu16]
-    - get: ubuntu-gpdb-dev-16
-  - aggregate:
-    - task: regression_tests_gpcloud_ubuntu
-      input_mapping:
-        bin_gpdb: compiled_bits_ubuntu16
-      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
-      image: ubuntu-gpdb-dev-16
-      params:
-        gpcloud_access_key_id: {{gpcloud-access-key-id}}
-        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
-        TARGET_OS: ubuntu
-
 - name: gate_icw_end
   plan:
   - aggregate:
@@ -2522,3 +2475,65 @@ jobs:
   - put: bin_gpdb_sles11_rc
     params:
       file: bin_gpdb_sles11/bin_gpdb.tar.gz
+
+## ======================================================================
+##    __      _                 _                 
+##   /__\_  _| |_ ___ _ __  ___(_) ___  _ __  ___ 
+##  /_\ \ \/ / __/ _ \ '_ \/ __| |/ _ \| '_ \/ __|
+## //__  >  <| ||  __/ | | \__ \ | (_) | | | \__ \
+## \__/ /_/\_\\__\___|_| |_|___/_|\___/|_| |_|___/
+## ======================================================================
+
+- name: icw_extensions_gpcloud_centos6
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_centos6]
+      trigger: true
+    - get: bin_gpdb_centos6
+      passed: [compile_gpdb_centos6]
+    - get: centos-gpdb-dev-6
+  - aggregate:
+    - task: unit_tests_gpcloud
+      file: gpdb_src/concourse/tasks/unit_tests_gpcloud.yml
+      image: centos-gpdb-dev-6
+      params:
+        TARGET_OS: centos
+    - task: regression_tests_gpcloud_centos
+      input_mapping:
+        bin_gpdb: bin_gpdb_centos6
+      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
+      image: centos-gpdb-dev-6
+      params:
+        gpcloud_access_key_id: {{gpcloud-access-key-id}}
+        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
+        TARGET_OS: centos
+    - task: gpcheckcloud_tests_gpcloud_centos
+      input_mapping:
+        bin_gpdb: bin_gpdb_centos6
+      file: gpdb_src/concourse/tasks/gpcheckcloud_tests_gpcloud.yml
+      image: centos-gpdb-dev-6
+      params:
+        gpcloud_access_key_id: {{gpcloud-access-key-id}}
+        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
+        TARGET_OS: centos
+
+- name: icw_extensions_gpcloud_ubuntu16
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_ubuntu16]
+      trigger: true
+    - get: compiled_bits_ubuntu16
+      passed: [compile_gpdb_ubuntu16]
+    - get: ubuntu-gpdb-dev-16
+  - aggregate:
+    - task: regression_tests_gpcloud_ubuntu
+      input_mapping:
+        bin_gpdb: compiled_bits_ubuntu16
+      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
+      image: ubuntu-gpdb-dev-16
+      params:
+        gpcloud_access_key_id: {{gpcloud-access-key-id}}
+        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
+        TARGET_OS: ubuntu

concourse/pipelines/templates/gpdb-tpl.yml

@@ -50,7 +50,6 @@ groups:
   - icw_gporca_centos6_gpos_memory
   - icw_planner_centos6
   - icw_planner_ictcp_centos6
-  - icw_extensions_gpcloud_centos6
 {% endif %}
 {% if "centos7" in os_types %}
   - icw_gporca_centos7
@@ -64,7 +63,6 @@ groups:
 {% if "ubuntu16" in os_types %}
   - icw_planner_ubuntu16
   - icw_gporca_conan_ubuntu16
-  - icw_extensions_gpcloud_ubuntu16
 {% endif %}
 {% if "ubuntu16" in os_types %}
   - gpdb_packaging_ubuntu16
@@ -129,6 +127,15 @@ groups:
   - MADlib_Test_planner_centos7
 {% endif %}
 {% endif %}
+ ## ---------------------------------------------------------------------
+{% if "Extensions" in test_sections %}
+{% if "centos6" in os_types %}
+  - icw_extensions_gpcloud_centos6
+{% endif %}
+{% if "ubuntu16" in os_types %}
+  - icw_extensions_gpcloud_ubuntu16
+{% endif %}
+{%endif %}
 
 
 {% if pipeline_type == "prod" %}
@@ -187,7 +194,6 @@ groups:
   - icw_gporca_centos6_gpos_memory
   - icw_planner_centos6
   - icw_planner_ictcp_centos6
-  - icw_extensions_gpcloud_centos6
   - compile_gpdb_centos6
 {% endif %}
 {% if "centos7" in os_types %}
@@ -206,11 +212,24 @@ groups:
   - icw_gporca_conan_ubuntu16
   - compile_gpdb_ubuntu16
   - gpdb_packaging_ubuntu16
-  - icw_extensions_gpcloud_ubuntu16
   - compile_gpdb_ubuntu16
 {% endif %}
   - gate_icw_end
 
+{% endif %}
+
+## ======================================================================
+
+{% if "Extensions" in test_sections %}
+- name: Extensions
+  jobs:
+{% if "centos6" in os_types %}
+  - icw_extensions_gpcloud_centos6
+{% endif %}
+{% if "ubuntu16" in os_types %}
+  - icw_extensions_gpcloud_ubuntu16
+{% endif %}
+
 {% endif %}
 {% if "Replication" in test_sections %}
 ## ======================================================================
@@ -1443,64 +1462,6 @@ jobs:
       DEBIAN_PACKAGE: {{deb_package_ubuntu16_versioned_file}}
 
 {% endif %}
-{% if 'centos6' in os_types%}
-- name: icw_extensions_gpcloud_centos6
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [compile_gpdb_centos6]
-      trigger: [[ test_trigger ]]
-    - get: bin_gpdb_centos6
-      passed: [compile_gpdb_centos6]
-    - get: centos-gpdb-dev-6
-  - aggregate:
-    - task: unit_tests_gpcloud
-      file: gpdb_src/concourse/tasks/unit_tests_gpcloud.yml
-      image: centos-gpdb-dev-6
-      params:
-        TARGET_OS: centos
-    - task: regression_tests_gpcloud_centos
-      input_mapping:
-        bin_gpdb: bin_gpdb_centos6
-      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
-      image: centos-gpdb-dev-6
-      params:
-        gpcloud_access_key_id: {{gpcloud-access-key-id}}
-        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
-        TARGET_OS: centos
-    - task: gpcheckcloud_tests_gpcloud_centos
-      input_mapping:
-        bin_gpdb: bin_gpdb_centos6
-      file: gpdb_src/concourse/tasks/gpcheckcloud_tests_gpcloud.yml
-      image: centos-gpdb-dev-6
-      params:
-        gpcloud_access_key_id: {{gpcloud-access-key-id}}
-        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
-        TARGET_OS: centos
-{% endif %}
-
-{% if 'ubuntu16' in os_types %}
-- name: icw_extensions_gpcloud_ubuntu16
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [compile_gpdb_ubuntu16]
-      trigger: [[ test_trigger ]]
-    - get: compiled_bits_ubuntu16
-      passed: [compile_gpdb_ubuntu16]
-    - get: ubuntu-gpdb-dev-16
-  - aggregate:
-    - task: regression_tests_gpcloud_ubuntu
-      input_mapping:
-        bin_gpdb: compiled_bits_ubuntu16
-      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
-      image: ubuntu-gpdb-dev-16
-      params:
-        gpcloud_access_key_id: {{gpcloud-access-key-id}}
-        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
-        TARGET_OS: ubuntu
-{% endif %}
-
 - name: gate_icw_end
   plan:
   - aggregate:
@@ -1511,7 +1472,9 @@ jobs:
       - icw_gporca_centos6_gpos_memory
       - icw_planner_centos6
       - icw_planner_ictcp_centos6
+{% if "Extensions" in test_sections %}
       - icw_extensions_gpcloud_centos6
+{% endif %}
 {% endif %}
     - get: gpdb_src
       passed:
@@ -1520,11 +1483,15 @@ jobs:
       - icw_gporca_centos6_gpos_memory
       - icw_planner_centos6
       - icw_planner_ictcp_centos6
+{% if "Extensions" in test_sections %}
       - icw_extensions_gpcloud_centos6
 {% endif %}
+{% endif %}
 {% if 'ubuntu16' in os_types %}
+{% if "Extensions" in test_sections %}
       - icw_extensions_gpcloud_ubuntu16
 {% endif %}
+{% endif %}
 {% if "centos7" in os_types %}
       - icw_gporca_centos7
 {% endif %}
@@ -2581,3 +2548,71 @@ jobs:
     params:
       file: bin_gpdb_sles11/bin_gpdb.tar.gz
 {% endif %}
+
+## ======================================================================
+##    __      _                 _                 
+##   /__\_  _| |_ ___ _ __  ___(_) ___  _ __  ___ 
+##  /_\ \ \/ / __/ _ \ '_ \/ __| |/ _ \| '_ \/ __|
+## //__  >  <| ||  __/ | | \__ \ | (_) | | | \__ \
+## \__/ /_/\_\\__\___|_| |_|___/_|\___/|_| |_|___/
+## ======================================================================
+
+{% if "Extensions" in test_sections %}
+{% if 'centos6' in os_types%}
+- name: icw_extensions_gpcloud_centos6
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_centos6]
+      trigger: [[ test_trigger ]]
+    - get: bin_gpdb_centos6
+      passed: [compile_gpdb_centos6]
+    - get: centos-gpdb-dev-6
+  - aggregate:
+    - task: unit_tests_gpcloud
+      file: gpdb_src/concourse/tasks/unit_tests_gpcloud.yml
+      image: centos-gpdb-dev-6
+      params:
+        TARGET_OS: centos
+    - task: regression_tests_gpcloud_centos
+      input_mapping:
+        bin_gpdb: bin_gpdb_centos6
+      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
+      image: centos-gpdb-dev-6
+      params:
+        gpcloud_access_key_id: {{gpcloud-access-key-id}}
+        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
+        TARGET_OS: centos
+    - task: gpcheckcloud_tests_gpcloud_centos
+      input_mapping:
+        bin_gpdb: bin_gpdb_centos6
+      file: gpdb_src/concourse/tasks/gpcheckcloud_tests_gpcloud.yml
+      image: centos-gpdb-dev-6
+      params:
+        gpcloud_access_key_id: {{gpcloud-access-key-id}}
+        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
+        TARGET_OS: centos
+{% endif %}
+
+{% if 'ubuntu16' in os_types %}
+- name: icw_extensions_gpcloud_ubuntu16
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_ubuntu16]
+      trigger: [[ test_trigger ]]
+    - get: compiled_bits_ubuntu16
+      passed: [compile_gpdb_ubuntu16]
+    - get: ubuntu-gpdb-dev-16
+  - aggregate:
+    - task: regression_tests_gpcloud_ubuntu
+      input_mapping:
+        bin_gpdb: compiled_bits_ubuntu16
+      file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
+      image: ubuntu-gpdb-dev-16
+      params:
+        gpcloud_access_key_id: {{gpcloud-access-key-id}}
+        gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
+        TARGET_OS: ubuntu
+{% endif %}
+{% endif %}