Add advanced analytics tests to gpdb master pipeline (#5187)

concourse/pipelines/gen_pipeline.py

@@ -53,6 +53,7 @@ JOBS_THAT_ARE_GATES = ['gate_icw_start',
                        'gate_resource_groups_start',
                        'gate_cli_start',
                        'gate_ud_start',
+                       'gate_advanced_analytics_start',
                        'gate_release_candidate_start']
 
 JOBS_THAT_SHOULD_NOT_BLOCK_RELEASE = ['compile_gpdb_binary_swap_centos6', 'icw_gporca_centos6_gpos_memory'] + RELEASE_VALIDATOR_JOB + JOBS_THAT_ARE_GATES
@@ -221,7 +222,7 @@ if __name__ == "__main__":
     PARSER.add_argument('-a', '--test_sections',
                         action='store',
                         dest='test_sections',
-                        choices=['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD'],
+                        choices=['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA'],
                         default=['ICW'],
                         nargs='+',
                         help='Select tests sections to run')
@@ -241,7 +242,7 @@ if __name__ == "__main__":
 
     if ARGS.pipeline_type == 'prod':
         ARGS.os_types = ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
-        ARGS.test_sections = ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD']
+        ARGS.test_sections = ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA']
 
     # if generating a dev pipeline but didn't specify an output, don't overwrite the master pipeline
     if ARGS.pipeline_type != 'prod' and os.path.basename(ARGS.output_filepath) == default_output_filename:

concourse/pipelines/gpdb_master-generated.yml

@@ -12,10 +12,10 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2018-06-27 00:48:09.907065
+## Generated by gen_pipeline.py at: 2018-06-29 11:58:39.648381
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
-## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD']
+## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA']
 ## ======================================================================
 
 ## ======================================================================
@@ -87,6 +87,13 @@ groups:
   - regression_tests_pxf
   - regression_tests_gphdfs_hadoop_centos
   - regression_tests_gphdfs_mapr_centos
+  ## --------------------------------------------------------------------
+  - gate_advanced_analytics_start
+  - MADlib_Test_orca_centos6
+  - MADlib_Test_planner_centos6
+  - MADlib_Test_orca_centos7
+  - MADlib_Test_planner_centos7
+
 
 ## ======================================================================
 
@@ -202,6 +209,15 @@ groups:
   - regression_tests_gphdfs_mapr_centos
   - compile_gpdb_centos6
 
+
+- name: AdvancedAnalytics
+  jobs:
+  - gate_advanced_analytics_start
+  - MADlib_Test_orca_centos6
+  - MADlib_Test_planner_centos6
+  - MADlib_Test_orca_centos7
+  - MADlib_Test_planner_centos7
+
 ## ======================================================================
 ##                                           _
 ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  | |_ _   _ _ __   ___  ___
@@ -497,6 +513,23 @@ resources:
     start: 6:00 AM
     stop: 7:00 AM
 
+- name: madlib_ci
+  type: git
+  source:
+    branch: {{madlib-ci-git-branch}}
+    uri: {{madlib-ci-git-remote}}
+    private_key: {{madlib-ci-git-key}}
+
+- name: madlib_gppkg
+  type: s3
+  source:
+    access_key_id: {{bucket-access-key-id}}
+    bucket: {{madlib-gppkg-bucket}}
+    region_name: {{aws-region}}
+    secret_access_key: {{bucket-secret-access-key}}
+    regexp: {{madlib-gpdb6-centos6-gppkg}}
+
+
 - name: reduced-frequency-trigger
   type: time
   source:
@@ -504,7 +537,6 @@ resources:
     days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday, Saturday]
     start: {{reduced-frequency-trigger-start}}
     stop: {{reduced-frequency-trigger-stop}}
-
 ## ======================================================================
 ##                   _
 ##   __ _ _ __   ___| |__   ___  _ __ ___
@@ -2167,6 +2199,109 @@ jobs:
       TARGET_OS: centos
       TARGET_OS_VERSION: 6
 
+
+## ======================================================================
+##     _        _
+##    / \      / \
+##   / _ \    / _ \
+##  / ___ \  / ___ \
+## /_/   \_\/_/   \_\
+## ======================================================================
+- name: gate_advanced_analytics_start
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed:
+      - compile_gpdb_centos6
+      - compile_gpdb_centos7
+      trigger: true
+    - get: bin_gpdb_centos6
+      passed:
+      - compile_gpdb_centos6
+    - get: bin_gpdb_centos7
+      passed:
+      - compile_gpdb_centos7
+
+- name: MADlib_Test_planner_centos6
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos6
+      trigger: true
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-dev-6
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-dev-6
+    params:
+      TEST_OS: centos
+      ORCA: "off"
+      DBVER: gpdb6-assert
+
+- name: MADlib_Test_orca_centos6
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos6
+      trigger: true
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-dev-6
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-dev-6
+    params:
+      TEST_OS: centos
+      ORCA: "on"
+      DBVER: gpdb6-assert
+
+- name: MADlib_Test_planner_centos7
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos7
+      trigger: true
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-test-7
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-test-7
+    params:
+      TEST_OS: centos
+      ORCA: "off"
+      DBVER: gpdb6-assert
+
+- name: MADlib_Test_orca_centos7
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos7
+      trigger: true
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-test-7
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-test-7
+    params:
+      TEST_OS: centos
+      ORCA: "on"
+      DBVER: gpdb6-assert
+
 ## ======================================================================
 ##  ____      _
 ## |  _ \ ___| | ___  __ _ ___  ___
@@ -2219,6 +2354,10 @@ jobs:
       - gpinitsystem
       - gpexpand_1
       - gpexpand_2
+      - MADlib_Test_orca_centos6
+      - MADlib_Test_planner_centos6
+      - MADlib_Test_orca_centos7
+      - MADlib_Test_planner_centos7
       - gpaddmirrors
       - gpexpand
       - gpcheck
@@ -2250,6 +2389,8 @@ jobs:
       - gpinitsystem
       - gpexpand_1
       - gpexpand_2
+      - MADlib_Test_orca_centos6
+      - MADlib_Test_planner_centos6
       - gpexpand
       - gpcheck
       - gppkg
@@ -2263,6 +2404,8 @@ jobs:
       - icw_gporca_centos7
       - icw_planner_centos7
       - resource_group_centos7
+      - MADlib_Test_orca_centos7
+      - MADlib_Test_planner_centos7
     - get: bin_gpdb_sles11
       trigger: true
       passed:

concourse/pipelines/templates/gpdb-tpl.yml

@@ -114,6 +114,19 @@ groups:
   - regression_tests_gphdfs_hadoop_centos
   - regression_tests_gphdfs_mapr_centos
 {% endif %}
+{% if "AA" in test_sections %}
+  ## --------------------------------------------------------------------
+  - gate_advanced_analytics_start
+{% if "centos6" in os_types %}
+  - MADlib_Test_orca_centos6
+  - MADlib_Test_planner_centos6
+{% endif %}
+{% if "centos7" in os_types %}
+  - MADlib_Test_orca_centos7
+  - MADlib_Test_planner_centos7
+{% endif %}
+{% endif %}
+
 
 {% if pipeline_type == "prod" %}
 ## ======================================================================
@@ -269,6 +282,21 @@ groups:
   - compile_gpdb_centos6
 
 {% endif %}
+
+{% if "AA" in test_sections %}
+- name: AdvancedAnalytics
+  jobs:
+  - gate_advanced_analytics_start
+{% if "centos6" in os_types %}
+  - MADlib_Test_orca_centos6
+  - MADlib_Test_planner_centos6
+{% endif %}
+{% if "centos7" in os_types %}
+  - MADlib_Test_orca_centos7
+  - MADlib_Test_planner_centos7
+{% endif %}
+{% endif %}
+
 ## ======================================================================
 ##                                           _
 ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  | |_ _   _ _ __   ___  ___
@@ -612,6 +640,25 @@ resources:
     stop: 7:00 AM
 
 {% endif %}
+{% if "AA" in test_sections %}
+- name: madlib_ci
+  type: git
+  source:
+    branch: {{madlib-ci-git-branch}}
+    uri: {{madlib-ci-git-remote}}
+    private_key: {{madlib-ci-git-key}}
+
+- name: madlib_gppkg
+  type: s3
+  source:
+    access_key_id: {{bucket-access-key-id}}
+    bucket: {{madlib-gppkg-bucket}}
+    region_name: {{aws-region}}
+    secret_access_key: {{bucket-secret-access-key}}
+    regexp: {{madlib-gpdb6-centos6-gppkg}}
+
+{% endif %}
+
 - name: reduced-frequency-trigger
   type: time
   source:
@@ -619,7 +666,6 @@ resources:
     days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday, Saturday]
     start: {{reduced-frequency-trigger-start}}
     stop: {{reduced-frequency-trigger-stop}}
-
 ## ======================================================================
 ##                   _
 ##   __ _ _ __   ___| |__   ___  _ __ ___
@@ -2198,6 +2244,123 @@ jobs:
       TARGET_OS_VERSION: 6
 
 {% endif %}
+
+{% if "AA" in test_sections %}
+## ======================================================================
+##     _        _
+##    / \      / \
+##   / _ \    / _ \
+##  / ___ \  / ___ \
+## /_/   \_\/_/   \_\
+## ======================================================================
+- name: gate_advanced_analytics_start
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed:
+{% if "centos6" in os_types %}
+      - compile_gpdb_centos6
+{% endif %}
+{% if "centos7" in os_types %}
+      - compile_gpdb_centos7
+{% endif %}
+      trigger: true
+{% if "centos6" in os_types %}
+    - get: bin_gpdb_centos6
+      passed:
+      - compile_gpdb_centos6
+{% endif %}
+{% if "centos7" in os_types %}
+    - get: bin_gpdb_centos7
+      passed:
+      - compile_gpdb_centos7
+{% endif %}
+
+{% if "centos6" in os_types %}
+- name: MADlib_Test_planner_centos6
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos6
+      trigger: [[ test_trigger ]]
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-dev-6
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-dev-6
+    params:
+      TEST_OS: centos
+      ORCA: "off"
+      DBVER: gpdb6-assert
+
+- name: MADlib_Test_orca_centos6
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos6
+      trigger: [[ test_trigger ]]
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-dev-6
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-dev-6
+    params:
+      TEST_OS: centos
+      ORCA: "on"
+      DBVER: gpdb6-assert
+{% endif %}
+
+{% if "centos7" in os_types %}
+- name: MADlib_Test_planner_centos7
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos7
+      trigger: [[ test_trigger ]]
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-test-7
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-test-7
+    params:
+      TEST_OS: centos
+      ORCA: "off"
+      DBVER: gpdb6-assert
+
+- name: MADlib_Test_orca_centos7
+  plan:
+  - aggregate:
+    - get: madlib_ci
+    - get: gpdb_src
+      passed: [gate_advanced_analytics_start]
+    - get: bin_gpdb
+      resource: bin_gpdb_centos7
+      trigger: [[ test_trigger ]]
+      passed: [gate_advanced_analytics_start]
+    - get: centos-gpdb-test-7
+    - get: madlib_gppkg
+  - task: MADlib_Test_gppkg
+    file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml
+    image: centos-gpdb-test-7
+    params:
+      TEST_OS: centos
+      ORCA: "on"
+      DBVER: gpdb6-assert
+{% endif %}
+{% endif %}
+
 {% if pipeline_type == "prod" %}
 ## ======================================================================
 ##  ____      _
@@ -2247,6 +2410,10 @@ jobs:
                     CLI_gpexpand_suites %}
       - [[ test_name ]]
 {% endfor %}
+      - MADlib_Test_orca_centos6
+      - MADlib_Test_planner_centos6
+      - MADlib_Test_orca_centos7
+      - MADlib_Test_planner_centos7
       - gpaddmirrors
       - gpexpand
       - gpcheck
@@ -2274,6 +2441,8 @@ jobs:
                     CLI_gpexpand_suites %}
       - [[ test_name ]]
 {% endfor %}
+      - MADlib_Test_orca_centos6
+      - MADlib_Test_planner_centos6
       - gpexpand
       - gpcheck
       - gppkg
@@ -2287,6 +2456,8 @@ jobs:
       - icw_gporca_centos7
       - icw_planner_centos7
       - resource_group_centos7
+      - MADlib_Test_orca_centos7
+      - MADlib_Test_planner_centos7
     - get: bin_gpdb_sles11
       trigger: true
       passed: