pipeline.yml

groups:
- name: all
  jobs:
  - compile_gpdb_centos6
  - compile_gpdb_open_source_centos6
  - compile_gpdb_binary_swap_centos6
  - compile_gpdb_centos7
  - compile_gpdb_sles11
  - compile_gpdb_ubuntu16
  - compile_gpdb_windows_cl
  - compile_gpdb_aix7_remote
  - icw_planner_centos6
  - icw_gporca_centos6
  - icw_gporca_centos7
  - icw_gporca_sles11
  - icw_planner_ictcp_centos6
  - client_loader_remote_test_aix
  - mpp_resource_group_centos6
  - MU_check_centos
  - MM_gpperfmon
  - MM_gpcheck
  - MM_analyzedb
  - DPM_backup-restore
  - MM_gppkg
  - MM_gpinitsystem
  - MM_gpinitstandby
  - MM_pt-rebuild
  - fts
  - storage
  - cs_walrep_1
  - cs_walrep_2
  - QP_memory-accounting
  - regression_tests_gpcloud_centos
  - regression_tests_gphdfs_centos
  - regression_tests_pxf_centos
  - gpdb_rc_packaging_centos
  - gpdb_rc_packaging_sles
  - DPM_backup_43_restore_5
  - MM_gpcheckcat
  - MM_gprecoverseg
  - MM_gpexpand
  - DPM_gptransfer-43x-to-5x
  - DPM_gptransfer-5x-to-5x
  - cs-pg-two-phase
  - cs-filerep-schema-topology-crashrecov
  - cs-filerep-end-to-end
  - cs-aoco-compression
  - mpp_interconnect
  - QP_runaway-query
  - QP_optimizer-functional
  - validate_pipeline

- name: Release
  jobs:
  - Release_Candidate

- name: Remaining Pulse
  jobs:
  - MM_gpcheckcat
  - MM_gpexpand
  - DPM_gptransfer-43x-to-5x
  - DPM_gptransfer-5x-to-5x
  - cs-pg-two-phase
  - cs-filerep-schema-topology-crashrecov
  - cs-filerep-end-to-end
  - cs-aoco-compression
  - mpp_interconnect

- name: Adopted CCP
  jobs:
  - cs_walrep_1
  - cs_walrep_2
  - mpp_resource_group_centos6
  - DPM_backup-restore
  - MM_gpcheckcat
  - MM_gppkg
  - MM_gprecoverseg
  - DPM_backup_43_restore_5

- name: Experimental Tests
  jobs:
  - icw_gporca_centos6_gpos_memory

## ======================================================================
## resource types
## ======================================================================

resource_types:
- name: terraform
  type: docker-image
  source:
    repository: ljfranklin/terraform-resource

## ======================================================================
## resources
## ======================================================================

resources:
- name: ccp_src
  type: git
  source:
    branch: {{ccp-git-branch}}
    private_key: {{ccp-git-key}}
    uri: {{ccp-git-remote}}
    tag_filter: 1.0.0-beta.1

- name: terraform
  type: terraform
  source:
    env:
      AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
      AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
    storage:
      access_key_id: {{tf-machine-access-key-id}}
      secret_access_key: {{tf-machine-secret-access-key}}
      region_name: {{aws-region}}
      # This is not parameterized, on purpose. All tfstates will go to this spot,
      # and different teams will place there clusters' tfstate files under different paths
      bucket: gpdb5-pipeline-dynamic-terraform
      ###########################################################
      # Two conventions are allowed for bucket_path:            #
      # 1) Cluster are expected to be destroyed automatically   #
      #    Toolsmiths will reap old and orphaned clusters       #
      #                                                         #
      #      bucket_path: prod/[Pipeline Name]/                 #
      #                                                         #
      # 2) Long lived clusters for development.                 #
      #    The team that creates it is responsible for cluster  #
      #                                                         #
      #     bucket_path: dev/[Team Name]/                       #
      #                                                         #
      ###########################################################
      bucket_path: {{tf-bucket-path}}

- name: gpdb_src
  type: git
  source:
    branch: {{gpdb-git-branch}}
    uri: {{gpdb-git-remote}}
    ignore_paths:
    - gpdb-doc/*
    - README*

- name: gpdb_src_binary_swap
  type: git
  source:
    branch: {{gpdb-binary-swap-git-tag}}
    uri: {{gpdb-binary-swap-git-remote}}
    ignore_paths:
    - gpdb-doc/*
    - README*

- name: gpaddon_src
  type: git
  source:
    branch: {{gpaddon-git-branch}}
    private_key: {{gpaddon-git-key}}
    uri: {{gpaddon-git-remote}}

- name: pxf_src
  type: git
  source:
    branch: {{pxf-git-branch}}
    private_key: {{pxf-git-key}}
    uri: {{pxf-git-remote}}

- name: pxf_automation_src
  type: git
  source:
    branch: {{pxf_automation-git-branch}}
    private_key: {{pxf-git-key}}
    uri: {{pxf_automation-git-remote}}

- name: centos-gpdb-dev-6
  type: docker-image
  source:
    repository: pivotaldata/centos-gpdb-dev
    tag: '6-gcc6.2-llvm3.7'

- name: centos-gpdb-dev-7
  type: docker-image
  source:
    repository: pivotaldata/centos-gpdb-dev
    tag: 7-gcc6.2-llvm3.7

- name: sles-gpdb-dev-11-beta
  type: docker-image
  source:
    repository: pivotaldata/sles-gpdb-dev
    tag: 11-beta

- name: ubuntu-gpdb-dev-16
  type: docker-image
  source:
    repository: pivotaldata/ubuntu-gpdb-dev
    tag: '16.04'

- name: centos-mingw
  type: docker-image
  source:
    repository: pivotaldata/centos-mingw

- name: bin_gpdb_centos6
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_centos_versioned_file}}

- name: binary_swap_gpdb_centos6
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{binary_swap_gpdb_centos_versioned_file}}

- name: bin_gpdb_centos7
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_centos7_versioned_file}}

- name: bin_gpdb_sles11
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_sles11_versioned_file}}

- name: bin_gpdb_windows_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: windows-cl/greenplum-clients-5.(.*)-WinXP-x86_32.msi

- name: bin_gpdb_windows_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: windows-cl/greenplum-loaders-5.(.*)-WinXP-x86_32.msi

- name: bin_gpdb4_centos6
  type: s3
  source:
    access_key_id: {{gpdb4-bucket-access-key-id}}
    bucket: {{gpdb4-bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{gpdb4-bucket-secret-access-key}}
    versioned_file: bin_gpdb_centos/bin_gpdb.tar.gz

- name: installer_rhel6_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-rhel6-x86_64.zip

- name: installer_rhel6_gpdb_rc_sha256
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-rhel6-x86_64.zip.sha256

- name: installer_rhel7_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-rhel7-x86_64.zip

- name: installer_rhel7_gpdb_rc_sha256
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-rhel7-x86_64.zip.sha256

- name: installer_sles11_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-sles11-x86_64.zip

- name: installer_sles11_gpdb_rc_sha256
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-sles11-x86_64.zip.sha256

- name: installer_appliance_rhel6_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/appliance_installer/greenplum-db-appliance-(.*)-rhel6-x86_64.zip

- name: installer_appliance_rhel6_gpdb_rc_sha256
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/appliance_installer/greenplum-db-appliance-(.*)-rhel6-x86_64.zip.sha256

- name: installer_rhel6_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-rhel6-x86_64.zip

- name: installer_rhel6_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-rhel6-x86_64.zip

- name: installer_appliance_rhel7_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/appliance_installer/greenplum-db-appliance-(.*)-rhel7-x86_64.zip

- name: installer_appliance_rhel7_gpdb_rc_sha256
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/appliance_installer/greenplum-db-appliance-(.*)-rhel7-x86_64.zip.sha256

- name: installer_rhel7_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-rhel7-x86_64.zip

- name: installer_rhel7_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-rhel7-x86_64.zip

- name: installer_aix7_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-aix7_ppc_64.zip

- name: installer_aix7_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-aix7_ppc_64.zip

- name: qautils_rhel6_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: deliverables/QAUtils-rhel6-x86_64.tar.gz

- name: qautils_rhel7_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: deliverables/QAUtils-rhel7-x86_64.tar.gz

- name: qautils_sles11_tarball
  type: s3
  source:
    private: true
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: deliverables/QAUtils-sles11-x86_64.tar.gz

- name: installer_sles11_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-sles11-x86_64.zip

- name: installer_sles11_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-sles11-x86_64.zip

- name: gpdb_src_tinc_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-db-(.*)-src.tar.gz

- name: gpdb_src_behave_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-db-(.*)-behave.tar.gz

- name: singlecluster-HDP
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    secret_access_key: {{bucket-secret-access-key}}
    bucket: {{pxf-aws-bucket-name}}
    region_name: {{aws-region}}
    versioned_file: singlecluster-without-pxf/singlecluster-HDP.tar.gz

- name: nightly-trigger
  type: time
  source:
    location: America/Los_Angeles
    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
    start: 6:00 AM
    stop: 7:00 AM

- name: reduced-frequency-trigger
  type: time
  source:
    location: America/Los_Angeles
    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
    start: {{reduced-frequency-trigger-start}}
    stop: {{reduced-frequency-trigger-stop}}

## ======================================================================
## reusable anchors
## ======================================================================
ccp_create_params_anchor: &ccp_default_params
  action: create
  delete_on_failure: true
  generate_random_name: true
  terraform_source: ccp_src/aws/

ccp_vars_anchor: &ccp_default_vars
  aws_instance-node-instance_type: t2.medium
  platform: centos6

ccp_destroy_anchor: &ccp_destroy
  put: terraform
  params:
    action: destroy
    env_name_file: terraform/name
    terraform_source: ccp_src/aws/
    vars:
      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
  get_params:
    action: destroy

debug_sleep_anchor: &debug_sleep
  do:
  - task: debug_sleep
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: alpine
          tag: latest
      run:
        path: 'sh'
        args: ['-c', 'sleep 6h']
    ensure:
      <<: *ccp_destroy

## ======================================================================
## jobs
## ======================================================================

# Stage 1: Build and C Unit Tests

jobs:

- name: compile_gpdb_centos6
  plan:
  - aggregate:
    - get: reduced-frequency-trigger
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src
      trigger: {{gpdb_src-trigger-flag}}
    - get: gpaddon_src
    - get: pxf_src
      trigger: true
    - get: centos-gpdb-dev-6
  - task: compile_gpdb
    file: gpdb_src/concourse/tasks/compile_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      IVYREPO_HOST: {{ivyrepo_host}}
      IVYREPO_REALM: {{ivyrepo_realm}}
      IVYREPO_USER: {{ivyrepo_user}}
      IVYREPO_PASSWD: {{ivyrepo_passwd}}
      CONFIGURE_FLAGS: {{configure_flags}}
      TARGET_OS: centos
      TARGET_OS_VERSION: 6
      BLD_TARGETS: "clients loaders"
  - aggregate:
    - put: bin_gpdb_centos6
      params:
        file: gpdb_artifacts/bin_gpdb.tar.gz
    - put: installer_rhel6_gpdb_clients
      params:
        file: gpdb_artifacts/greenplum-clients-*-rhel6-x86_64.zip
    - put: installer_rhel6_gpdb_loaders
      params:
        file: gpdb_artifacts/greenplum-loaders-*-rhel6-x86_64.zip

- name: compile_gpdb_open_source_centos6
  public: true
  plan:
  - aggregate:
    - get: reduced-frequency-trigger
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src
      trigger: {{gpdb_src-trigger-flag}}
    - get: centos-gpdb-dev-6
  - task: compile_gpdb
    image: centos-gpdb-dev-6
    file: gpdb_src/concourse/tasks/compile_gpdb_open_source_centos.yml
    params:
        CONFIGURE_FLAGS: {{configure_flags}}

- name: compile_gpdb_centos7
  plan:
  - aggregate:
    - get: reduced-frequency-trigger
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src
      trigger: {{gpdb_src-trigger-flag}}
    - get: gpaddon_src
    - get: pxf_src
      trigger: true
    - get: centos-gpdb-dev-7
  - task: compile_gpdb
    image: centos-gpdb-dev-7
    file: gpdb_src/concourse/tasks/compile_gpdb.yml
    params:
      IVYREPO_HOST: {{ivyrepo_host}}
      IVYREPO_REALM: {{ivyrepo_realm}}
      IVYREPO_USER: {{ivyrepo_user}}
      IVYREPO_PASSWD: {{ivyrepo_passwd}}
      CONFIGURE_FLAGS: {{configure_flags}}
      TARGET_OS: centos
      TARGET_OS_VERSION: 7
      BLD_TARGETS: "clients loaders"
  - aggregate:
    - put: bin_gpdb_centos7
      params:
        file: gpdb_artifacts/bin_gpdb.tar.gz
    - put: installer_rhel7_gpdb_clients
      params:
        file: gpdb_artifacts/greenplum-clients-*-rhel7-x86_64.zip
    - put: installer_rhel7_gpdb_loaders
      params:
        file: gpdb_artifacts/greenplum-loaders-*-rhel7-x86_64.zip

- name: compile_gpdb_ubuntu16
  plan:
  - aggregate:
    - get: reduced-frequency-trigger
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src
      trigger: {{gpdb_src-trigger-flag}}
    - get: ubuntu-gpdb-dev-16
  - task: compile_gpdb
    image: ubuntu-gpdb-dev-16
    file: gpdb_src/concourse/tasks/compile_gpdb_open_source_ubuntu.yml
    params:
      CONFIGURE_FLAGS: {{configure_flags}}

- name: compile_gpdb_sles11
  plan:
  - aggregate:
    - get: reduced-frequency-trigger
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src
      trigger: {{gpdb_src-trigger-flag}}
    - get: gpaddon_src
    - get: pxf_src
  - task: compile_gpdb
    file: gpdb_src/concourse/tasks/compile_gpdb_sles11.yml
    params:
      IVYREPO_HOST: {{ivyrepo_host}}
      IVYREPO_REALM: {{ivyrepo_realm}}
      IVYREPO_USER: {{ivyrepo_user}}
      IVYREPO_PASSWD: {{ivyrepo_passwd}}
      CONFIGURE_FLAGS: {{configure_flags}}
      TARGET_OS: sles
      TARGET_OS_VERSION:
      BLD_TARGETS: "clients loaders"
  - put: bin_gpdb_sles11
    params:
      file: gpdb_artifacts/bin_gpdb.tar.gz
  - put: installer_sles11_gpdb_clients
    params:
      file: gpdb_artifacts/greenplum-clients-*-sles11-x86_64.zip
  - put: installer_sles11_gpdb_loaders
    params:
      file: gpdb_artifacts/greenplum-loaders-*-sles11-x86_64.zip

- name: compile_gpdb_windows_cl
  plan:
  - aggregate:
    - get: reduced-frequency-trigger
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src
      trigger: {{gpdb_src-trigger-flag}}
    - get: gpaddon_src
    - get: pxf_src
    - get: centos-mingw
  - task: compile_gpdb
    file: gpdb_src/concourse/tasks/compile_gpdb.yml
    image: centos-mingw
    params:
      TARGET_OS: win32
      TARGET_OS_VERSION:
      BLD_TARGETS: clients loaders
      IVYREPO_HOST: {{ivyrepo_host}}
      IVYREPO_REALM: {{ivyrepo_realm}}
      IVYREPO_USER: {{ivyrepo_user}}
      IVYREPO_PASSWD: {{ivyrepo_passwd}}
  - aggregate:
    - put: bin_gpdb_windows_clients
      params:
        file: gpdb_artifacts/greenplum-clients-*-WinXP-x86_32.msi
    - put: bin_gpdb_windows_loaders
      params:
        file: gpdb_artifacts/greenplum-loaders-*-WinXP-x86_32.msi

# Compile gpdb on a remote AIX machine, triggered by concourse.
# We need to serialize this job to avoid overwhelming workload
# on remote machine.
- name: compile_gpdb_aix7_remote
  serial: true
  plan:
  - aggregate:
    - get: nightly-trigger
      trigger: true
    - get: gpdb_src
    - get: gpaddon_src
    - get: centos-gpdb-dev-6
  - task: compile_gpdb_aix7_remote
    file: gpdb_src/concourse/tasks/compile_gpdb_remote.yml
    image: centos-gpdb-dev-6
    params:
      REMOTE_HOST: {{remote_host}}
      REMOTE_PORT: {{remote_port}}
      REMOTE_USER: {{remote_user}}
      REMOTE_KEY: {{remote_key}}

      IVYREPO_HOST: {{ivyrepo_host}}
      IVYREPO_REALM: {{ivyrepo_realm}}
      IVYREPO_USER: {{ivyrepo_user}}
      IVYREPO_PASSWD: {{ivyrepo_passwd}}

      BLD_TARGETS: "clients loaders"
  - aggregate:
    - put: installer_aix7_gpdb_clients
      params:
        file: gpdb_artifacts/greenplum-clients-*-aix7_ppc_64.zip
    - put: installer_aix7_gpdb_loaders
      params:
        file: gpdb_artifacts/greenplum-loaders-*-aix7_ppc_64.zip

# This acts like a cache as this job will only be run once to get a
# binary to use for our binary swap compatibility tests. Setting a new
# tag or branch for the gpdb_src_binary_swap resource via set-pipeline
# will replace the cached binary.
- name: compile_gpdb_binary_swap_centos6
  plan:
  - aggregate:
    - get: gpdb_src
      resource: gpdb_src_binary_swap
      trigger: true
    - get: gpaddon_src
    - get: pxf_src
    - get: centos-gpdb-dev-6
  - task: compile_gpdb
    file: gpdb_src/concourse/tasks/compile_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      IVYREPO_HOST: {{ivyrepo_host}}
      IVYREPO_REALM: {{ivyrepo_realm}}
      IVYREPO_USER: {{ivyrepo_user}}
      IVYREPO_PASSWD: {{ivyrepo_passwd}}
      TARGET_OS: centos
      TARGET_OS_VERSION: 6
  - aggregate:
    - put: binary_swap_gpdb_centos6
      params:
        file: gpdb_artifacts/bin_gpdb.tar.gz

# Stage 2a: Run regression tests against binaries (make installcheck-world and similar)

- name: icw_planner_centos6
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: binary_swap_gpdb
      resource: binary_swap_gpdb_centos6
      passed: [compile_gpdb_binary_swap_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: ic_gpdb
    file: gpdb_src/concourse/tasks/ic_gpdb_binary_swap.yml
    image: centos-gpdb-dev-6
    params:
      MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off' installcheck-world
      BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
      TEST_OS: centos
      TEST_BINARY_SWAP: false
      CONFIGURE_FLAGS: {{configure_flags}}

- name: icw_gporca_centos6
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: ic_gpdb
    file: gpdb_src/concourse/tasks/ic_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=on' installcheck-world
      BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
      TEST_OS: centos
      CONFIGURE_FLAGS: {{configure_flags}}

- name: icw_gporca_centos6_gpos_memory
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: ic_gpdb
    file: gpdb_src/concourse/tasks/ic_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=on' installcheck-world
      BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off | optimizer_use_gpdb_allocators=on"
      TEST_OS: centos
      CONFIGURE_FLAGS: {{configure_flags}}

- name: icw_gporca_centos7
  serial: true
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos7]
    - get: bin_gpdb
      resource: bin_gpdb_centos7
      passed: [compile_gpdb_centos7]
      trigger: true
    - get: centos-gpdb-dev-7
  - task: ic_gpdb
    file: gpdb_src/concourse/tasks/ic_gpdb.yml
    image: centos-gpdb-dev-7
    params:
      MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=on' installcheck-world
      BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
      TEST_OS: centos
      CONFIGURE_FLAGS: {{configure_flags}}

- name: icw_gporca_sles11
  serial: true
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_sles11]
    - get: bin_gpdb
      resource: bin_gpdb_sles11
      passed: [compile_gpdb_sles11]
      trigger: true
    - get: sles-gpdb-dev-11-beta
  - task: ic_gpdb
    file: gpdb_src/concourse/tasks/ic_gpdb.yml
    image: sles-gpdb-dev-11-beta
    params:
      MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=on' installcheck-world
      BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
      TEST_OS: sles
      CONFIGURE_FLAGS: {{configure_flags}}

- name: icw_planner_ictcp_centos6
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: ic_gpdb
    file: gpdb_src/concourse/tasks/ic_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      MAKE_TEST_COMMAND: PGOPTIONS='-c gp_interconnect_type=tcp -c optimizer=off' installcheck-world
      BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
      TEST_OS: centos

- name: client_loader_remote_test_aix
  serial: true
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_aix7_remote]
    - get: installer_aix7_gpdb_clients
      passed: [compile_gpdb_aix7_remote]
    - get: installer_aix7_gpdb_loaders
      passed: [compile_gpdb_aix7_remote]
      trigger: true
    - get: bin_gpdb
      passed: [compile_gpdb_centos6]
      resource: bin_gpdb_centos6
    - get: centos-gpdb-dev-6
  - task: ic_gpdb_cl
    file: gpdb_src/concourse/tasks/ic_gpdb_remote.yml
    image: centos-gpdb-dev-6
    params:
      BLD_TARGETS: "clients loaders"
      REMOTE_HOST: {{remote_host}}
      REMOTE_PORT: {{remote_port}}
      REMOTE_USER: {{remote_user}}
      REMOTE_KEY: {{remote_key}}

- name: mpp_resource_group_centos6
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    input_mapping:
      gpdb_binary: bin_gpdb
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml
    image: centos-gpdb-dev-6
    params:
      TEST_OS: centos6
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: MU_check_centos
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: MU_check_centos
    file: gpdb_src/concourse/tasks/gpMgmt_check_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      TEST_OS: centos

- name: regression_tests_gpcloud_centos
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: regression_tests_gpcloud
    file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml
    image: centos-gpdb-dev-6
    params:
      gpcloud_access_key_id: {{gpcloud-access-key-id}}
      gpcloud_secret_access_key: {{gpcloud-secret-access-key}}
      overwrite_gpcloud: false
      TARGET_OS: centos
      TARGET_OS_VERSION: 6

- name: regression_tests_pxf_centos
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: singlecluster
      resource: singlecluster-HDP
      trigger: true
    - get: pxf_automation_src
      trigger: true
    - get: centos-gpdb-dev-6
  - task: regression_tests_pxf
    file: gpdb_src/concourse/tasks/regression_tests_pxf.yml
    image: centos-gpdb-dev-6
    params:
      GROUP: gpdb
      TARGET_OS: centos
      TARGET_OS_VERSION: 6

- name: MM_gpperfmon
  plan:
  - aggregate: &gets_for_behave
    - get: gpdb_src
      params:
        submodules:
        - gpMgmt/bin/pythonSrc/ext
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - task: gpperfmon
    file: gpdb_src/concourse/tasks/behave_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_TAGS: gpperfmon

- name: MM_gpcheck
  plan:
  - aggregate: *gets_for_behave
  - task: gpcheck_as_gpadmin
    file: gpdb_src/concourse/tasks/behave_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_TAGS: gpcheck_as_gpadmin
      GPCHECK_SETUP: true

- name: MM_analyzedb
  plan:
  - aggregate: *gets_for_behave
  - task: analyzedb
    file: gpdb_src/concourse/tasks/behave_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_TAGS: analyzedb

- name: DPM_backup-restore
  plan:
  - aggregate:
    - get: gpdb_src
      params:
        submodules:
        - gpMgmt/bin/pythonSrc/ext
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/run_behave.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_FLAGS: --tags=backups,backup_and_restore_backups,backup_and_restore_restores,restores --tags=-nbuonly --tags=-ddonly
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: DPM_backup_43_restore_5
  plan:
  - get: nightly-trigger
    trigger: {{nightly-trigger-flag}}
  - aggregate:
    - get: gpdb_src
      params:
        submodules:
        - gpMgmt/bin/pythonSrc/ext
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb4_binary
      resource: bin_gpdb4_centos6
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: setup_gpdb4
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: centos
          tag: "7"
      inputs:
      - name: terraform
      - name: gpdb4_binary
      - name: ccp_src
      - name: gpdb_src
      outputs:
      - name: cluster_env_files
      run:
        path: bash
        args:
        - -c
        - |
          set -ex
          source ccp_src/aws/generate_env_files.sh
          ccp_src/aws/setup_ssh_to_cluster.sh
          ccp_src/aws/preinstall_gpdb.sh
          ccp_src/aws/install_gpdb.sh gpdb4_binary
          ccp_src/aws/run_inspec.sh
    on_failure:
      <<: *ccp_destroy
  - task: run_backup_tests
    file: gpdb_src/concourse/tasks/run_behave.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_FLAGS: --tags=backup_and_restore_backups --tags=-nbuonly --tags=-ddonly --tags=-skip_for_gpdb_43
    on_failure:
      <<: *debug_sleep
  - task: setup_gpdb5
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: centos
          tag: "7"
      inputs:
      - name: terraform
      - name: gpdb_binary
      - name: ccp_src
      - name: gpdb_src
      - name: cluster_env_files
      run:
        path: bash
        args:
        - -c
        - |
          set -ex
          ccp_src/aws/setup_ssh_to_cluster.sh
          scp cluster_env_files/hostfile_all mdw:/home/gpadmin/hostfile_all
          ssh -t mdw "source /home/gpadmin/gpdb_src/concourse/scripts/backup_utils.sh; copy_backup_files"
          ssh -t mdw "source /home/gpadmin/gpdb_src/concourse/scripts/backup_utils.sh; destroy_gpdb"
          ccp_src/aws/install_gpdb.sh gpdb_binary
          ssh -t mdw "source /home/gpadmin/gpdb_src/concourse/scripts/backup_utils.sh; restore_backup_files"
    on_failure:
      <<: *debug_sleep
  - task: run_restore_tests
    file: gpdb_src/concourse/tasks/run_behave.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_FLAGS: --tags=backup_and_restore_restores --tags=-nbuonly --tags=-ddonly --tags=-skip_for_gpdb_43
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: MM_gppkg
  plan:
  - aggregate:
    - get: gpdb_src
      params:
        submodules:
        - gpMgmt/bin/pythonSrc/ext
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    on_failure:
      <<: *ccp_destroy
  - task: setup_gppkg_second_install
    file: gpdb_src/concourse/tasks/gppkg_behave.yml
    params:
      SECOND_BINARY_INSTALL_LOCATION: /tmp/gppkg_migrate
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/run_behave.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_FLAGS: --tags=gppkg
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: MM_gpinitsystem
  plan:
  - aggregate: *gets_for_behave
  - task: gpinitsystem
    file: gpdb_src/concourse/tasks/behave_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_TAGS: gpinitsystem

- name: MM_gpinitstandby
  plan:
  - aggregate: *gets_for_behave
  - task: gpinitstandby
    file: gpdb_src/concourse/tasks/behave_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_TAGS: gpinitstandby

- name: MM_pt-rebuild
  plan:
  - aggregate: *gets_for_behave
  - aggregate:
    - task: persistent_table_rebuild
      file: gpdb_src/concourse/tasks/behave_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        BEHAVE_TAGS: persistent_rebuild
    - task: persistent_table_rebuild_crash_recovery
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        MAKE_TEST_COMMAND: persistent_table_rebuild
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}

- name: fts
  plan:
  - aggregate:
    - get: gpdb_src
      params: {submodules: none}
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - aggregate:
    - task: fts_transitions_part01
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        MAKE_TEST_COMMAND: fts_transitions_part01
        BLDWRAP_POSTGRES_CONF_ADDONS: gp_segment_connect_timeout=35 gp_fts_probe_interval=20
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
    - task: fts_transitions_part02
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        MAKE_TEST_COMMAND: fts_transitions_part02
        BLDWRAP_POSTGRES_CONF_ADDONS: gp_segment_connect_timeout=35 gp_fts_probe_interval=20
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
    - task: fts_transitions_part03
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        MAKE_TEST_COMMAND: fts_transitions_part03
        BLDWRAP_POSTGRES_CONF_ADDONS: gp_segment_connect_timeout=35 gp_fts_probe_interval=20
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}

- name: storage
  plan:
  - aggregate:
    - get: gpdb_src
      params: {submodules: none}
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - aggregate:
    - task: aocoalter_catalog_loaders
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      params:
        MAKE_TEST_COMMAND: aocoalter_catalog_loaders
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
      image: centos-gpdb-dev-6
    - task: storage_persistent_accessmethods_and_vacuum
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      params:
        MAKE_TEST_COMMAND: storage_persistent_accessmethods_and_vacuum
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
      image: centos-gpdb-dev-6
      timeout: 3h
    - task: storage_filerep
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      params:
        MAKE_TEST_COMMAND: storage_filerep
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
      image: centos-gpdb-dev-6
      timeout: 3h
    - task: storage_uao_transactionmanagement
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      params:
        MAKE_TEST_COMMAND: storage_uao_and_transactionmanagement
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
      image: centos-gpdb-dev-6
      timeout: 3h
    - task: storage_vacuum_xidlimits
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      params:
        MAKE_TEST_COMMAND: storage_vacuum_xidlimits
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
      image: centos-gpdb-dev-6
      timeout: 3h

- name: cs_walrep_1
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/run_tinc.yml
    image: centos-gpdb-dev-6
    params:
      TINC_TARGET: walrep_1
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: cs_walrep_2
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/run_tinc.yml
    image: centos-gpdb-dev-6
    params:
      TINC_TARGET: walrep_2
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: QP_memory-accounting
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: bin_gpdb
      passed: [compile_gpdb_centos6]
      resource: bin_gpdb_centos6
    - get: centos-gpdb-dev-6
  - task: memory-accounting
    timeout: 3h
    file: gpdb_src/concourse/tasks/tinc_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      MAKE_TEST_COMMAND: memory_accounting
      TEST_OS: "centos"
      CONFIGURE_FLAGS: {{configure_flags}}

- name: regression_tests_gphdfs_centos
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      passed: [compile_gpdb_centos6]
      trigger: true
      resource: bin_gpdb_centos6
    - get: centos-gpdb-dev-6
  - task: regression_tests_gphdfs
    file: gpdb_src/concourse/tasks/regression_tests_gphdfs.yml
    image: centos-gpdb-dev-6
    params:
      TARGET_OS: centos
      TARGET_OS_VERSION: 6

# Stage 2b: Packaging

- name: gpdb_rc_packaging_centos
  plan:
  - aggregate:
    - get: gpdb_src
      passed:
      - compile_gpdb_centos6
      - compile_gpdb_centos7
    - get: gpaddon_src
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: bin_gpdb_centos7
      passed: [compile_gpdb_centos7]
      trigger: true
    - get: centos-gpdb-dev-6
    - get: centos-gpdb-dev-7
  - task: separate_qautils_files_for_rc_centos6
    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
    image: centos-gpdb-dev-6
    input_mapping:
      bin_gpdb: bin_gpdb_centos6
    output_mapping:
      rc_bin_gpdb: rc_bin_gpdb_rhel6
    params:
      QAUTILS_TARBALL: rc_bin_gpdb/QAUtils-rhel6-x86_64.tar.gz

  - task: separate_qautils_files_for_rc_centos7
    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
    image: centos-gpdb-dev-7
    input_mapping:
      bin_gpdb: bin_gpdb_centos7
    output_mapping:
      rc_bin_gpdb: rc_bin_gpdb_rhel7
    params:
      QAUTILS_TARBALL: rc_bin_gpdb/QAUtils-rhel7-x86_64.tar.gz

  - task: gpdb_src_tinc_packaging
    file: gpdb_src/concourse/tasks/gpdb_src_tinc_packaging.yml
    image: centos-gpdb-dev-6
    input_mapping:
      bin_gpdb: bin_gpdb_centos6
    output_mapping:
      rc_bin_gpdb: packaged_gpdb_src_tinc
    params:
      GPDB_SRC_TAR_GZ: rc_bin_gpdb/greenplum-db-@GP_VERSION@-src.tar.gz

  - task: gpdb_src_behave_packaging
    file: gpdb_src/concourse/tasks/gpdb_src_behave_packaging.yml
    image: centos-gpdb-dev-6
    input_mapping:
      bin_gpdb: bin_gpdb_centos6
    output_mapping:
      rc_bin_gpdb: packaged_gpdb_src_behave
    params:
      GPDB_SRC_TAR_GZ: rc_bin_gpdb/greenplum-db-@GP_VERSION@-behave.tar.gz

  - aggregate:
    - task: gpdb_rc_packaging_centos6
      file: gpdb_src/concourse/tasks/gpdb_packaging.yml
      image: centos-gpdb-dev-6
      input_mapping:
        bin_gpdb: rc_bin_gpdb_rhel6
      output_mapping:
        packaged_gpdb: packaged_gpdb_rc_centos6
      params:
        INSTALL_SCRIPT_SRC: gpdb_src/gpAux/addon/license/installer-header-rhel-gpdb.sh
        INSTALLER_ZIP: packaged_gpdb/greenplum-db-@GP_VERSION@-rhel6-x86_64.zip
        ADD_README_INSTALL: true
    - task: gpdb_appliance_rhel6_rc_packaging
      file: gpdb_src/concourse/tasks/gpdb_packaging.yml
      image: centos-gpdb-dev-6
      input_mapping:
        bin_gpdb: rc_bin_gpdb_rhel6
      output_mapping:
        packaged_gpdb: packaged_gpdb_appliance_rc_centos6
      params:
        INSTALL_SCRIPT_SRC: gpdb_src/gpAux/addon/license/installer-appliance-header-rhel-gpdb.sh
        INSTALLER_ZIP: packaged_gpdb/greenplum-db-appliance-@GP_VERSION@-rhel6-x86_64.zip

    - task: gpdb_rc_packaging_centos7
      file: gpdb_src/concourse/tasks/gpdb_packaging.yml
      image: centos-gpdb-dev-7
      input_mapping:
        bin_gpdb: rc_bin_gpdb_rhel7
      output_mapping:
        packaged_gpdb: packaged_gpdb_rc_centos7
      params:
        INSTALL_SCRIPT_SRC: gpdb_src/gpAux/addon/license/installer-header-rhel-gpdb.sh
        INSTALLER_ZIP: packaged_gpdb/greenplum-db-@GP_VERSION@-rhel7-x86_64.zip
        ADD_README_INSTALL: true
    - task: gpdb_appliance_rhel7_rc_packaging
      file: gpdb_src/concourse/tasks/gpdb_packaging.yml
      image: centos-gpdb-dev-7
      input_mapping:
        bin_gpdb: rc_bin_gpdb_rhel7
      output_mapping:
        packaged_gpdb: packaged_gpdb_appliance_rc_centos7
      params:
        INSTALL_SCRIPT_SRC: gpdb_src/gpAux/addon/license/installer-appliance-header-rhel-gpdb.sh
        INSTALLER_ZIP: packaged_gpdb/greenplum-db-appliance-@GP_VERSION@-rhel7-x86_64.zip
  - aggregate:
    # RHEL 6
    - put: installer_rhel6_gpdb_rc
      params:
        file: packaged_gpdb_rc_centos6/greenplum-db-*-rhel6-x86_64.zip
    - put: installer_rhel6_gpdb_rc_sha256
      params:
        file: packaged_gpdb_rc_centos6/greenplum-db-*-rhel6-x86_64.zip.sha256
    - put: installer_appliance_rhel6_gpdb_rc
      params:
        file: packaged_gpdb_appliance_rc_centos6/greenplum-db-appliance-*-rhel6-x86_64.zip
    - put: installer_appliance_rhel6_gpdb_rc_sha256
      params:
        file: packaged_gpdb_appliance_rc_centos6/greenplum-db-appliance-*-rhel6-x86_64.zip.sha256
    - put: qautils_rhel6_tarball
      params:
        file: rc_bin_gpdb_rhel6/QAUtils-rhel6-x86_64.tar.gz

    # RHEL 7
    - put: installer_rhel7_gpdb_rc
      params:
        file: packaged_gpdb_rc_centos7/greenplum-db-*-rhel7-x86_64.zip
    - put: installer_rhel7_gpdb_rc_sha256
      params:
        file: packaged_gpdb_rc_centos7/greenplum-db-*-rhel7-x86_64.zip.sha256
    - put: installer_appliance_rhel7_gpdb_rc
      params:
        file: packaged_gpdb_appliance_rc_centos7/greenplum-db-appliance-*-rhel7-x86_64.zip
    - put: installer_appliance_rhel7_gpdb_rc_sha256
      params:
        file: packaged_gpdb_appliance_rc_centos7/greenplum-db-appliance-*-rhel7-x86_64.zip.sha256
    - put: qautils_rhel7_tarball
      params:
        file: rc_bin_gpdb_rhel7/QAUtils-rhel7-x86_64.tar.gz

    # Source
    - put: gpdb_src_tinc_tarball
      params:
        file: packaged_gpdb_src_tinc/greenplum-db-*-src.tar.gz
    - put: gpdb_src_behave_tarball
      params:
        file: packaged_gpdb_src_behave/greenplum-db-*-behave.tar.gz

- name: gpdb_rc_packaging_sles
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_sles11]
    - get: gpaddon_src
      passed: [compile_gpdb_sles11]
    - get: bin_gpdb
      resource: bin_gpdb_sles11
      passed: [compile_gpdb_sles11]
      trigger: true
    - get: sles-gpdb-dev-11-beta
  - task: separate_qautils_files_for_rc
    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
    image: sles-gpdb-dev-11-beta
    params:
      QAUTILS_TARBALL: rc_bin_gpdb/QAUtils-sles11-x86_64.tar.gz
  - task: gpdb_rc_packaging_sles
    file: gpdb_src/concourse/tasks/gpdb_packaging.yml
    image: sles-gpdb-dev-11-beta
    input_mapping:
      bin_gpdb: rc_bin_gpdb
    params:
      INSTALL_SCRIPT_SRC: gpdb_src/gpAux/addon/license/installer-header-sles-gpdb.sh
      INSTALLER_ZIP: packaged_gpdb/greenplum-db-@GP_VERSION@-sles11-x86_64.zip
      ADD_README_INSTALL: true
  - put: installer_sles11_gpdb_rc
    params:
      file: packaged_gpdb/greenplum-db-*-sles11-x86_64.zip
  - put: installer_sles11_gpdb_rc_sha256
    params:
      file: packaged_gpdb/greenplum-db-*-sles11-x86_64.zip.sha256
  - put: qautils_sles11_tarball
    params:
      file: rc_bin_gpdb/QAUtils-sles11-x86_64.tar.gz

- name: MM_gpcheckcat
  plan:
  - aggregate:
    - get: gpdb_src
      params:
        submodules:
        - gpMgmt/bin/pythonSrc/ext
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/run_behave.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_FLAGS: --tags=gpcheckcat
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

- name: MM_gprecoverseg
  plan:
  - aggregate:
    - get: gpdb_src
      params:
        submodules:
        - gpMgmt/bin/pythonSrc/ext
      passed: [compile_gpdb_centos6]
    - get: gpdb_binary
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
  - put: terraform
    params:
      <<: *ccp_default_params
      vars:
        <<: *ccp_default_vars
  - task: gen_cluster
    file: ccp_src/ci/tasks/gen_cluster.yml
    on_failure:
      <<: *ccp_destroy
  - task: run_tests
    file: gpdb_src/concourse/tasks/run_behave.yml
    image: centos-gpdb-dev-6
    params:
      BEHAVE_FLAGS: --tags=gprecoverseg
    on_failure:
      <<: *debug_sleep
  - *ccp_destroy

# Stage 3: Trigger jobs that rely on packaging
- name: MM_gpexpand
  plan:
  - get: nightly-trigger
    trigger: {{nightly-trigger-flag}}
  - aggregate: &post_packaging_gets_trigger_based_on_flag
    - get: gpdb_src
      params: {submodules: none}
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
      trigger: {{reduced-frequency-trigger-flag}}
    - get: gpdb_src_tinc_tarball
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
    - get: installer_rhel6_gpdb_rc
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
    - get: qautils_rhel6_tarball
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
    - get: gpdb_src_behave_tarball
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: &input_mappings
      gpdb_src_archive: gpdb_src_tinc_tarball
      installer_gpdb_rc: installer_rhel6_gpdb_rc
      qautils_tarball: qautils_rhel6_tarball
      gpdb_src_behave_tarball: gpdb_src_behave_tarball
    params: &pulse_properties
      PULSE_URL: {{pulse_url}}
      PULSE_PROJECT_NAME: "GPDB-gpexpand-parallel"
      PULSE_USERNAME: {{pulse_username}}
      PULSE_PASSWORD: {{pulse_password}}
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params: *pulse_properties

- name: DPM_gptransfer-43x-to-5x
  plan:
  - get: nightly-trigger
    trigger: {{nightly-trigger-flag}}
  - aggregate: *post_packaging_gets_trigger_based_on_flag
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "GPDB-behave_gptransfer_43x_to_5x"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "GPDB-behave_gptransfer_43x_to_5x"

- name: DPM_gptransfer-5x-to-5x
  plan:
  - aggregate: &post_packaging_gets_trigger_true
    - get: gpdb_src
      params: {submodules: none}
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
      trigger: true
    - get: gpdb_src_tinc_tarball
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
    - get: installer_rhel6_gpdb_rc
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
    - get: qautils_rhel6_tarball
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
    - get: gpdb_src_behave_tarball
      tags: ["gpdb5-pulse-worker"]
      passed: [gpdb_rc_packaging_centos]
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "GPDB-behave_gptransfer_5x_to_5x"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "GPDB-behave_gptransfer_5x_to_5x"


- name: cs-pg-two-phase
  plan:
  - aggregate: *post_packaging_gets_trigger_true
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-pg-two-phase"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-pg-two-phase"

- name: cs-filerep-schema-topology-crashrecov
  plan:
  - aggregate: *post_packaging_gets_trigger_true
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-filerep-schema-topology-crashrecov"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-filerep-schema-topology-crashrecov"

- name: cs-filerep-end-to-end
  plan:
  - get: nightly-trigger
    trigger: {{nightly-trigger-flag}}
  - aggregate: *post_packaging_gets_trigger_based_on_flag
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-filerep-end-to-end"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-filerep-end-to-end"

- name: cs-aoco-compression
  plan:
  - get: nightly-trigger
    trigger: {{nightly-trigger-flag}}
  - aggregate: *post_packaging_gets_trigger_based_on_flag
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-aoco-compression"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "cs-aoco-compression"

- name: mpp_interconnect
  plan:
  - aggregate: *post_packaging_gets_trigger_true
  - task: trigger_pulse
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/trigger_pulse.yml
    input_mapping: *input_mappings
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "mpp-interconnect"
  - task: monitor_pulse
    attempts: 2
    tags: ["gpdb5-pulse-worker"]
    file: gpdb_src/ci/pulse/api/monitor_pulse.yml
    params:
      <<: *pulse_properties
      PULSE_PROJECT_NAME: "mpp-interconnect"

- name: QP_runaway-query
  plan:
  - aggregate:
    - get: gpdb_src
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: bin_gpdb
      passed: [compile_gpdb_centos6]
      resource: bin_gpdb_centos6
    - get: centos-gpdb-dev-6
  - task: runaway-query
    timeout: 3h
    file: gpdb_src/concourse/tasks/tinc_gpdb.yml
    image: centos-gpdb-dev-6
    params:
      MAKE_TEST_COMMAND: runaway_query
      TEST_OS: "centos"
      CONFIGURE_FLAGS: {{configure_flags}}

- name: QP_optimizer-functional
  plan:
  - aggregate:
    - get: gpdb_src
      params: {submodules: none}
      passed: [compile_gpdb_centos6]
    - get: bin_gpdb
      resource: bin_gpdb_centos6
      passed: [compile_gpdb_centos6]
      trigger: true
    - get: centos-gpdb-dev-6
  - aggregate:
    - task: optimizer_functional_part1
      timeout: 3h
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        MAKE_TEST_COMMAND: optimizer_functional_part1
        BLDWRAP_POSTGRES_CONF_ADDONS: fsync=off optimizer_print_missing_stats=off
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}
    - task: optimizer_functional_part2
      timeout: 3h
      file: gpdb_src/concourse/tasks/tinc_gpdb.yml
      image: centos-gpdb-dev-6
      params:
        MAKE_TEST_COMMAND: optimizer_functional_part2
        BLDWRAP_POSTGRES_CONF_ADDONS: fsync=off optimizer_print_missing_stats=off
        TEST_OS: centos
        CONFIGURE_FLAGS: {{configure_flags}}

- name: validate_pipeline
  plan:
  - aggregate:
    - get: gpdb_src
      trigger: true
    - get: centos-gpdb-dev-6
  - task: validate_pipeline
    file: gpdb_src/concourse/tasks/validate_pipeline.yml
    image: centos-gpdb-dev-6

- name: Release_Candidate
  plan:
  - get: gpdb_src
    trigger: true
    passed:
    - compile_gpdb_centos6
    - compile_gpdb_open_source_centos6
    - compile_gpdb_centos7
    - compile_gpdb_sles11
    - compile_gpdb_ubuntu16
    - compile_gpdb_windows_cl
    - compile_gpdb_aix7_remote
    - icw_planner_centos6
    - icw_gporca_centos6
    - icw_gporca_centos7
    - icw_gporca_sles11
    - icw_planner_ictcp_centos6
    - client_loader_remote_test_aix
    - mpp_resource_group_centos6
    - MU_check_centos
    - MM_analyzedb
    - MM_gpinitsystem
    - MM_gpperfmon
    - MM_gpcheck
    - DPM_backup-restore
    - MM_gppkg
    - MM_pt-rebuild
    - fts
    - storage
    - cs_walrep_1
    - cs_walrep_2
    - QP_memory-accounting
    - regression_tests_gpcloud_centos
    - regression_tests_gphdfs_centos
    - regression_tests_pxf_centos
    - gpdb_rc_packaging_centos
    - gpdb_rc_packaging_sles
    - DPM_backup_43_restore_5
    - MM_gpcheckcat
    - MM_gprecoverseg
    - MM_gpexpand
    - MM_gpinitstandby
    - DPM_gptransfer-43x-to-5x
    - DPM_gptransfer-5x-to-5x
    - cs-pg-two-phase
    - cs-filerep-schema-topology-crashrecov
    - cs-filerep-end-to-end
    - cs-aoco-compression
    - mpp_interconnect
    - QP_runaway-query
    - QP_optimizer-functional
    - validate_pipeline