{% set walrep_suites = ['walrep_2'] %} {% set CLI_1_suites = ['gprecoverseg', 'gpcheckcat', 'gpinitstandby', 'gpactivatestandby'] %} {% set CLI_2_suites = ['analyzedb', 'gpperfmon', 'gpinitsystem'] %} {% set CLI_gpexpand_suites = ['gpexpand_1', 'gpexpand_2'] %} ## ====================================================================== ## __ _ _ __ ___ _ _ _ __ ___ ## / _` | '__/ _ \| | | | '_ \/ __| ## | (_| | | | (_) | |_| | |_) \__ \ ## \__, |_| \___/ \__,_| .__/|___/ ## |___/ |_| ## ====================================================================== groups: - name: all jobs: {% if "centos6" in os_types %} - compile_gpdb_centos6 - compile_gpdb_binary_swap_centos6 - compile_gpdb_open_source_centos6 {% endif %} {% if "centos7" in os_types %} - compile_gpdb_centos7 {% endif %} {% if "sles" in os_types %} - compile_gpdb_sles11 {% endif %} {% if "ubuntu16" in os_types %} - compile_gpdb_ubuntu16 {% endif %} {% if "win" in os_types %} - compile_gpdb_windows_cl {% endif %} {% if "aix7" in os_types %} - compile_gpdb_aix7_remote {% endif %} {% if "aix7" in os_types %} - client_loader_remote_test_aix {% endif %} {% if "ICW" in test_sections %} ## -------------------------------------------------------------------- {% if "centos6" in os_types %} - icw_gporca_centos6 - icw_gporca_centos6_gpos_memory - icw_planner_centos6 - icw_planner_ictcp_centos6 - icw_extensions_gpcloud_centos6 {% endif %} {% if "centos7" in os_types %} - icw_gporca_centos7 - icw_planner_centos7 {% endif %} {% if "sles" in os_types %} - icw_gporca_sles11 - icw_gporca_sles12 - icw_planner_sles12 {% endif %} {% if "ubuntu16" in os_types %} - icw_planner_ubuntu16 - icw_gporca_conan_ubuntu16 - icw_extensions_gpcloud_ubuntu16 {% endif %} {% if "ubuntu16" in os_types %} - gpdb_packaging_ubuntu16 {% endif %} - gate_icw_end {% endif %} {% if "Replication" in test_sections %} ## -------------------------------------------------------------------- - gate_replication_start - walrep_2 - segwalrep_mirrorless_centos6 {% endif %} {% if "Interconnect" in test_sections %} ## -------------------------------------------------------------------- - interconnect {% endif %} {% if "ResourceGroups" in test_sections %} ## -------------------------------------------------------------------- - gate_resource_groups_start - resource_group_centos6 {% if "centos7" in os_types %} - resource_group_centos7 {% endif %} {% if "sles" in os_types %} - resource_group_sles12 {% endif %} {% endif %} {% if "CLI" in test_sections %} ## -------------------------------------------------------------------- - gate_cli_start {% for test_name in CLI_1_suites + CLI_2_suites + CLI_gpexpand_suites %} - [[ test_name ]] {% endfor %} - gpaddmirrors - gpexpand - gpcheck - gppkg - check_centos - gptransfer {% endif %} {% if "UD" in test_sections %} ## -------------------------------------------------------------------- - gate_ud_start - regression_tests_pxf - regression_tests_gphdfs_hadoop_centos - regression_tests_gphdfs_mapr_centos {% endif %} {% if "AA" in test_sections %} ## -------------------------------------------------------------------- - gate_advanced_analytics_start {% if "centos6" in os_types %} - MADlib_Test_orca_centos6 - MADlib_Test_planner_centos6 {% endif %} {% if "centos7" in os_types %} - MADlib_Test_orca_centos7 - MADlib_Test_planner_centos7 {% endif %} {% endif %} {% if pipeline_type == "prod" %} ## ====================================================================== - name: Release jobs: - gate_release_candidate_start - Release_Candidate {% endif %} ## ====================================================================== - name: Compile jobs: {% if "centos6" in os_types %} - compile_gpdb_centos6 - compile_gpdb_binary_swap_centos6 - compile_gpdb_open_source_centos6 {% endif %} {% if "centos7" in os_types %} - compile_gpdb_centos7 {% endif %} {% if "sles" in os_types %} - compile_gpdb_sles11 {% endif %} {% if "ubuntu16" in os_types %} - compile_gpdb_ubuntu16 {% endif %} {% if "win" in os_types %} - compile_gpdb_windows_cl {% endif %} {% if "aix7" in os_types %} - compile_gpdb_aix7_remote {% endif %} ## ====================================================================== - name: ClientsLoaders jobs: {% if "win" in os_types %} - compile_gpdb_windows_cl {% endif %} {% if "aix7" in os_types %} - compile_gpdb_aix7_remote - client_loader_remote_test_aix {% endif %} {% if "ICW" in test_sections %} ## ====================================================================== - name: ICW jobs: {% if "centos6" in os_types %} - icw_gporca_centos6 - icw_gporca_centos6_gpos_memory - icw_planner_centos6 - icw_planner_ictcp_centos6 - icw_extensions_gpcloud_centos6 - compile_gpdb_centos6 {% endif %} {% if "centos7" in os_types %} - icw_gporca_centos7 - icw_planner_centos7 - compile_gpdb_centos7 {% endif %} {% if "sles" in os_types %} - icw_gporca_sles11 - icw_gporca_sles12 - icw_planner_sles12 - compile_gpdb_sles11 {% endif %} {% if "ubuntu16" in os_types %} - icw_planner_ubuntu16 - icw_gporca_conan_ubuntu16 - compile_gpdb_ubuntu16 - gpdb_packaging_ubuntu16 - icw_extensions_gpcloud_ubuntu16 - compile_gpdb_ubuntu16 {% endif %} - gate_icw_end {% endif %} {% if "Replication" in test_sections %} ## ====================================================================== - name: Replication jobs: - gate_replication_start - walrep_2 - segwalrep_mirrorless_centos6 - compile_gpdb_centos6 {% endif %} {% if "Interconnect" in test_sections %} ## ====================================================================== {% if "centos6" in os_types %} - name: Interconnect jobs: - compile_gpdb_centos6 - interconnect {% endif %} {% endif %} {% if "ResourceGroups" in test_sections %} ## ====================================================================== - name: ResourceGroups jobs: - gate_resource_groups_start - resource_group_centos6 - compile_gpdb_centos6 {% if "centos7" in os_types %} - resource_group_centos7 - compile_gpdb_centos7 {% endif %} {% if "sles" in os_types %} - resource_group_sles12 - compile_gpdb_sles11 {% endif %} {% endif %} {% if "CLI" in test_sections %} ## ====================================================================== - name: CLI jobs: - gate_cli_start {% for test_name in CLI_1_suites + CLI_2_suites + CLI_gpexpand_suites %} - [[ test_name ]] {% endfor %} - gpaddmirrors - gpexpand - gpcheck - gppkg - check_centos - compile_gpdb_centos6 - gptransfer - compile_gpdb_centos6 {% endif %} {% if "UD" in test_sections %} ## ====================================================================== - name: UD jobs: - gate_ud_start - regression_tests_pxf - regression_tests_gphdfs_hadoop_centos - regression_tests_gphdfs_mapr_centos - compile_gpdb_centos6 {% endif %} {% if "AA" in test_sections %} - name: AdvancedAnalytics jobs: - gate_advanced_analytics_start {% if "centos6" in os_types %} - MADlib_Test_orca_centos6 - MADlib_Test_planner_centos6 {% endif %} {% if "centos7" in os_types %} - MADlib_Test_orca_centos7 - MADlib_Test_planner_centos7 {% endif %} {% endif %} ## ====================================================================== ## _ ## _ __ ___ ___ ___ _ _ _ __ ___ ___ | |_ _ _ _ __ ___ ___ ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \ | __| | | | '_ \ / _ \/ __| ## | | | __/\__ \ (_) | |_| | | | (_| __/ | |_| |_| | |_) | __/\__ \ ## |_| \___||___/\___/ \__,_|_| \___\___| \__|\__, | .__/ \___||___/ ## |___/|_| ## ====================================================================== resource_types: - name: terraform type: docker-image source: repository: ljfranklin/terraform-resource ## ====================================================================== ## _ __ ___ ___ ___ _ _ _ __ ___ ___ ___ ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __| ## | | | __/\__ \ (_) | |_| | | | (_| __/\__ \ ## |_| \___||___/\___/ \__,_|_| \___\___||___/ ## ====================================================================== resources: {% if ("ICW" in test_sections and "sles" in os_types) or "Replication" in test_sections or "ResourceGroups" in test_sections or "Interconnect" in test_sections or "CLI" in test_sections or "UD" in test_sections %} - name: ccp_src type: git source: branch: {{ccp-git-branch}} private_key: {{ccp-git-key}} uri: {{ccp-git-remote}} {% if "centos6" in os_types or "ICW" in test_sections %} - name: terraform type: terraform source: env: AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} GOOGLE_CREDENTIALS: {{google-service-account-key}} vars: project_id: {{google-project-id}} storage: access_key_id: {{tf-machine-access-key-id}} secret_access_key: {{tf-machine-secret-access-key}} region_name: {{aws-region}} # This is not parameterized, on purpose. All tfstates will go to this spot, # and different teams will place there clusters' tfstate files under different paths bucket: gpdb5-pipeline-dynamic-terraform bucket_path: clusters-google/ {% endif %} {% endif %} {% if "CLI" in test_sections %} # a second cluster for gptransfer tests - name: terraform2 type: terraform source: env: AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} GOOGLE_CREDENTIALS: {{google-service-account-key}} vars: project_id: {{google-project-id}} storage: access_key_id: {{tf-machine-access-key-id}} secret_access_key: {{tf-machine-secret-access-key}} region_name: {{aws-region}} bucket: gpdb5-pipeline-dynamic-terraform bucket_path: clusters-google/ {% endif %} {% if "aix7" in os_types %} - name: aix_environments type: pool source: uri: {{concourse-resource-pools-git-remote}} branch: master pool: client_loader_remote_aix private_key: {{concourse-resource-pools-git-key}} {% endif %} - name: gpdb_src type: git source: branch: {{gpdb-git-branch}} uri: {{gpdb-git-remote}} ignore_paths: - gpdb-doc/* - README* {% if "centos6" in os_types %} - name: gpdb_src_binary_swap type: git source: branch: {{gpdb-binary-swap-git-tag}} uri: {{gpdb-binary-swap-git-remote}} ignore_paths: - gpdb-doc/* - README* {% endif %} - name: gpaddon_src type: git source: branch: {{gpaddon-git-branch}} private_key: {{gpaddon-git-key}} uri: {{gpaddon-git-remote}} {% if "UD" in test_sections %} - name: pxf_infra_src type: git source: branch: {{pxf_infra-git-branch}} private_key: {{pxf_infra-git-key}} uri: {{pxf_infra-git-remote}} - name: centos6-gpdb-dev-pxf-hdp type: docker-image source: repository: pivotaldata/gpdb-dev tag: centos6-pxf-hdp-server {% endif %} {% if "ubuntu16" in os_types %} - name: debian_release type: git source: branch: ((debian-release-git-branch)) uri: ((debian-release-git-remote)) {% endif %} {% if "centos6" in os_types or "sles" in os_types or "aix7" in os_types %} - name: centos-gpdb-dev-6 type: docker-image source: repository: pivotaldata/centos-gpdb-dev tag: '6-gcc6.2-llvm3.7' {% endif %} {% if "centos7" in os_types %} - name: centos-gpdb-build-7 type: docker-image source: repository: pivotaldata/centos7-build tag: 'gpdb5-latest' - name: centos-gpdb-test-7 type: docker-image source: repository: pivotaldata/centos7-test tag: 'gpdb5-latest' {% endif %} {% if "ubuntu16" in os_types %} - name: ubuntu-gpdb-dev-16 type: docker-image source: repository: pivotaldata/ubuntu-gpdb-dev tag: '16.04_gcc_6_3' - name: ubuntu-gpdb-debian-dev-16 type: docker-image source: repository: pivotaldata/ubuntu-gpdb-debian-dev tag: '16.04' {% endif %} {% if "centos6" in os_types %} - name: bin_gpdb_centos6 type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: {{bin_gpdb_centos_versioned_file}} {% endif %} {% if pipeline_type == "prod" %} - name: bin_gpdb_centos6_icw_green type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: bin_gpdb_centos6/gpdb_branch_master/icw_green/bin_gpdb.tar.gz - name: bin_gpdb_centos6_rc type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{gpdb-stable-builds-bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: release_candidates/bin_gpdb_centos6/gpdb6/((rc-build-type))/bin_gpdb.tar.gz - name: bin_gpdb_centos7_rc type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{gpdb-stable-builds-bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: release_candidates/bin_gpdb_centos7/gpdb6/((rc-build-type))/bin_gpdb.tar.gz - name: bin_gpdb_sles11_rc type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{gpdb-stable-builds-bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: release_candidates/bin_gpdb_sles11/gpdb6/((rc-build-type))/bin_gpdb.tar.gz {% endif %} {% if "UD" in test_sections %} - name: pxf_tarball type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{pxf-aws-bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: pxf_artifacts/prod/gpdb_branch_master/stable/pxf.tar.gz {% endif %} {% if "ubuntu16" in os_types %} - name: compiled_bits_ubuntu16 type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: {{compiled_bits_ubuntu16_versioned_file}} {% endif %} {% if "centos6" in os_types %} - name: binary_swap_gpdb_centos6 type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: {{binary_swap_gpdb_centos_versioned_file}} {% endif %} {% if "centos7" in os_types %} - name: bin_gpdb_centos7 type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: {{bin_gpdb_centos7_versioned_file}} {% endif %} {% if "sles" in os_types %} - name: bin_gpdb_sles11 type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: {{bin_gpdb_sles11_versioned_file}} {% endif %} {% if "win" in os_types %} - name: centos-mingw type: docker-image source: repository: pivotaldata/centos-mingw - name: bin_gpdb_windows_clients type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} regexp: windows-cl/greenplum-clients-(.*)-WinXP-x86_32.msi - name: bin_gpdb_windows_loaders type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} regexp: windows-cl/greenplum-loaders-(.*)-WinXP-x86_32.msi {% endif %} {% if "aix7" in os_types %} - name: installer_aix7_gpdb_clients type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} regexp: deliverables/greenplum-clients-(.*)-aix7_ppc_64.zip - name: installer_aix7_gpdb_loaders type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} regexp: deliverables/greenplum-loaders-(.*)-aix7_ppc_64.zip {% endif %} {% if "ubuntu16" in os_types %} - name: deb_package_ubuntu16 type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{bucket-name}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} versioned_file: {{deb_package_ubuntu16_versioned_file}} {% endif %} {% if "aix7" in os_types %} - name: nightly-trigger type: time source: location: America/Los_Angeles days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday] start: 6:00 AM stop: 7:00 AM {% endif %} {% if "AA" in test_sections %} - name: madlib_ci type: git source: branch: {{madlib-ci-git-branch}} uri: {{madlib-ci-git-remote}} private_key: {{madlib-ci-git-key}} - name: madlib_gppkg type: s3 source: access_key_id: {{bucket-access-key-id}} bucket: {{madlib-gppkg-bucket}} region_name: {{aws-region}} secret_access_key: {{bucket-secret-access-key}} regexp: {{madlib-gpdb6-centos6-gppkg}} {% endif %} - name: reduced-frequency-trigger type: time source: location: America/Los_Angeles days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday, Saturday] start: {{reduced-frequency-trigger-start}} stop: {{reduced-frequency-trigger-stop}} ## ====================================================================== ## _ ## __ _ _ __ ___| |__ ___ _ __ ___ ## / _` | '_ \ / __| '_ \ / _ \| '__/ __| ## | (_| | | | | (__| | | | (_) | | \__ \ ## \__,_|_| |_|\___|_| |_|\___/|_| |___/ ## ====================================================================== anchors: - &ccp_default_params action: create delete_on_failure: true generate_random_name: true terraform_source: ccp_src/google/ - &ccp_default_vars instance_type: n1-standard-1 PLATFORM: centos6 - &ccp_destroy put: terraform params: action: destroy env_name_file: terraform/name terraform_source: ccp_src/google/ vars: aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. aws_ebs_volume_type: standard get_params: action: destroy - &ccp_destroy_two_clusters do: - *ccp_destroy - put: terraform2 params: action: destroy env_name_file: terraform2/name terraform_source: ccp_src/google/ vars: PLATFORM: centos6 aws_instance-node-instance_type: t2.micro cluster_suffix: "-2" get_params: action: destroy - &ccp_gen_cluster_default_params AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} AWS_DEFAULT_REGION: {{aws-region}} BUCKET_PATH: clusters-google/ BUCKET_NAME: {{tf-bucket-name}} CLOUD_PROVIDER: google - &set_failed do: - task: on_failure_set_failed config: platform: linux image_resource: type: docker-image source: repository: pivotaldata/ccp tag: "7" inputs: - name: ccp_src - name: terraform run: path: 'ccp_src/google/ccp_failed_test.sh' params: GOOGLE_CREDENTIALS: {{google-service-account-key}} GOOGLE_PROJECT_ID: {{google-project-id}} GOOGLE_ZONE: {{google-zone}} GOOGLE_SERVICE_ACCOUNT: {{google-service-account}} AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} AWS_DEFAULT_REGION: {{tf-machine-region}} BUCKET_PATH: clusters-google/ BUCKET_NAME: {{tf-bucket-name}} - &set_failed_two_clusters do: - *set_failed - task: on_failure_set_failed config: platform: linux image_resource: type: docker-image source: repository: pivotaldata/ccp tag: "7" inputs: - name: ccp_src - name: terraform2 run: path: 'ccp_src/google/ccp_failed_test.sh' params: GOOGLE_CREDENTIALS: {{google-service-account-key}} GOOGLE_PROJECT_ID: {{google-project-id}} GOOGLE_ZONE: {{google-zone}} GOOGLE_SERVICE_ACCOUNT: {{google-service-account}} AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} AWS_DEFAULT_REGION: {{tf-machine-region}} BUCKET_PATH: clusters-google/ BUCKET_NAME: {{tf-bucket-name}} replication_ccp_gen_cluster_params_anchor: &replication_ccp_gen_cluster_params file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params replication_ccp_run_tests_params_anchor: &replication_ccp_run_tests_params file: gpdb_src/concourse/tasks/run_tinc.yml image: centos-gpdb-dev-6 on_success: <<: *ccp_destroy ## ====================================================================== ## _ _ ## (_) ___ | |__ ___ ## | |/ _ \| '_ \/ __| ## | | (_) | |_) \__ \ ## _/ |\___/|_.__/|___/ ## |__/ ## ====================================================================== jobs: ## ====================================================================== ## ____ _ _ ## / ___|___ _ __ ___ _ __ (_) | ___ ## | | / _ \| '_ ` _ \| '_ \| | |/ _ \ ## | |__| (_) | | | | | | |_) | | | __/ ## \____\___/|_| |_| |_| .__/|_|_|\___| ## |_| ## ====================================================================== {% if "centos6" in os_types %} - name: compile_gpdb_centos6 plan: - aggregate: - get: gpdb_src trigger: ((gpdb_src-trigger-flag)) - get: reduced-frequency-trigger trigger: ((reduced-frequency-trigger-flag)) - get: gpaddon_src - get: centos-gpdb-dev-6 - task: sync_tools file: gpdb_src/concourse/tasks/sync_tools.yml image: centos-gpdb-dev-6 params: IVYREPO_HOST: {{ivyrepo_host}} IVYREPO_REALM: {{ivyrepo_realm}} IVYREPO_USER: {{ivyrepo_user}} IVYREPO_PASSWD: {{ivyrepo_passwd}} TARGET_OS: centos TARGET_OS_VERSION: 6 TASK_OS: centos TASK_OS_VERSION: 6 - task: compile_gpdb file: gpdb_src/concourse/tasks/compile_gpdb.yml image: centos-gpdb-dev-6 params: CONFIGURE_FLAGS: {{configure_flags}} TARGET_OS: centos TARGET_OS_VERSION: 6 BLD_TARGETS: "clients loaders" - aggregate: - put: bin_gpdb_centos6 params: file: gpdb_artifacts/bin_gpdb.tar.gz {% endif %} {% if "centos7" in os_types %} - name: compile_gpdb_centos7 plan: - aggregate: - get: reduced-frequency-trigger trigger: ((reduced-frequency-trigger-flag)) - get: gpdb_src trigger: ((gpdb_src-trigger-flag)) - get: gpaddon_src - get: centos-gpdb-build-7 - task: sync_tools file: gpdb_src/concourse/tasks/sync_tools.yml image: centos-gpdb-build-7 params: IVYREPO_HOST: {{ivyrepo_host}} IVYREPO_REALM: {{ivyrepo_realm}} IVYREPO_USER: {{ivyrepo_user}} IVYREPO_PASSWD: {{ivyrepo_passwd}} TARGET_OS: centos TARGET_OS_VERSION: 7 TASK_OS: centos TASK_OS_VERSION: 7 - task: compile_gpdb image: centos-gpdb-build-7 file: gpdb_src/concourse/tasks/compile_gpdb.yml params: CONFIGURE_FLAGS: {{configure_flags}} TARGET_OS: centos TARGET_OS_VERSION: 7 BLD_TARGETS: "clients loaders" - aggregate: - put: bin_gpdb_centos7 params: file: gpdb_artifacts/bin_gpdb.tar.gz {% endif %} {% if "sles" in os_types %} - name: compile_gpdb_sles11 plan: - aggregate: - get: reduced-frequency-trigger trigger: ((reduced-frequency-trigger-flag)) - get: gpdb_src trigger: ((gpdb_src-trigger-flag)) - get: gpaddon_src - get: centos-gpdb-build-7 - task: sync_tools file: gpdb_src/concourse/tasks/sync_tools.yml image: centos-gpdb-build-7 params: IVYREPO_HOST: {{ivyrepo_host}} IVYREPO_REALM: {{ivyrepo_realm}} IVYREPO_USER: {{ivyrepo_user}} IVYREPO_PASSWD: {{ivyrepo_passwd}} TARGET_OS: sles TARGET_OS_VERSION: TASK_OS: centos TASK_OS_VERSION: 7 - task: compile_gpdb file: gpdb_src/concourse/tasks/compile_gpdb_sles11.yml params: CONFIGURE_FLAGS: {{configure_flags}} TARGET_OS: sles TARGET_OS_VERSION: BLD_TARGETS: "clients loaders" - put: bin_gpdb_sles11 params: file: gpdb_artifacts/bin_gpdb.tar.gz {% endif %} {% if "ubuntu16" in os_types %} - name: compile_gpdb_ubuntu16 plan: - aggregate: - get: reduced-frequency-trigger trigger: ((reduced-frequency-trigger-flag)) - get: gpdb_src trigger: ((gpdb_src-trigger-flag)) - get: ubuntu-gpdb-dev-16 - get: ubuntu-gpdb-debian-dev-16 - get: debian_release - aggregate: - task: compile_gpdb image: ubuntu-gpdb-dev-16 file: gpdb_src/concourse/tasks/compile_gpdb_open_source_ubuntu.yml params: CONFIGURE_FLAGS: {{configure_flags}} - task: deb_create_package file: gpdb_src/concourse/tasks/deb_create_package.yml image: ubuntu-gpdb-debian-dev-16 params: DEBFULLNAME: ((debian-package-maintainer-fullname)) DEBEMAIL: ((debian-package-maintainer-email)) - aggregate: - put: compiled_bits_ubuntu16 params: file: compiled_bits_ubuntu16/compiled_bits_ubuntu16.tar.gz - put: deb_package_ubuntu16 params: file: {{deb_package_ubuntu16_versioned_file}} {% endif %} {% if "centos6" in os_types %} - name: compile_gpdb_open_source_centos6 public: true plan: - aggregate: - get: reduced-frequency-trigger trigger: ((reduced-frequency-trigger-flag)) - get: gpdb_src trigger: ((gpdb_src-trigger-flag)) - get: centos-gpdb-dev-6 - task: compile_gpdb image: centos-gpdb-dev-6 file: gpdb_src/concourse/tasks/compile_gpdb_open_source_centos.yml params: CONFIGURE_FLAGS: {{configure_flags}} {% endif %} {% if "centos6" in os_types %} - name: compile_gpdb_binary_swap_centos6 plan: # This acts like a cache as this job will only be run once to get a # binary to use for our binary swap compatibility tests. Setting a new # tag or branch for the gpdb_src_binary_swap resource via set-pipeline # will replace the cached binary. - aggregate: - get: gpdb_src resource: gpdb_src_binary_swap trigger: true - get: gpaddon_src - get: centos-gpdb-dev-6 - task: compile_gpdb file: gpdb_src/concourse/tasks/compile_gpdb.yml image: centos-gpdb-dev-6 params: IVYREPO_HOST: {{ivyrepo_host}} IVYREPO_REALM: {{ivyrepo_realm}} IVYREPO_USER: {{ivyrepo_user}} IVYREPO_PASSWD: {{ivyrepo_passwd}} TARGET_OS: centos TARGET_OS_VERSION: 6 - aggregate: - put: binary_swap_gpdb_centos6 params: file: gpdb_artifacts/bin_gpdb.tar.gz {% endif %} {% if "win" in os_types %} - name: compile_gpdb_windows_cl plan: - aggregate: - get: reduced-frequency-trigger trigger: ((reduced-frequency-trigger-flag)) tags: ["wix"] - get: gpdb_src trigger: ((gpdb_src-trigger-flag)) tags: ["wix"] - get: gpaddon_src tags: ["wix"] - get: centos-mingw tags: ["wix"] - get: centos-gpdb-build-7 tags: ["wix"] - task: sync_tools tags: ["wix"] file: gpdb_src/concourse/tasks/sync_tools.yml image: centos-gpdb-build-7 params: IVYREPO_HOST: {{ivyrepo_host}} IVYREPO_REALM: {{ivyrepo_realm}} IVYREPO_USER: {{ivyrepo_user}} IVYREPO_PASSWD: {{ivyrepo_passwd}} TARGET_OS: win32 TARGET_OS_VERSION: TASK_OS: centos TASK_OS_VERSION: 7 - task: compile_gpdb tags: ["wix"] file: gpdb_src/concourse/tasks/compile_gpdb.yml image: centos-mingw params: CONFIGURE_FLAGS: {{configure_flags}} TARGET_OS: win32 TARGET_OS_VERSION: BLD_TARGETS: "clients loaders" - aggregate: - put: bin_gpdb_windows_clients tags: ["wix"] params: file: gpdb_artifacts/greenplum-clients-*-WinXP-x86_32.msi - put: bin_gpdb_windows_loaders tags: ["wix"] params: file: gpdb_artifacts/greenplum-loaders-*-WinXP-x86_32.msi {% endif %} {% if "aix7" in os_types %} - name: compile_gpdb_aix7_remote serial: true plan: # Compile gpdb on a remote AIX machine, triggered by concourse. # We need to serialize this job to avoid overwhelming workload # on remote machine. - aggregate: - get: nightly-trigger trigger: true - get: gpdb_src - get: gpaddon_src - get: centos-gpdb-dev-6 - task: compile_gpdb_aix7_remote file: gpdb_src/concourse/tasks/compile_gpdb_remote.yml image: centos-gpdb-dev-6 params: REMOTE_HOST: {{remote_host_build}} REMOTE_PORT: {{remote_port_build}} REMOTE_USER: {{remote_user_build}} REMOTE_KEY: {{remote_key_build}} IVYREPO_HOST: {{ivyrepo_host}} IVYREPO_REALM: {{ivyrepo_realm}} IVYREPO_USER: {{ivyrepo_user}} IVYREPO_PASSWD: {{ivyrepo_passwd}} BLD_TARGETS: "clients loaders" - aggregate: - put: installer_aix7_gpdb_clients params: file: gpdb_artifacts/greenplum-clients-*-aix7_ppc_64.zip - put: installer_aix7_gpdb_loaders params: file: gpdb_artifacts/greenplum-loaders-*-aix7_ppc_64.zip - name: client_loader_remote_test_aix serial: true plan: - aggregate: - get: gpdb_src passed: - compile_gpdb_aix7_remote - compile_gpdb_centos6 - get: installer_aix7_gpdb_clients passed: - compile_gpdb_aix7_remote - get: installer_aix7_gpdb_loaders passed: - compile_gpdb_aix7_remote trigger: true - get: bin_gpdb passed: - compile_gpdb_centos6 resource: bin_gpdb_centos6 - get: centos-gpdb-dev-6 - put: aix_environments params: {acquire: true} - task: ic_gpdb_cl file: gpdb_src/concourse/tasks/ic_gpdb_remote.yml image: centos-gpdb-dev-6 params: BLD_TARGETS: "clients loaders" REMOTE_HOST: {{remote_host_test}} REMOTE_PORT: {{remote_port_test}} REMOTE_USER: {{remote_user_test}} REMOTE_KEY: {{remote_key_test}} ensure: do: - task: cleanup_aix file: gpdb_src/concourse/tasks/aix_remote_cleanup.yml image: centos-gpdb-dev-6 params: REMOTE_HOST: {{remote_host_test}} REMOTE_PORT: {{remote_port_test}} REMOTE_USER: {{remote_user_test}} REMOTE_KEY: {{remote_key_test}} - put: aix_environments params: {release: aix_environments} {% endif %} {% if "ICW" in test_sections %} ## ====================================================================== ## ___ ______ __ ## |_ _/ ___\ \ / / ## | | | \ \ /\ / / ## | | |___ \ V V / ## |___\____| \_/\_/ ## ====================================================================== {% if "centos6" in os_types %} - name: icw_planner_centos6 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos6] - get: bin_gpdb passed: [compile_gpdb_centos6] resource: bin_gpdb_centos6 trigger: [[ test_trigger ]] - get: binary_swap_gpdb passed: [compile_gpdb_binary_swap_centos6] resource: binary_swap_gpdb_centos6 trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world TEST_OS: centos TEST_BINARY_SWAP: false CONFIGURE_FLAGS: {{configure_flags}} - name: icw_gporca_centos6 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos6] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [compile_gpdb_centos6] trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on' installcheck-world TEST_OS: centos CONFIGURE_FLAGS: {{configure_flags}} - name: icw_gporca_centos6_gpos_memory plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos6] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [compile_gpdb_centos6] trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on' installcheck-world BLDWRAP_POSTGRES_CONF_ADDONS: optimizer_use_gpdb_allocators=on TEST_OS: centos CONFIGURE_FLAGS: {{configure_flags}} - name: icw_planner_ictcp_centos6 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos6] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [compile_gpdb_centos6] trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=tcp -c optimizer=off' installcheck-world TEST_OS: centos {% endif %} {% if "centos7" in os_types %} - name: icw_gporca_centos7 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos7] - get: bin_gpdb resource: bin_gpdb_centos7 passed: [compile_gpdb_centos7] trigger: [[ test_trigger ]] - get: centos-gpdb-test-7 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-test-7 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on' installcheck-world TEST_OS: centos CONFIGURE_FLAGS: {{configure_flags}} - name: icw_planner_centos7 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos7] - get: bin_gpdb passed: [compile_gpdb_centos7] resource: bin_gpdb_centos7 trigger: [[ test_trigger ]] - get: centos-gpdb-test-7 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-test-7 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world TEST_OS: centos TEST_BINARY_SWAP: false CONFIGURE_FLAGS: {{configure_flags}} {% endif %} {% if "sles" in os_types %} - name: icw_gporca_sles11 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_sles11] - get: bin_gpdb resource: bin_gpdb_sles11 passed: [compile_gpdb_sles11] trigger: [[ test_trigger ]] - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb_sles11.yml params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on' installcheck-world TEST_OS: sles CONFIGURE_FLAGS: {{configure_flags}} - name: icw_gporca_sles12 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_sles11] - get: gpdb_binary resource: bin_gpdb_sles11 passed: [compile_gpdb_sles11] trigger: true - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars PLATFORM: sles12 disk_size: 128 number_of_nodes: 1 default_image_user: root instance_type: n1-standard-4 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params # PLATFORM is necessary to generate correct ssh config PLATFORM: sles12 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_sles12.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on' installcheck-world TEST_OS: sles12 on_success: <<: *ccp_destroy ensure: <<: *set_failed - name: icw_planner_sles12 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_sles11] - get: gpdb_binary resource: bin_gpdb_sles11 passed: [compile_gpdb_sles11] trigger: true - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars PLATFORM: sles12 disk_size: 128 number_of_nodes: 1 default_image_user: root instance_type: n1-standard-4 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params # PLATFORM is necessary to generate correct ssh config PLATFORM: sles12 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_sles12.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world TEST_OS: sles12 on_success: <<: *ccp_destroy ensure: <<: *set_failed {% endif %} {% if "ubuntu16" in os_types %} - name: icw_planner_ubuntu16 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_ubuntu16] - get: compiled_bits_ubuntu16 passed: [compile_gpdb_ubuntu16] trigger: [[ test_trigger ]] - get: ubuntu-gpdb-dev-16 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb_ubuntu.yml image: ubuntu-gpdb-dev-16 params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world CONFIGURE_FLAGS: {{configure_flags}} - name: icw_gporca_conan_ubuntu16 plan: - aggregate: - get: gpdb_src params: submodules: none passed: [compile_gpdb_ubuntu16] - get: bin_gpdb passed: [compile_gpdb_ubuntu16] resource: compiled_bits_ubuntu16 trigger: [[ test_trigger ]] - get: ubuntu-gpdb-dev-16 - task: icw_with_orca image: ubuntu-gpdb-dev-16 file: gpdb_src/concourse/tasks/test_with_orca_conan.yml params: TEST_SUITE: "icw" - name: gpdb_packaging_ubuntu16 plan: - aggregate: - get: gpdb_src passed: - compile_gpdb_ubuntu16 - get: ubuntu-gpdb-debian-dev-16 - get: deb_package_ubuntu16 passed: - compile_gpdb_ubuntu16 trigger: [[ test_trigger ]] - task: test_gpdb_deb_package file: gpdb_src/concourse/tasks/deb_test_package.yml image: ubuntu-gpdb-debian-dev-16 params: DEBIAN_PACKAGE: {{deb_package_ubuntu16_versioned_file}} {% endif %} {% if 'centos6' in os_types%} - name: icw_extensions_gpcloud_centos6 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos6] trigger: [[ test_trigger ]] - get: bin_gpdb_centos6 passed: [compile_gpdb_centos6] - get: centos-gpdb-dev-6 - aggregate: - task: unit_tests_gpcloud file: gpdb_src/concourse/tasks/unit_tests_gpcloud.yml image: centos-gpdb-dev-6 params: TARGET_OS: centos - task: regression_tests_gpcloud_centos input_mapping: bin_gpdb: bin_gpdb_centos6 file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml image: centos-gpdb-dev-6 params: gpcloud_access_key_id: {{gpcloud-access-key-id}} gpcloud_secret_access_key: {{gpcloud-secret-access-key}} TARGET_OS: centos - task: gpcheckcloud_tests_gpcloud_centos input_mapping: bin_gpdb: bin_gpdb_centos6 file: gpdb_src/concourse/tasks/gpcheckcloud_tests_gpcloud.yml image: centos-gpdb-dev-6 params: gpcloud_access_key_id: {{gpcloud-access-key-id}} gpcloud_secret_access_key: {{gpcloud-secret-access-key}} TARGET_OS: centos {% endif %} {% if 'ubuntu16' in os_types %} - name: icw_extensions_gpcloud_ubuntu16 plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_ubuntu16] trigger: [[ test_trigger ]] - get: compiled_bits_ubuntu16 passed: [compile_gpdb_ubuntu16] - get: ubuntu-gpdb-dev-16 - aggregate: - task: regression_tests_gpcloud_ubuntu input_mapping: bin_gpdb: compiled_bits_ubuntu16 file: gpdb_src/concourse/tasks/regression_tests_gpcloud.yml image: ubuntu-gpdb-dev-16 params: gpcloud_access_key_id: {{gpcloud-access-key-id}} gpcloud_secret_access_key: {{gpcloud-secret-access-key}} TARGET_OS: ubuntu {% endif %} - name: gate_icw_end plan: - aggregate: {% if "centos6" in os_types %} - get: bin_gpdb_centos6 passed: - icw_gporca_centos6 - icw_gporca_centos6_gpos_memory - icw_planner_centos6 - icw_planner_ictcp_centos6 - icw_extensions_gpcloud_centos6 {% endif %} - get: gpdb_src passed: {% if "centos6" in os_types %} - icw_gporca_centos6 - icw_gporca_centos6_gpos_memory - icw_planner_centos6 - icw_planner_ictcp_centos6 - icw_extensions_gpcloud_centos6 {% endif %} {% if 'ubuntu16' in os_types %} - icw_extensions_gpcloud_ubuntu16 {% endif %} {% if "centos7" in os_types %} - icw_gporca_centos7 {% endif %} {% if "ubuntu16" in os_types %} - icw_planner_ubuntu16 - icw_gporca_conan_ubuntu16 {% endif %} {% if "sles" in os_types %} - icw_gporca_sles11 - icw_gporca_sles12 - icw_planner_sles12 {% endif %} trigger: true {% if pipeline_type == "prod" and "centos6" in os_types %} - put: bin_gpdb_centos6_icw_green params: file: bin_gpdb_centos6/bin_gpdb.tar.gz {% endif %} {% endif %} {% if "Replication" in test_sections %} ## ====================================================================== ## ____ _ _ _ _ ## | _ \ ___ _ __ | (_) ___ __ _| |_(_) ___ _ __ ## | |_) / _ \ '_ \| | |/ __/ _` | __| |/ _ \| '_ \ ## | _ < __/ |_) | | | (_| (_| | |_| | (_) | | | | ## |_| \_\___| .__/|_|_|\___\__,_|\__|_|\___/|_| |_| ## |_| ## ====================================================================== - name: gate_replication_start plan: - aggregate: - get: gpdb_src passed: - compile_gpdb_centos6 trigger: true - get: bin_gpdb_centos6 passed: - compile_gpdb_centos6 {% for test_name in walrep_suites %} - name: [[ test_name ]] plan: - aggregate: - get: gpdb_src passed: [gate_replication_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_replication_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/run_tinc.yml image: centos-gpdb-dev-6 params: TINC_TARGET: [[ test_name ]] on_success: <<: *ccp_destroy ensure: <<: *set_failed {% endfor %} - name: segwalrep_mirrorless_centos6 plan: - aggregate: - get: gpdb_src passed: [gate_replication_start] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [gate_replication_start] trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: ic_gpdb file: gpdb_src/concourse/tasks/ic_gpdb.yml image: centos-gpdb-dev-6 params: MAKE_TEST_COMMAND: "-C src/test/regress && make -C src/test/walrep install installcheck" TEST_OS: centos CONFIGURE_FLAGS: {{configure_flags}} WITH_MIRRORS: false {% endif %} {% if "Interconnect" in test_sections %} ## ====================================================================== ## ___ _ _ _____ _____ ____ ____ ___ _ _ _ _ _____ ____ _____ ## |_ _| \ | |_ _| ____| _ \ / ___/ _ \| \ | | \ | | ____/ ___|_ _| ## | || \| | | | | _| | |_) | | | | | | \| | \| | _|| | | | ## | || |\ | | | | |___| _ <| |__| |_| | |\ | |\ | |__| |___ | | ## |___|_| \_| |_| |_____|_| \_\\____\___/|_| \_|_| \_|_____\____| |_| ## ====================================================================== {% if "centos6" in os_types %} - name: interconnect plan: - aggregate: - get: gpdb_src passed: [compile_gpdb_centos6] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [compile_gpdb_centos6] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars - task: gen_cluster <<: *replication_ccp_gen_cluster_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: pre_run_test_setup image: centos-gpdb-dev-6 config: platform: linux inputs: - name: ccp_src - name: cluster_env_files run: path: bash args: - -c - | set -ex ccp_src/scripts/setup_ssh_to_cluster.sh ssh -t centos@mdw << 'SSH_CMD' sudo yum --cacheonly list installed kernel-devel-$(uname -r) SSH_CMD - task: run_tests <<: *replication_ccp_run_tests_params params: TINC_TARGET: mpp_interconnect # ccp_destroy should be part of *replication_ccp_run_tests_params ensure: <<: *set_failed {% endif %} {% endif %} {% if "ResourceGroups" in test_sections %} ## ====================================================================== ## ____ ____ ## | _ \ ___ ___ ___ _ _ _ __ ___ ___ / ___|_ __ ___ _ _ _ __ ___ ## | |_) / _ \/ __|/ _ \| | | | '__/ __/ _ \ | _| '__/ _ \| | | | '_ \/ __| ## | _ < __/\__ \ (_) | |_| | | | (_| __/ |_| | | | (_) | |_| | |_) \__ \ ## |_| \_\___||___/\___/ \__,_|_| \___\___|\____|_| \___/ \__,_| .__/|___/ ## |_| ## ====================================================================== - name: gate_resource_groups_start plan: - aggregate: - get: gpdb_src passed: - compile_gpdb_centos6 {% if "sles" in os_types %} - compile_gpdb_sles11 {% endif %} {% if "centos7" in os_types %} - compile_gpdb_centos7 {% endif %} trigger: true - get: bin_gpdb_centos6 passed: - compile_gpdb_centos6 {% if "centos7" in os_types %} - get: bin_gpdb_centos7 passed: - compile_gpdb_centos7 {% endif %} {% if "sles" in os_types %} - get: bin_gpdb_sles11 passed: - compile_gpdb_sles11 {% endif %} {% if "centos6" in os_types or "centos7" in os_types %} - get: binary_swap_gpdb passed: [compile_gpdb_binary_swap_centos6] resource: binary_swap_gpdb_centos6 trigger: [[ test_trigger ]] {% endif %} - name: resource_group_centos6 plan: - aggregate: - get: gpdb_src passed: [gate_resource_groups_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_resource_groups_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - get: binary_swap_gpdb passed: [gate_resource_groups_start] resource: binary_swap_gpdb_centos6 trigger: [[ test_trigger ]] - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars instance_type: n1-standard-2 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: centos-gpdb-dev-6 params: TEST_OS: centos6 on_success: <<: *ccp_destroy ensure: <<: *set_failed {% if "centos7" in os_types %} - name: resource_group_centos7 plan: - aggregate: - get: gpdb_src passed: [gate_resource_groups_start] - get: gpdb_binary resource: bin_gpdb_centos7 passed: [gate_resource_groups_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - get: binary_swap_gpdb passed: [gate_resource_groups_start] resource: binary_swap_gpdb_centos6 trigger: [[ test_trigger ]] - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars PLATFORM: centos7 instance_type: n1-standard-2 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params PLATFORM: centos7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: centos-gpdb-dev-6 params: TEST_OS: centos7 on_success: <<: *ccp_destroy ensure: <<: *set_failed {% endif %} {% if "sles" in os_types %} - name: resource_group_sles12 plan: - aggregate: - get: gpdb_src passed: [gate_resource_groups_start] - get: gpdb_binary resource: bin_gpdb_sles11 passed: [gate_resource_groups_start] trigger: true - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars PLATFORM: sles12 instance_type: n1-standard-2 default_image_user: root - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params # PLATFORM is necessary to generate correct ssh config PLATFORM: sles12 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: centos-gpdb-dev-6 params: TEST_OS: sles12 on_success: <<: *ccp_destroy ensure: <<: *set_failed {% endif %} {% endif %} {% if "CLI" in test_sections %} ## ====================================================================== ## ____ _ ___ ## / ___| | |_ _| ## | | | | | | ## | |___| |___ | | ## \____|_____|___| ## ====================================================================== - name: gate_cli_start plan: - aggregate: - get: gpdb_src passed: - compile_gpdb_centos6 trigger: true - get: bin_gpdb_centos6 passed: - compile_gpdb_centos6 - name: gppkg plan: - aggregate: - get: gpdb_src params: submodules: - gpMgmt/bin/pythonSrc/ext passed: [gate_cli_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: setup_gppkg_second_install file: gpdb_src/concourse/tasks/gppkg_behave.yml params: SECOND_BINARY_INSTALL_LOCATION: /tmp/gppkg_migrate - task: run_tests file: gpdb_src/concourse/tasks/run_behave.yml image: centos-gpdb-dev-6 params: BEHAVE_FLAGS: --tags=gppkg on_success: <<: *ccp_destroy ensure: <<: *set_failed {% for test_name in CLI_1_suites %} - name: [[ test_name ]] plan: - aggregate: - get: gpdb_src params: submodules: - gpMgmt/bin/pythonSrc/ext passed: [gate_cli_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/run_behave.yml image: centos-gpdb-dev-6 params: BEHAVE_FLAGS: --tags=[[ test_name ]] on_success: <<: *ccp_destroy ensure: <<: *set_failed {% endfor %} - name: gpcheck plan: - aggregate: &gets_for_behave - get: gpdb_src params: submodules: - gpMgmt/bin/pythonSrc/ext passed: [gate_cli_start] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: gpcheck_as_gpadmin file: gpdb_src/concourse/tasks/behave_gpdb.yml image: centos-gpdb-dev-6 params: BEHAVE_TAGS: gpcheck_as_gpadmin GPCHECK_SETUP: true {% for test_name in CLI_2_suites %} - name: [[ test_name ]] plan: - aggregate: *gets_for_behave - task: [[ test_name ]] file: gpdb_src/concourse/tasks/behave_gpdb.yml image: centos-gpdb-dev-6 params: BEHAVE_TAGS: [[ test_name ]] {% endfor %} - name: check_centos plan: - aggregate: - get: gpdb_src passed: [gate_cli_start] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: centos-gpdb-dev-6 - task: check_centos file: gpdb_src/concourse/tasks/gpMgmt_check_gpdb.yml image: centos-gpdb-dev-6 params: TEST_OS: centos - name: gpaddmirrors plan: - aggregate: - get: gpdb_src params: submodules: - gpMgmt/bin/pythonSrc/ext passed: [gate_cli_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars number_of_nodes: 4 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: run_tests file: gpdb_src/concourse/tasks/run_behave.yml image: centos-gpdb-dev-6 params: BEHAVE_FLAGS: --tags=gpaddmirrors on_success: <<: *ccp_destroy ensure: <<: *set_failed - name: gpexpand plan: - aggregate: - get: gpdb_src params: submodules: - gpMgmt/bin/pythonSrc/ext passed: [gate_cli_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars number_of_nodes: 5 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: pre_run_test_setup file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml image: centos-gpdb-dev-6 - task: run_tests file: gpdb_src/concourse/tasks/run_behave.yml params: BEHAVE_FLAGS: --tags=gpexpand image: centos-gpdb-dev-6 on_success: <<: *ccp_destroy ensure: <<: *set_failed {% for test_name in CLI_gpexpand_suites %} - name: [[ test_name ]] plan: - aggregate: - get: gpdb_src params: submodules: - gpMgmt/bin/pythonSrc/ext passed: [gate_cli_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars number_of_nodes: 5 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: pre_run_test_setup image: centos-gpdb-dev-6 file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml - task: run_tests file: gpdb_src/concourse/tasks/run_tinc.yml image: centos-gpdb-dev-6 params: TINC_TARGET: [[ test_name ]] CUSTOM_ENV: HOST1=mdw HOST2=sdw1 HOST3=sdw2 HOST4=sdw3 HOST5=sdw4 on_success: <<: *ccp_destroy ensure: <<: *set_failed {% endfor %} - name: gptransfer plan: - aggregate: - get: gpdb_src passed: [gate_cli_start] - get: gpdb_binary resource: bin_gpdb_centos6 passed: [gate_cli_start] trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 # The separate clusters can be created in parallel with the aggregate and do blocks # The terraform put and gen cluster that correspond must still happen serially - aggregate: - do: - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars instance_type: n1-standard-2 - task: gen_cluster1 file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params - task: gpinitsystem config: platform: linux image_resource: type: docker-image source: repository: pivotaldata/ccp tag: "7" inputs: - name: terraform - name: ccp_src - name: cluster_env_files run: path: ccp_src/scripts/gpinitsystem.sh - do: - put: terraform2 params: <<: *ccp_default_params vars: <<: *ccp_default_vars instance_type: n1-standard-2 cluster_suffix: "-2" - task: gen_cluster2 file: ccp_src/ci/tasks/gen_cluster.yml params: <<: *ccp_gen_cluster_default_params input_mapping: terraform: terraform2 output_mapping: cluster_env_files: cluster_env_files2 - task: gpinitsystem config: platform: linux image_resource: type: docker-image source: repository: pivotaldata/ccp tag: "7" inputs: - name: terraform - name: ccp_src - name: cluster_env_files2 path: cluster_env_files run: path: ccp_src/scripts/gpinitsystem.sh - task: gptransfer_pre_test_setup config: platform: linux inputs: - name: cluster_env_files - name: cluster_env_files2 - name: ccp_src - name: gpdb_src image_resource: type: docker-image source: repository: pivotaldata/ccp tag: "7" run: path: sh args: - -exc - | source gpdb_src/concourse/scripts/transfer_utils.sh; setup_gptransfer - task: run_gptransfer_tests file: gpdb_src/concourse/tasks/run_behave.yml image: centos-gpdb-dev-6 params: BEHAVE_FLAGS: --tags=gptransfer --tags=-skip_source_5 CUSTOM_ENV: export GPTRANSFER_DEST_HOST=mdw; export GPTRANSFER_DEST_PORT=5432; export GPTRANSFER_DEST_USER=gpadmin; export GPTRANSFER_MAP_FILE=/tmp/source_map_file; export GPTRANSFER_SOURCE_HOST=mdw-2; export GPTRANSFER_SOURCE_PORT=5432; export GPTRANSFER_SOURCE_USER=gpadmin; on_success: <<: *ccp_destroy_two_clusters ensure: <<: *set_failed_two_clusters {% endif %} {% if "UD" in test_sections %} ## ====================================================================== ## _ _ ____ ## | | | | _ \ ## | | | | | | | ## | |_| | |_| | ## \___/|____/ ## ====================================================================== - name: gate_ud_start plan: - aggregate: - get: gpdb_src passed: - compile_gpdb_centos6 trigger: true - get: bin_gpdb_centos6 passed: - compile_gpdb_centos6 - name: regression_tests_pxf plan: - aggregate: - get: gpdb_src passed: [gate_ud_start] - get: bin_gpdb resource: bin_gpdb_centos6 passed: [gate_ud_start] trigger: [[ test_trigger ]] - get: pxf_tarball trigger: [[ test_trigger ]] - get: pxf_infra_src - get: centos6-gpdb-dev-pxf-hdp - task: test_pxf file: pxf_infra_src/concourse/test_pxf.yml image: centos6-gpdb-dev-pxf-hdp params: GROUP: smoke HADOOP_CLIENT: HDP TARGET_OS: centos TARGET_OS_VERSION: 6 - name: regression_tests_gphdfs_hadoop_centos plan: - aggregate: - get: gpdb_src passed: [gate_ud_start] - get: bin_gpdb passed: [gate_ud_start] trigger: [[ test_trigger ]] resource: bin_gpdb_centos6 - get: centos-gpdb-dev-6 - task: regression_tests_gphdfs file: gpdb_src/concourse/tasks/regression_tests_gphdfs.yml image: centos-gpdb-dev-6 params: TARGET_OS: centos TARGET_OS_VERSION: 6 - name: regression_tests_gphdfs_mapr_centos ensure: <<: *set_failed on_success: <<: *ccp_destroy plan: - aggregate: - get: gpdb_src passed: [gate_ud_start] - get: bin_gpdb passed: [gate_ud_start] trigger: [[ test_trigger ]] resource: bin_gpdb_centos6 - get: ccp_src - get: centos-gpdb-dev-6 - put: terraform params: <<: *ccp_default_params vars: <<: *ccp_default_vars number_of_nodes: 1 PLATFORM: centos7 instance_type: n1-standard-4 - task: gen_and_initialize_mapr file: gpdb_src/concourse/tasks/gen_mapr.yml params: <<: *ccp_gen_cluster_default_params - task: regression_tests_gphdfs_mapr file: gpdb_src/concourse/tasks/regression_tests_gphdfs_mapr.yml image: centos-gpdb-dev-6 params: TARGET_OS: centos TARGET_OS_VERSION: 6 {% endif %} {% if "AA" in test_sections %} ## ====================================================================== ## _ _ ## / \ / \ ## / _ \ / _ \ ## / ___ \ / ___ \ ## /_/ \_\/_/ \_\ ## ====================================================================== - name: gate_advanced_analytics_start plan: - aggregate: - get: gpdb_src passed: {% if "centos6" in os_types %} - compile_gpdb_centos6 {% endif %} {% if "centos7" in os_types %} - compile_gpdb_centos7 {% endif %} trigger: true {% if "centos6" in os_types %} - get: bin_gpdb_centos6 passed: - compile_gpdb_centos6 {% endif %} {% if "centos7" in os_types %} - get: bin_gpdb_centos7 passed: - compile_gpdb_centos7 {% endif %} {% if "centos6" in os_types %} - name: MADlib_Test_planner_centos6 plan: - aggregate: - get: madlib_ci - get: gpdb_src passed: [gate_advanced_analytics_start] - get: bin_gpdb resource: bin_gpdb_centos6 trigger: [[ test_trigger ]] passed: [gate_advanced_analytics_start] - get: centos-gpdb-dev-6 - get: madlib_gppkg - task: MADlib_Test_gppkg file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml image: centos-gpdb-dev-6 params: TEST_OS: centos ORCA: "off" DBVER: gpdb6-assert - name: MADlib_Test_orca_centos6 plan: - aggregate: - get: madlib_ci - get: gpdb_src passed: [gate_advanced_analytics_start] - get: bin_gpdb resource: bin_gpdb_centos6 trigger: [[ test_trigger ]] passed: [gate_advanced_analytics_start] - get: centos-gpdb-dev-6 - get: madlib_gppkg - task: MADlib_Test_gppkg file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml image: centos-gpdb-dev-6 params: TEST_OS: centos ORCA: "on" DBVER: gpdb6-assert {% endif %} {% if "centos7" in os_types %} - name: MADlib_Test_planner_centos7 plan: - aggregate: - get: madlib_ci - get: gpdb_src passed: [gate_advanced_analytics_start] - get: bin_gpdb resource: bin_gpdb_centos7 trigger: [[ test_trigger ]] passed: [gate_advanced_analytics_start] - get: centos-gpdb-test-7 - get: madlib_gppkg - task: MADlib_Test_gppkg file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml image: centos-gpdb-test-7 params: TEST_OS: centos ORCA: "off" DBVER: gpdb6-assert - name: MADlib_Test_orca_centos7 plan: - aggregate: - get: madlib_ci - get: gpdb_src passed: [gate_advanced_analytics_start] - get: bin_gpdb resource: bin_gpdb_centos7 trigger: [[ test_trigger ]] passed: [gate_advanced_analytics_start] - get: centos-gpdb-test-7 - get: madlib_gppkg - task: MADlib_Test_gppkg file: madlib_ci/concourse/tasks/madlib_test_gppkg.yml image: centos-gpdb-test-7 params: TEST_OS: centos ORCA: "on" DBVER: gpdb6-assert {% endif %} {% endif %} {% if pipeline_type == "prod" %} ## ====================================================================== ## ____ _ ## | _ \ ___| | ___ __ _ ___ ___ ## | |_) / _ \ |/ _ \/ _` / __|/ _ \ ## | _ < __/ | __/ (_| \__ \ __/ ## |_| \_\___|_|\___|\__,_|___/\___| ## ====================================================================== - name: gate_release_candidate_start plan: - aggregate: - get: gpdb_src trigger: true passed: - compile_gpdb_centos6 - compile_gpdb_open_source_centos6 - compile_gpdb_centos7 - compile_gpdb_sles11 - compile_gpdb_ubuntu16 - compile_gpdb_windows_cl - compile_gpdb_aix7_remote - icw_planner_centos6 - icw_gporca_centos6 - icw_gporca_centos7 - icw_planner_centos7 - icw_gporca_sles11 - icw_gporca_sles12 - icw_planner_sles12 - icw_planner_ubuntu16 - icw_gporca_conan_ubuntu16 - gpdb_packaging_ubuntu16 - icw_planner_ictcp_centos6 - icw_extensions_gpcloud_centos6 - icw_extensions_gpcloud_ubuntu16 - client_loader_remote_test_aix - resource_group_centos6 - resource_group_centos7 - resource_group_sles12 - segwalrep_mirrorless_centos6 - walrep_2 - regression_tests_gphdfs_hadoop_centos - regression_tests_gphdfs_mapr_centos - regression_tests_pxf {% for test_name in CLI_1_suites + CLI_2_suites + CLI_gpexpand_suites %} - [[ test_name ]] {% endfor %} - MADlib_Test_orca_centos6 - MADlib_Test_planner_centos6 - MADlib_Test_orca_centos7 - MADlib_Test_planner_centos7 - gpaddmirrors - gpexpand - gpcheck - gppkg - check_centos - gptransfer - interconnect - get: bin_gpdb_centos6 trigger: true passed: - compile_gpdb_centos6 - icw_planner_centos6 - icw_gporca_centos6 - icw_planner_ictcp_centos6 - icw_extensions_gpcloud_centos6 - client_loader_remote_test_aix - resource_group_centos6 - segwalrep_mirrorless_centos6 - walrep_2 - regression_tests_gphdfs_hadoop_centos - regression_tests_gphdfs_mapr_centos - regression_tests_pxf {% for test_name in CLI_1_suites + CLI_2_suites + CLI_gpexpand_suites %} - [[ test_name ]] {% endfor %} - MADlib_Test_orca_centos6 - MADlib_Test_planner_centos6 - gpexpand - gpcheck - gppkg - check_centos - gptransfer - interconnect - get: bin_gpdb_centos7 trigger: true passed: - compile_gpdb_centos7 - icw_gporca_centos7 - icw_planner_centos7 - resource_group_centos7 - MADlib_Test_orca_centos7 - MADlib_Test_planner_centos7 - get: bin_gpdb_sles11 trigger: true passed: - compile_gpdb_sles11 - icw_gporca_sles11 - icw_gporca_sles12 - icw_planner_sles12 - resource_group_sles12 - name: Release_Candidate plan: - aggregate: - get: gpdb_src trigger: true passed: [gate_release_candidate_start] - get: bin_gpdb_centos6 trigger: true passed: [gate_release_candidate_start] - get: bin_gpdb_centos7 trigger: true passed: [gate_release_candidate_start] - get: bin_gpdb_sles11 trigger: true passed: [gate_release_candidate_start] - task: verify_gpdb_versions file: gpdb_src/concourse/tasks/verify_gpdb_versions.yml - put: bin_gpdb_centos6_rc params: file: bin_gpdb_centos6/bin_gpdb.tar.gz - put: bin_gpdb_centos7_rc params: file: bin_gpdb_centos7/bin_gpdb.tar.gz - put: bin_gpdb_sles11_rc params: file: bin_gpdb_sles11/bin_gpdb.tar.gz {% endif %}