Add pr dev pipeline to mimic pr pipeline

[#134457777]

concourse/README.md

@@ -8,12 +8,13 @@ The Concourse directory should contain this README and three sub-directories onl
 * scripts
 
 ##### Pipelines Directory
-There should be five pipelines in this directory:
+There should be seven pipelines in this directory:
 
 * `pipeline.yml` the pipeline that compiles, tests, and produces installers from the master branch of gpdb.
-* `dev_pipeline.yml` a safe-to-duplicate version of `pipeline.yml` which saves artifacts to dev buckets.
-* `dev_generate_installer.yml` which compiles and generates an installer for the given source and saves it to dev buckets.
+* `dev_pipeline.yml` a safe-to-duplicate version of `pipeline.yml` which saves artifacts to a dev bucket.
+* `dev_generate_installer.yml` which compiles and generates an installer for the given source and saves it to a dev bucket.
 * `pr_pipeline.yml` which compiles and tests pull requests.
+* `dev_pr_pipeline.yml` which safely mimics the `pr_pipeline.yml`, and saves an installer to the dev bucket.
 * `concourse-upgrade.yml` which uses Concourse to upgrade itself.
 * `pipeline_tinc.yml` run TINC tests against gpdb master branch.
 

concourse/pipelines/dev_pr_pipeline.yml

+## ======================================================================
+## resources
+## ======================================================================
+
+resources:
+  - name: gpdb_pr
+    type: git
+    source:
+      branch: {{gpdb-git-branch}}
+      private_key: {{gpdb-git-key}}
+      uri: {{gpdb-git-remote}}
+
+  - name: gpaddon_src
+    type: git
+    source:
+      branch: {{gpaddon-git-branch}}
+      private_key: {{gpdb-git-key}}
+      uri: {{gpaddon-git-remote}}
+
+  - name: centos67-gpdb-gcc6-llvm-image
+    type: docker-image
+    source:
+      repository: pivotaldata/centos67-gpdb-gcc6-llvm-image
+      username: {{docker_username}}
+      password: {{docker_password}}
+
+  - name: sync_tools_gpdb_centos
+    type: s3
+    source:
+      access_key_id: {{bucket-access-key-id}}
+      bucket: {{dev-bucket-name}}
+      region_name: {{aws-region}}
+      secret_access_key: {{bucket-secret-access-key}}
+      versioned_file: gpdb_pr/sync_tools_gpdb/sync_tools_gpdb.tar.gz
+
+  - name: bin_gpdb_centos
+    type: s3
+    source:
+      access_key_id: {{bucket-access-key-id}}
+      bucket: {{dev-bucket-name}}
+      region_name: {{aws-region}}
+      secret_access_key: {{bucket-secret-access-key}}
+      versioned_file: gpdb_pr/gpdb_artifacts/bin_gpdb.tar.gz
+
+  - name: installer_rhel6_gpdb_rc
+    type: s3
+    source:
+      access_key_id: {{bucket-access-key-id}}
+      bucket: {{dev-bucket-name}}
+      region_name: {{aws-region}}
+      secret_access_key: {{bucket-secret-access-key}}
+      versioned_file: deliverables/greenplum-db-4.3.99.0-build-1-rhel6-x86_64.zip
+
+  - name: installer_rhel6_gpdb_rc_md5
+    type: s3
+    source:
+      access_key_id: {{bucket-access-key-id}}
+      bucket: {{dev-bucket-name}}
+      region_name: {{aws-region}}
+      secret_access_key: {{bucket-secret-access-key}}
+      versioned_file: deliverables/greenplum-db-4.3.99.0-build-1-rhel6-x86_64.zip.md5
+
+  - name: qautils_rhel6_tarball
+    type: s3
+    source:
+      access_key_id: {{bucket-access-key-id}}
+      bucket: {{dev-bucket-name}}
+      region_name: {{aws-region}}
+      secret_access_key: {{bucket-secret-access-key}}
+      versioned_file: deliverables/QAUtils-rhel6-x86_64.tar.gz
+
+jobs:
+  - name: compile_gpdb_centos6
+    plan:
+    - aggregate:
+      - get: gpdb_pr
+        trigger: true
+      - get: centos67-gpdb-gcc6-llvm-image
+    - task: compile_gpdb
+      file: gpdb_pr/concourse/tasks/compile_gpdb.yml
+      image: centos67-gpdb-gcc6-llvm-image
+      input_mapping:
+        gpdb_src: gpdb_pr
+      params:
+        IVYREPO_HOST: {{ivyrepo_host}}
+        IVYREPO_REALM: {{ivyrepo_realm}}
+        IVYREPO_USER: {{ivyrepo_user}}
+        IVYREPO_PASSWD: {{ivyrepo_passwd}}
+        TARGET_OS: centos
+        TARGET_OS_VERSION: 6
+        BLD_TARGETS: ""
+    - aggregate:
+      - put: bin_gpdb_centos
+        params:
+          file: gpdb_artifacts/bin_gpdb.tar.gz
+      - put: sync_tools_gpdb_centos
+        params:
+          file: sync_tools_gpdb/sync_tools_gpdb.tar.gz
+
+  - name: compile_gpdb_custom_config_centos6
+    public: true
+    plan:
+    - aggregate:
+      - get: gpdb_pr
+        trigger: true
+      - get: centos67-gpdb-gcc6-llvm-image
+    - task: compile_gpdb
+      file: gpdb_pr/concourse/tasks/compile_gpdb_custom_config.yml
+      image: centos67-gpdb-gcc6-llvm-image
+      input_mapping:
+        gpdb_src: gpdb_pr
+
+  # Stage 2: IC Tests
+  - name: icg_planner_centos6
+    plan:
+    - aggregate:
+      - get: gpdb_pr
+        passed: [compile_gpdb_centos6]
+      - get: sync_tools_gpdb_centos
+        resource: sync_tools_gpdb_centos
+        passed: [compile_gpdb_centos6]
+      - get: bin_gpdb
+        resource: bin_gpdb_centos
+        passed: [compile_gpdb_centos6]
+        trigger: true
+      - get: centos67-gpdb-gcc6-llvm-image
+    - task: ic_gpdb
+      file: gpdb_pr/concourse/tasks/ic_gpdb.yml
+      image: centos67-gpdb-gcc6-llvm-image
+      input_mapping:
+        gpdb_src: gpdb_pr
+      params:
+        MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off -c codegen=off' installcheck-good
+        BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
+        TEST_OS: centos
+      timeout: 2h30m
+
+  # Stage 3: Packaging
+
+  - name: gpdb_rc_packaging_centos
+    serial: true
+    plan:
+    - aggregate:
+      - get: gpdb_pr
+        passed: [icg_planner_centos6]
+      - get: gpaddon_src
+      - get: bin_gpdb
+        resource: bin_gpdb_centos
+        passed: [icg_planner_centos6]
+        trigger: true
+      - get: centos67-gpdb-gcc6-llvm-image
+    - task: separate_qautils_files_for_rc
+      file: gpdb_src/concourse/separate_qautils_files_for_rc.yml
+      image: centos67-gpdb-gcc6-llvm-image
+      params:
+        QAUTILS_TARBALL: rc_bin_gpdb/QAUtils-rhel6-x86_64.tar.gz
+    - aggregate:
+      - task: gpdb_rc_packaging_centos
+        file: gpdb_src/concourse/gpdb_packaging.yml
+        input_mapping:
+          gpdb_src: gpdb_pr
+        image: centos67-gpdb-gcc6-llvm-image
+        input_mapping:
+          bin_gpdb: rc_bin_gpdb
+        output_mapping:
+          packaged_gpdb: packaged_gpdb_rc
+        params:
+          INSTALL_SCRIPT_SRC: gpdb_src/gpAux/addon/license/installer-header-rhel-gpdb.sh
+          INSTALLER_ZIP: packaged_gpdb/greenplum-db-4.3.99.0-build-1-rhel6-x86_64.zip
+          ADD_README_INSTALL: true
+    - aggregate:
+      - put: installer_rhel6_gpdb_rc
+        params:
+          file: packaged_gpdb_rc/greenplum-db-4.3.99.0-build-1-rhel6-x86_64.zip
+      - put: installer_rhel6_gpdb_rc_md5
+        params:
+          file: packaged_gpdb_rc/greenplum-db-4.3.99.0-build-1-rhel6-x86_64.zip.md5
+      - put: qautils_rhel6_tarball
+        params:
+          file: rc_bin_gpdb/QAUtils-rhel6-x86_64.tar.gz