Use binaries from PivNet for binary_swap tests

Instead of re-compiling from source at a release tag, the pipeline downloads a
released RPM from PivNet and transforms it to a tarball for use in binary_swap
tests.
Co-authored-by: NAsim Praveen <apraveen@pivotal.io>
Co-authored-by: NBradford D. Boyle <bboyle@pivotal.io>

concourse/pipelines/gen_pipeline.py

@@ -68,7 +68,7 @@ JOBS_THAT_ARE_GATES = [
 
 JOBS_THAT_SHOULD_NOT_BLOCK_RELEASE = (
     [
-        'compile_gpdb_binary_swap_centos6',
+        'prepare_binary_swap_gpdb_centos6',
         'compile_gpdb_clients_windows',
         'test_gpdb_clients_windows',
         'icw_gporca_centos6_gpos_memory',

concourse/pipelines/gpdb_6X_STABLE-generated.yml

@@ -12,7 +12,7 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2019-06-25 14:57:22.783904
+## Generated by gen_pipeline.py at: 2019-06-25 16:11:11.640969
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'ubuntu18.04', 'win']
 ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions', 'Gpperfmon']
@@ -30,7 +30,7 @@ groups:
 - name: all
   jobs:
   - compile_gpdb_centos6
-  - compile_gpdb_binary_swap_centos6
+  - prepare_binary_swap_gpdb_centos6
   - compile_gpdb_open_source_centos6
   - compile_gpdb_centos7
   - compile_gpdb_ubuntu18.04
@@ -110,7 +110,6 @@ groups:
 - name: Compile
   jobs:
   - compile_gpdb_centos6
-  - compile_gpdb_binary_swap_centos6
   - compile_gpdb_open_source_centos6
   - compile_gpdb_centos7
   - compile_gpdb_ubuntu18.04
@@ -121,6 +120,7 @@ groups:
 
 - name: ICW
   jobs:
+  - prepare_binary_swap_gpdb_centos6
   - icw_gporca_centos6
   - icw_gporca_centos6_gpos_memory
   - icw_planner_centos6
@@ -234,6 +234,11 @@ resource_types:
   type: docker-image
   source:
     repository: frodenas/gcs-resource
+- name: pivnet
+  type: docker-image
+  source:
+    repository: pivotalcf/pivnet-resource
+    tag: latest-final
 ## ======================================================================
 ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  ___
 ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __|
@@ -289,14 +294,12 @@ resources:
     - gpdb-doc/*
     - README*
 
-- name: gpdb_src_binary_swap
-  type: git
+- name: previous_gpdb_rpm
+  type: pivnet
   source:
-    branch: {{gpdb-binary-swap-git-tag}}
-    uri: {{gpdb-binary-swap-git-remote}}
-    ignore_paths:
-    - gpdb-doc/*
-    - README*
+    api_token: ((pivotal-gpdb-pivnet-api-token))
+    product_slug: pivotal-gpdb
+    product_version: ((pivotal-gpdb-pivnet-product-version))
 
 
 - name: libquicklz-centos6
@@ -823,37 +826,22 @@ jobs:
     params:
         CONFIGURE_FLAGS: {{configure_flags}}
 
-- name: compile_gpdb_binary_swap_centos6
+- name: prepare_binary_swap_gpdb_centos6
   plan:
-  # This acts like a cache as this job will only be run once to get a
-  # binary to use for our binary swap compatibility tests. Setting a new
-  # tag or branch for the gpdb_src_binary_swap resource via set-pipeline
-  # will replace the cached binary.
   - aggregate:
     - get: gpdb_src
-      resource: gpdb_src_binary_swap
+    - get: gpdb_rpm
+      resource: previous_gpdb_rpm
       trigger: true
+      params:
+        globs: [greenplum-db-6*-rhel6-x86_64.rpm]
     - get: gpdb6-centos6-build
-    - get: libquicklz-installer
-      resource: libquicklz-centos6
-    - get: libquicklz-devel-installer
-      resource: libquicklz-devel-centos6
-    - get: libsigar-installer
-      resource: libsigar-centos6
-    - get: python-tarball
-      resource: python-centos6
-  - task: compile_gpdb
-    file: gpdb_src/concourse/tasks/compile_gpdb.yml
+  - task: generate_previous_bin_gpdb
+    file: gpdb_src/concourse/tasks/extract_rpm.yml
     image: gpdb6-centos6-build
+  - put: binary_swap_gpdb_centos6
     params:
-      CONFIGURE_FLAGS: {{configure_flags_with_extensions}}
-      TARGET_OS: centos
-      TARGET_OS_VERSION: 6
-      BLD_TARGETS: "clients loaders"
-  - aggregate:
-    - put: binary_swap_gpdb_centos6
-      params:
-        file: gpdb_artifacts/bin_gpdb.tar.gz
+      file: gpdb_artifacts/bin_gpdb.tar.gz
 
 - name: compile_gpdb_ubuntu18.04
   plan:
@@ -957,7 +945,7 @@ jobs:
       resource: bin_gpdb_centos6
       trigger: true
     - get: binary_swap_gpdb
-      passed: [compile_gpdb_binary_swap_centos6]
+      passed: [prepare_binary_swap_gpdb_centos6]
       resource: binary_swap_gpdb_centos6
       trigger: true
     - get: gpdb6-centos6-test
@@ -1175,7 +1163,7 @@ jobs:
       passed:
       - compile_gpdb_centos7
     - get: binary_swap_gpdb
-      passed: [compile_gpdb_binary_swap_centos6]
+      passed: [prepare_binary_swap_gpdb_centos6]
       resource: binary_swap_gpdb_centos6
       trigger: true
 
@@ -1281,7 +1269,7 @@ jobs:
       passed:
       - compile_gpdb_centos7
     - get: binary_swap_gpdb
-      passed: [compile_gpdb_binary_swap_centos6]
+      passed: [prepare_binary_swap_gpdb_centos6]
       resource: binary_swap_gpdb_centos6
       trigger: true
 

concourse/pipelines/templates/gpdb-tpl.yml

@@ -43,7 +43,7 @@ groups:
   jobs:
 {% if "centos6" in os_types %}
   - compile_gpdb_centos6
-  - compile_gpdb_binary_swap_centos6
+  - prepare_binary_swap_gpdb_centos6
   - compile_gpdb_open_source_centos6
 {% endif %}
 {% if "centos7" in os_types %}
@@ -145,7 +145,6 @@ groups:
   jobs:
 {% if "centos6" in os_types %}
   - compile_gpdb_centos6
-  - compile_gpdb_binary_swap_centos6
   - compile_gpdb_open_source_centos6
 {% endif %}
 {% if "centos7" in os_types %}
@@ -165,6 +164,7 @@ groups:
 - name: ICW
   jobs:
 {% if "centos6" in os_types %}
+  - prepare_binary_swap_gpdb_centos6
   - icw_gporca_centos6
   - icw_gporca_centos6_gpos_memory
   - icw_planner_centos6
@@ -292,6 +292,11 @@ resource_types:
   type: docker-image
   source:
     repository: frodenas/gcs-resource
+- name: pivnet
+  type: docker-image
+  source:
+    repository: pivotalcf/pivnet-resource
+    tag: latest-final
 ## ======================================================================
 ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  ___
 ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __|
@@ -355,14 +360,12 @@ resources:
     - README*
 
 {% if "centos6" in os_types %}
-- name: gpdb_src_binary_swap
-  type: git
+- name: previous_gpdb_rpm
+  type: pivnet
   source:
-    branch: {{gpdb-binary-swap-git-tag}}
-    uri: {{gpdb-binary-swap-git-remote}}
-    ignore_paths:
-    - gpdb-doc/*
-    - README*
+    api_token: ((pivotal-gpdb-pivnet-api-token))
+    product_slug: pivotal-gpdb
+    product_version: ((pivotal-gpdb-pivnet-product-version))
 
 {% endif %}
 
@@ -929,37 +932,22 @@ jobs:
 
 {% endif %}
 {% if "centos6" in os_types %}
-- name: compile_gpdb_binary_swap_centos6
+- name: prepare_binary_swap_gpdb_centos6
   plan:
-  # This acts like a cache as this job will only be run once to get a
-  # binary to use for our binary swap compatibility tests. Setting a new
-  # tag or branch for the gpdb_src_binary_swap resource via set-pipeline
-  # will replace the cached binary.
   - aggregate:
     - get: gpdb_src
-      resource: gpdb_src_binary_swap
+    - get: gpdb_rpm
+      resource: previous_gpdb_rpm
       trigger: true
+      params:
+        globs: [greenplum-db-6*-rhel6-x86_64.rpm]
     - get: gpdb6-centos6-build
-    - get: libquicklz-installer
-      resource: libquicklz-centos6
-    - get: libquicklz-devel-installer
-      resource: libquicklz-devel-centos6
-    - get: libsigar-installer
-      resource: libsigar-centos6
-    - get: python-tarball
-      resource: python-centos6
-  - task: compile_gpdb
-    file: gpdb_src/concourse/tasks/compile_gpdb.yml
+  - task: generate_previous_bin_gpdb
+    file: gpdb_src/concourse/tasks/extract_rpm.yml
     image: gpdb6-centos6-build
+  - put: binary_swap_gpdb_centos6
     params:
-      CONFIGURE_FLAGS: {{configure_flags_with_extensions}}
-      TARGET_OS: centos
-      TARGET_OS_VERSION: 6
-      BLD_TARGETS: "clients loaders"
-  - aggregate:
-    - put: binary_swap_gpdb_centos6
-      params:
-        file: gpdb_artifacts/bin_gpdb.tar.gz
+      file: gpdb_artifacts/bin_gpdb.tar.gz
 
 {% endif %}
 {% if "ubuntu18.04" in os_types %}
@@ -1070,7 +1058,7 @@ jobs:
       resource: bin_gpdb_centos6
       trigger: [[ test_trigger ]]
     - get: binary_swap_gpdb
-      passed: [compile_gpdb_binary_swap_centos6]
+      passed: [prepare_binary_swap_gpdb_centos6]
       resource: binary_swap_gpdb_centos6
       trigger: [[ test_trigger ]]
     - get: gpdb6-centos6-test
@@ -1318,7 +1306,7 @@ jobs:
 {% endif %}
 {% if "centos6" in os_types or "centos7" in os_types %}
     - get: binary_swap_gpdb
-      passed: [compile_gpdb_binary_swap_centos6]
+      passed: [prepare_binary_swap_gpdb_centos6]
       resource: binary_swap_gpdb_centos6
       trigger: [[ test_trigger ]]
 {% endif %}
@@ -1434,7 +1422,7 @@ jobs:
 {% endif %}
 {% if "centos6" in os_types or "centos7" in os_types %}
     - get: binary_swap_gpdb
-      passed: [compile_gpdb_binary_swap_centos6]
+      passed: [prepare_binary_swap_gpdb_centos6]
       resource: binary_swap_gpdb_centos6
       trigger: [[ test_trigger ]]
 {% endif %}

concourse/scripts/extract_rpm.bash

+#!/usr/bin/env bash
+
+extract_rpm_to_tar () {
+    rpm2cpio gpdb_rpm/greenplum-db-*.rpm | cpio -idvm
+
+    local tarball="${PWD}/gpdb_artifacts/bin_gpdb.tar.gz"
+    pushd usr/local/greenplum-db-*
+        tar czf "${tarball}" ./*
+    popd
+}
+
+extract_rpm_to_tar "${@}"
\ No newline at end of file

concourse/tasks/extract_rpm.yml

+platform: linux
+image_resource:
+  type: docker-image
+inputs:
+  - name: gpdb_rpm
+  - name: gpdb_src
+
+outputs:
+  - name: gpdb_artifacts
+run:
+  path: gpdb_src/concourse/scripts/extract_rpm.bash