Publish server builds from compile tasks

Server release candidate artifacts are not published until after an
extensive set of tests have passed in the CI pipeline. These tests
include ICW and all the CLI test suites. It is not unusual for time
between a commit being pushed to a release candidate being published to
be several hours, potentially slowing down the feedback cycle for
component teams.

This commit adds a "published" output to the compliation tasks. The new
build artifact is stored in an immutable GCS bucket with the version in
the filename. This makes it trivial for other pipelines to safely
consume the latest build.

This artifact **has not** passed any sort of testing (e.g., ICW) and
should only be used in development pipelines that need
near-instantaneous feedback on commits going into GPDB.

For the server-build artifact, `((rc-build-type-gcs))` will resolve to
`.debug` for the with asserts pipeline and `''` (i.e., empty string) for the
without asserts pipeline. The two types of server artifacts that are
"published" are:

1. server-build
2. server-rc

server-build is the output of the compilation task and has had no
testing; server-rc is a release candidate of the server component.
Authored-by: NBradford D. Boyle <bboyle@pivotal.io>

concourse/pipelines/templates/gpdb-tpl.yml

@@ -517,6 +517,13 @@ resources:
     json_key: ((concourse-gcs-resources-service-account-key))
     versioned_file: ((pipeline-name))/bin_gpdb_clients_centos6/bin_gpdb_clients.tar.gz
 
+- name: server-build-centos6
+  type: gcs
+  source:
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/master/server-build-(.*)-rhel6_x86_64((rc-build-type-gcs)).tar.gz
+
 {% endif %}
 {% if pipeline_configuration == "prod" %}
 - name: bin_gpdb_centos6_icw_green
@@ -595,6 +602,13 @@ resources:
     json_key: ((concourse-gcs-resources-service-account-key))
     versioned_file: ((pipeline-name))/bin_gpdb_clients_centos7/bin_gpdb_clients.tar.gz
 
+- name: server-build-centos7
+  type: gcs
+  source:
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/master/server-build-(.*)-rhel7_x86_64((rc-build-type-gcs)).tar.gz
+
 {% endif %}
 {% if "ubuntu18.04" in os_types %}
 - name: bin_gpdb_ubuntu18.04
@@ -611,6 +625,13 @@ resources:
     json_key: ((concourse-gcs-resources-service-account-key))
     versioned_file: ((pipeline-name))/bin_gpdb_clients_ubuntu18.04/bin_gpdb_clients.tar.gz
 
+- name: server-build-ubuntu18.04
+  type: gcs
+  source:
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/master/server-build-(.*)-ubuntu18.04_x86_64((rc-build-type-gcs)).tar.gz
+
 {% endif %}
 {% if "win" in os_types %}
 - name: terraform_windows
@@ -893,6 +914,7 @@ jobs:
       TARGET_OS: centos
       TARGET_OS_VERSION: 6
       BLD_TARGETS: "clients"
+      RC_BUILD_TYPE_GCS: ((rc-build-type-gcs))
   - aggregate:
     - put: bin_gpdb_centos6
       params:
@@ -900,6 +922,9 @@ jobs:
     - put: bin_gpdb_clients_centos6
       params:
         file: gpdb_artifacts/gpdb-clients-centos6.tar.gz
+    - put: server-build-centos6
+      params:
+        file: gpdb_artifacts/server-build-*-rhel6*.tar.gz
 
 {% endif %}
 {% if "centos7" in os_types %}
@@ -927,6 +952,7 @@ jobs:
       TARGET_OS: centos
       TARGET_OS_VERSION: 7
       BLD_TARGETS: "clients"
+      RC_BUILD_TYPE_GCS: ((rc-build-type-gcs))
   - aggregate:
     - put: bin_gpdb_centos7
       params:
@@ -934,6 +960,9 @@ jobs:
     - put: bin_gpdb_clients_centos7
       params:
         file: gpdb_artifacts/gpdb-clients-centos7.tar.gz
+    - put: server-build-centos7
+      params:
+        file: gpdb_artifacts/server-build-*-rhel7*.tar.gz
 
 {% endif %}
 {% if "centos6" in os_types %}
@@ -964,6 +993,7 @@ jobs:
       TARGET_OS: centos
       TARGET_OS_VERSION: 6
       BLD_TARGETS: "clients loaders"
+      RC_BUILD_TYPE_GCS: ((rc-build-type-gcs))
   - aggregate:
     - put: binary_swap_gpdb_centos6
       params:
@@ -991,6 +1021,7 @@ jobs:
         TARGET_OS: ubuntu
         TARGET_OS_VERSION: "18.04"
         BLD_TARGETS: "clients"
+        RC_BUILD_TYPE_GCS: ((rc-build-type-gcs))
     - aggregate:
         - put: bin_gpdb_ubuntu18.04
           params:
@@ -998,6 +1029,9 @@ jobs:
         - put: bin_gpdb_clients_ubuntu18.04
           params:
             file: gpdb_artifacts/gpdb-clients-ubuntu18.04.tar.gz
+        - put: server-build-ubuntu18.04
+          params:
+            file: gpdb_artifacts/server-build-*-ubuntu18.04*.tar.gz
 
 {% endif %}
 

concourse/scripts/compile_gpdb.bash

@@ -157,12 +157,20 @@ function include_libstdcxx() {
 
 function export_gpdb() {
   TARBALL="${GPDB_ARTIFACTS_DIR}/${GPDB_BIN_FILENAME}"
+  local server_version
+  server_version="$("${GPDB_SRC_PATH}"/getversion --short)"
+
+  local server_build
+  server_build="${GPDB_ARTIFACTS_DIR}/server-build-${server_version}-${BLD_ARCH}${RC_BUILD_TYPE_GCS}.tar.gz"
+
   pushd ${GREENPLUM_INSTALL_DIR}
     source greenplum_path.sh
     python -m compileall -q -x test .
     chmod -R 755 .
     tar -czf "${TARBALL}" ./*
   popd
+
+  cp "${TARBALL}" "${server_build}"
 }
 
 function export_gpdb_extensions() {

concourse/tasks/compile_gpdb.yml

@@ -20,3 +20,4 @@ params:
   OUTPUT_ARTIFACT_DIR: gpdb_artifacts
   CONFIGURE_FLAGS:
   ORCA_TAG: v3.74.0
+  RC_BUILD_TYPE_GCS: