From b679ac6f48084c3630d6845b1e0653800e0dc690 Mon Sep 17 00:00:00 2001 From: "Bradford D. Boyle" Date: Mon, 7 Oct 2019 15:20:03 -0700 Subject: [PATCH] Publish server builds from compile tasks Server release candidate artifacts are not published until after an extensive set of tests have passed in the CI pipeline. These tests include ICW and all the CLI test suites. It is not unusual for time between a commit being pushed to a release candidate being published to be several hours, potentially slowing down the feedback cycle for component teams. This commit adds a "published" output to the compliation tasks. The new build artifact is stored in an immutable GCS bucket with the version in the filename. This makes it trivial for other pipelines to safely consume the latest build. This artifact **has not** passed any sort of testing (e.g., ICW) and should only be used in development pipelines that need near-instantaneous feedback on commits going into GPDB. For the server-build artifact, `((rc-build-type-gcs))` will resolve to `.debug` for the with asserts pipeline and `''` (i.e., empty string) for the without asserts pipeline. The two types of server artifacts that are "published" are: 1. server-build 2. server-rc server-build is the output of the compilation task and has had no testing; server-rc is a release candidate of the server component. Authored-by: Bradford D. Boyle (cherry picked from commit 94a8ffc9719f922149a99b323030c33a5eb35732) --- concourse/pipelines/templates/gpdb-tpl.yml | 33 ++++++++++++++++++++++ concourse/scripts/compile_gpdb.bash | 8 ++++++ concourse/tasks/compile_gpdb.yml | 1 + 3 files changed, 42 insertions(+) diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index b71b0593c0..0fe380f3d6 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -516,6 +516,13 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_clients_centos6/bin_gpdb_clients.tar.gz +- name: server-build-centos6 + type: gcs + source: + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/master/server-build-(.*)-rhel6_x86_64((rc-build-type-gcs)).tar.gz + {% endif %} {% if pipeline_configuration == "prod" %} - name: bin_gpdb_centos6_icw_green @@ -594,6 +601,13 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_clients_centos7/bin_gpdb_clients.tar.gz +- name: server-build-centos7 + type: gcs + source: + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/master/server-build-(.*)-rhel7_x86_64((rc-build-type-gcs)).tar.gz + {% endif %} {% if "ubuntu18.04" in os_types %} - name: bin_gpdb_ubuntu18.04 @@ -610,6 +624,13 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_clients_ubuntu18.04/bin_gpdb_clients.tar.gz +- name: server-build-ubuntu18.04 + type: gcs + source: + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/master/server-build-(.*)-ubuntu18.04_x86_64((rc-build-type-gcs)).tar.gz + {% endif %} {% if "win" in os_types %} - name: terraform_windows @@ -893,6 +914,7 @@ jobs: TARGET_OS: centos TARGET_OS_VERSION: 6 BLD_TARGETS: "clients" + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) - aggregate: - put: bin_gpdb_centos6 params: @@ -900,6 +922,9 @@ jobs: - put: bin_gpdb_clients_centos6 params: file: gpdb_artifacts/gpdb-clients-centos6.tar.gz + - put: server-build-centos6 + params: + file: gpdb_artifacts/server-build-*-rhel6*.tar.gz {% endif %} {% if "centos7" in os_types %} @@ -927,6 +952,7 @@ jobs: TARGET_OS: centos TARGET_OS_VERSION: 7 BLD_TARGETS: "clients" + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) - aggregate: - put: bin_gpdb_centos7 params: @@ -934,6 +960,9 @@ jobs: - put: bin_gpdb_clients_centos7 params: file: gpdb_artifacts/gpdb-clients-centos7.tar.gz + - put: server-build-centos7 + params: + file: gpdb_artifacts/server-build-*-rhel7*.tar.gz {% endif %} {% if "centos6" in os_types %} @@ -976,6 +1005,7 @@ jobs: TARGET_OS: ubuntu TARGET_OS_VERSION: "18.04" BLD_TARGETS: "clients" + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) - aggregate: - put: bin_gpdb_ubuntu18.04 params: @@ -983,6 +1013,9 @@ jobs: - put: bin_gpdb_clients_ubuntu18.04 params: file: gpdb_artifacts/gpdb-clients-ubuntu18.04.tar.gz + - put: server-build-ubuntu18.04 + params: + file: gpdb_artifacts/server-build-*-ubuntu18.04*.tar.gz {% endif %} {% if "win" in os_types %} diff --git a/concourse/scripts/compile_gpdb.bash b/concourse/scripts/compile_gpdb.bash index e6339a65e9..465baf25cd 100755 --- a/concourse/scripts/compile_gpdb.bash +++ b/concourse/scripts/compile_gpdb.bash @@ -157,12 +157,20 @@ function include_libstdcxx() { function export_gpdb() { TARBALL="${GPDB_ARTIFACTS_DIR}/${GPDB_BIN_FILENAME}" + local server_version + server_version="$("${GPDB_SRC_PATH}"/getversion --short)" + + local server_build + server_build="${GPDB_ARTIFACTS_DIR}/server-build-${server_version}-${BLD_ARCH}${RC_BUILD_TYPE_GCS}.tar.gz" + pushd ${GREENPLUM_INSTALL_DIR} source greenplum_path.sh python -m compileall -q -x test . chmod -R 755 . tar -czf "${TARBALL}" ./* popd + + cp "${TARBALL}" "${server_build}" } function export_gpdb_extensions() { diff --git a/concourse/tasks/compile_gpdb.yml b/concourse/tasks/compile_gpdb.yml index 64be3ab154..3a8a52a081 100644 --- a/concourse/tasks/compile_gpdb.yml +++ b/concourse/tasks/compile_gpdb.yml @@ -20,3 +20,4 @@ params: OUTPUT_ARTIFACT_DIR: gpdb_artifacts CONFIGURE_FLAGS: ORCA_TAG: v3.74.0 + RC_BUILD_TYPE_GCS: -- GitLab