Prebuilt binaries of Open Source Greenplum 6 Database Server as a

Tarball

Configure with specific configure flags for open source gpdb.
Except for python, others quicklz, sigar are not built with open source
gpdb. Includes python dependencies from pythonsrc-ext.

For more info, refer https://github.com/pivotal/gp-releng/blob/master/docs/OpenSource-Greenplum-Database-Server-Feature-Component-Packaging-Guidelines.mdCo-authored-by: NBradford D. Boyle <bboyle@pivotal.io>
Co-authored-by: NShaoqi Bai <sbai@pivotal.io>
(cherry picked from commit 4b48556f)

concourse/pipelines/gpdb_github_release.yml

----
-
-resources:
-- name: gpdb_src
-  type: git
-  source:
-    branch: ((gpdb-git-branch))
-    uri: ((gpdb-git-remote))
-    tag_filter: 6.*
-
-- name: gpdb_release
-  type: github-release
-  source:
-    owner: ((gpdb-release-owner))
-    repository: ((gpdb-release-repository))
-    access_token: ((gpdb-release-access-token))
-
-jobs:
-- name: publish_gpdb_github_release
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      trigger: true
-  - task: gpdb_github_release
-    file: gpdb_src/concourse/tasks/gpdb_github_release.yml
-  - put: gpdb_release
-    params:
-      name: release_artifacts/name
-      tag: release_artifacts/tag
-      body: release_artifacts/body
-      globs:
-        - release_artifacts/*.tar.gz
-        - release_artifacts/*.zip

concourse/pipelines/gpdb_opensource_release.yml

+resource_types:
+- name: gcs
+  type: docker-image
+  source:
+    repository: frodenas/gcs-resource
+
+resources:
+## Image Resources
+- name: gpdb6-centos6-build
+  type: docker-image
+  source:
+    repository: pivotaldata/gpdb6-centos6-build
+    tag: 'latest'
+
+- name: gpdb6-centos7-build
+  type: docker-image
+  source:
+    repository: pivotaldata/gpdb6-centos7-build
+    tag: 'latest'
+
+- name: gpdb6-ubuntu18.04-build
+  type: docker-image
+  source:
+    repository: pivotaldata/gpdb6-ubuntu18.04-build
+    tag: 'latest'
+
+- name: gpdb_src
+  type: git
+  source:
+    branch: ((gpdb-git-branch))
+    uri: ((gpdb-git-remote))
+    tag_filter: ((gpdb-git-tag-filter))
+
+- name: gpdb_release
+  type: github-release
+  source:
+    owner: ((gpdb-release-owner))
+    repository: ((gpdb-release-repository))
+    access_token: ((gpdb-release-access-token))
+
+## gp internal artifacts
+- name: python-centos6
+  type: gcs
+  source:
+    bucket: ((pivotal-gp-internal-artifacts-gcs-bucket))
+    json_key: ((gcs-key))
+    versioned_file: centos6/python-2.7.12.tar.gz
+
+- name: python-centos7
+  type: gcs
+  source:
+    bucket: ((pivotal-gp-internal-artifacts-gcs-bucket))
+    json_key: ((gcs-key))
+    versioned_file: centos7/python-2.7.12.tar.gz
+
+- name: python-ubuntu18.04
+  type: gcs
+  source:
+    bucket: gp-internal-artifacts
+    json_key: ((gcs-key))
+    versioned_file: ubuntu18.04/python-2.7.12.tar.gz
+
+## RHEL6 Resources
+- name: bin_gpdb_centos6
+  type: gcs
+  source:
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    bucket: {{gcs-bucket-intermediates-for-oss}}
+    versioned_file: ((pipeline-name))/bin_gpdb_centos6/bin_gpdb.tar.gz
+
+- name: bin_gpdb_centos6_release
+  type: gcs
+  source:
+    bucket: ((gcs-bucket-for-oss))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: greenplum-oss-server/released/gpdb6/server-(.*)-rhel6_x86_64.tar.gz
+
+## RHEL7 Resources
+- name: bin_gpdb_centos7
+  type: gcs
+  source:
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    bucket: {{gcs-bucket-intermediates-for-oss}}
+    versioned_file: ((pipeline-name))/bin_gpdb_centos7/bin_gpdb.tar.gz
+
+- name: bin_gpdb_centos7_release
+  type: gcs
+  source:
+    bucket: ((gcs-bucket-for-oss))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: greenplum-oss-server/released/gpdb6/server-(.*)-rhel7_x86_64.tar.gz
+
+## Ubuntu18.04 Resources
+- name: bin_gpdb_ubuntu18.04
+  type: gcs
+  source:
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    bucket: {{gcs-bucket-intermediates-for-oss}}
+    versioned_file: ((pipeline-name))/bin_gpdb_ubuntu18.04/bin_gpdb.tar.gz
+
+- name: bin_gpdb_ubuntu18.04_release
+  type: gcs
+  source:
+    bucket: ((gcs-bucket-for-oss))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: greenplum-oss-server/released/gpdb6/server-(.*)-ubuntu18.04_x86_64.tar.gz
+
+jobs:
+- name: compile_gpdb_centos6
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+    - get: gpdb6-centos6-build
+    - get: python-tarball
+      resource: python-centos6
+  - task: compile_gpdb
+    file: gpdb_src/concourse/tasks/compile_gpdb_oss.yml
+    image: gpdb6-centos6-build
+  - put: bin_gpdb_centos6
+    params:
+      file: gpdb_artifacts/bin_gpdb.tar.gz
+
+- name: compile_gpdb_centos7
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+    - get: gpdb6-centos7-build
+    - get: python-tarball
+      resource: python-centos7
+  - task: compile_gpdb
+    image: gpdb6-centos7-build
+    file: gpdb_src/concourse/tasks/compile_gpdb_oss.yml
+  - put: bin_gpdb_centos7
+    params:
+      file: gpdb_artifacts/bin_gpdb.tar.gz
+
+- name: compile_gpdb_ubuntu18.04
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+    - get: gpdb6-ubuntu18.04-build
+    - get: python-tarball
+      resource: python-ubuntu18.04
+  - task: compile_gpdb
+    image: gpdb6-ubuntu18.04-build
+    file: gpdb_src/concourse/tasks/compile_gpdb_oss.yml
+  - put:  bin_gpdb_ubuntu18.04
+    params:
+      file: gpdb_artifacts/bin_gpdb.tar.gz
+
+- name: create_gpdb_tarball_centos6
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+      passed: [compile_gpdb_centos6]
+    - get: bin_gpdb_centos6
+      passed: [compile_gpdb_centos6]
+    - get: gpdb6-centos6-build
+  # Even though we don't ship the QA utils, we still need to strip out non-shipping files
+  - task: separate_qautils_files_for_rc_centos6
+    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
+    image: gpdb6-centos6-build
+    input_mapping:
+      bin_gpdb: bin_gpdb_centos6
+    output_mapping:
+      rc_bin_gpdb: rc_bin_gpdb_rhel6
+    params:
+      QAUTILS_TARBALL: "rc_bin_gpdb/QAUtils-rhel6-x86_64.tar.gz"
+  - put: bin_gpdb_centos6
+    params:
+      file: rc_bin_gpdb_rhel6/bin_gpdb.tar.gz
+
+- name: create_gpdb_tarball_centos7
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+      passed: [compile_gpdb_centos7]
+    - get: bin_gpdb_centos7
+      passed: [compile_gpdb_centos7]
+    - get: gpdb6-centos7-build
+  # Even though we don't ship the QA utils, we still need to strip out non-shipping files
+  - task: separate_qautils_files_for_rc_centos7
+    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
+    image: gpdb6-centos7-build
+    input_mapping:
+      bin_gpdb: bin_gpdb_centos7
+    output_mapping:
+      rc_bin_gpdb: rc_bin_gpdb_rhel7
+    params:
+      QAUTILS_TARBALL: "rc_bin_gpdb/QAUtils-rhel7-x86_64.tar.gz"
+  - put: bin_gpdb_centos7
+    params:
+      file: rc_bin_gpdb_rhel7/bin_gpdb.tar.gz
+
+- name: create_gpdb_tarball_ubuntu18.04
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+      passed: [compile_gpdb_ubuntu18.04]
+    - get: bin_gpdb_ubuntu18.04
+      passed: [compile_gpdb_ubuntu18.04]
+    - get: gpdb6-ubuntu18.04-build
+  # Even though we don't ship the QA utils, we still need to strip out non-shipping files
+  - task: separate_qautils_files_for_rc_ubuntu18.04
+    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
+    image: gpdb6-ubuntu18.04-build
+    input_mapping:
+      bin_gpdb: bin_gpdb_ubuntu18.04
+    output_mapping:
+      rc_bin_gpdb: rc_bin_gpdb_ubuntu18.04
+    params:
+      QAUTILS_TARBALL: "rc_bin_gpdb/QAUtils-ubuntu18.04-amd64.tar.gz"
+  - put: bin_gpdb_ubuntu18.04
+    params:
+      file: rc_bin_gpdb_ubuntu18.04/bin_gpdb.tar.gz
+
+- name: release
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed:
+      - create_gpdb_tarball_centos6
+      - create_gpdb_tarball_centos7
+      - create_gpdb_tarball_ubuntu18.04
+    - get: bin_gpdb_centos6
+      passed: [create_gpdb_tarball_centos6]
+      trigger: true
+    - get: bin_gpdb_centos7
+      passed: [create_gpdb_tarball_centos7]
+      trigger: true
+    - get: bin_gpdb_ubuntu18.04
+      passed: [create_gpdb_tarball_ubuntu18.04]
+      trigger: true
+    - get: gpdb6-centos6-build
+  - task: rename_gpdb_tarball
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source:
+          repository: pivotaldata/gpdb6-centos6-build
+      inputs:
+      - name: gpdb_src
+      - name: bin_gpdb_centos6
+      - name: bin_gpdb_centos7
+      - name: bin_gpdb_ubuntu18.04
+      outputs:
+      - name: releases
+      run:
+        path: bash
+        args:
+        - -ec
+        - |
+          gpdb_semver=$(gpdb_src/getversion | cut -d' ' -f1)
+          cp -v bin_gpdb_centos6/bin_gpdb.tar.gz releases/server-${gpdb_semver}-rhel6_x86_64.tar.gz
+          cp -v bin_gpdb_centos7/bin_gpdb.tar.gz releases/server-${gpdb_semver}-rhel7_x86_64.tar.gz
+          cp -v bin_gpdb_ubuntu18.04/bin_gpdb.tar.gz releases/server-${gpdb_semver}-ubuntu18.04_x86_64.tar.gz
+  - task: verify_gpdb_versions
+    file: gpdb_src/concourse/tasks/verify_gpdb_versions.yml
+  - aggregate:
+      - put: bin_gpdb_centos6_release
+        params:
+          file: "releases/server-*rhel6*.tar.gz"
+      - put: bin_gpdb_centos7_release
+        params:
+          file: "releases/server-*rhel7*.tar.gz"
+      - put: bin_gpdb_ubuntu18.04_release
+        params:
+          file: "releases/server-*ubuntu18.04*.tar.gz"
+
+- name: publish_gpdb_github_release
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      trigger: true
+      passed: [release]
+  - task: gpdb_github_release
+    file: gpdb_src/concourse/tasks/gpdb_github_release.yml
+  - put: gpdb_release
+    params:
+      name: release_artifacts/name
+      tag: release_artifacts/tag
+      body: release_artifacts/body
+      globs:
+        - release_artifacts/*.tar.gz
+        - release_artifacts/*.zip
\ No newline at end of file

concourse/scripts/compile_gpdb_oss.bash

+#!/usr/bin/env bash
+
+set -euox pipefail
+
+fetch_orca_src () {
+    local orca_tag="${1}"
+
+    echo "Downloading greenplum-db/gporca@${orca_tag}"
+    mkdir orca_src
+    wget --quiet --output-document=- "https://github.com/greenplum-db/gporca/archive/${orca_tag}.tar.gz" \
+        | tar xzf - --strip-components=1 --directory=orca_src
+}
+
+build_xerces () {
+    echo "Building Xerces-C"
+    # TODO this works when OUTPUT_DIR is a relative path but fails if an absolute path
+    # TODO this does not work when the output dir is outside the current dir
+    mkdir -p xerces_patch/concourse
+    cp -r orca_src/concourse/xerces-c xerces_patch/concourse
+    cp -r orca_src/patches/ xerces_patch
+    /usr/bin/python xerces_patch/concourse/xerces-c/build_xerces.py --output_dir="/usr/local"
+    rm -rf build
+}
+
+build_orca () {
+    echo "Building orca"
+    orca_src/concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir="/usr/local" --skiptests
+    # FIXME: build_and_test.py prepends `..` to the output_dir so we have to copy in to /usr/local
+    # ourselves
+    cp -a usr/local/* /usr/local
+    rm -rf usr build
+    # RHEL does not include `/usr/local/lib` in the default search path
+    echo "/usr/local/lib" >> /etc/ld.so.conf.d/gpdb.conf
+    ldconfig
+}
+
+install_python () {
+    echo "Installing python"
+    tar xzf python-tarball/python-*.tar.gz -C /opt --strip-components=2
+    export PATH="/opt/python-2.7.12/bin:${PATH}"
+    export PYTHONHOME=/opt/python-2.7.12
+    echo "/opt/python-2.7.12/lib" >> /etc/ld.so.conf.d/gpdb.conf
+    ldconfig
+}
+
+generate_build_number() {
+  pushd gpdb_src
+    #Only if its git repro, add commit SHA as build number
+    # BUILD_NUMBER file is used by getversion file in GPDB to append to version
+    if [ -d .git ] ; then
+      echo "commit:$(git rev-parse HEAD)" > BUILD_NUMBER
+    fi
+  popd
+}
+
+build_gpdb () {
+    local greenplum_install_dir="${1}"
+
+    pushd gpdb_src
+        CC="gcc" CFLAGS="-O3 -fargument-noalias-global -fno-omit-frame-pointer -g" \
+            ./configure \
+                --enable-orca \
+                --with-zstd \
+                --with-gssapi \
+                --with-libxml \
+                --with-perl \
+                --with-python \
+                --with-openssl \
+                --with-pam \
+                --with-ldap \
+                --prefix="${greenplum_install_dir}" \
+                --mandir="${greenplum_install_dir}/man"
+        make -j
+        make install
+    popd
+
+
+    mkdir -p "${greenplum_install_dir}/etc"
+    mkdir -p "${greenplum_install_dir}/include"
+}
+
+include_xerces () {
+    local greenplum_install_dir="${1}"
+
+    echo "Including libxerces-c in greenplum package"
+    cp --archive /usr/local/lib/libxerces-c*.so "${greenplum_install_dir}/lib"
+}
+
+include_gporca () {
+    local greenplum_install_dir="${1}"
+
+    echo "Including libgpopt in greenplum package"
+    cp --archive /usr/local/lib/libgpopt.so.* "${greenplum_install_dir}/lib"
+}
+
+include_python () {
+    local greenplum_install_dir="${1}"
+
+    # Create the python directory to flag to build scripts that python has been handled
+    mkdir -p "${greenplum_install_dir}/ext/python"
+    echo "Copying python from /opt/python-2.7.12 into ${greenplum_install_dir}/ext/python..."
+    cp --archive /opt/python-2.7.12/* "${greenplum_install_dir}/ext/python"
+}
+
+include_libstdcxx () {
+    local greenplum_install_dir="${1}"
+
+    # if this is a platform that we use a non-system toolchain for, we need to
+    # vendor libstdc++
+    if [ -d /opt/gcc-6.4.0 ]; then
+        cp --archive /opt/gcc-6.4.0/lib64/libstdc++.so.6{,.0.22} "${greenplum_install_dir}/lib"
+    fi
+}
+
+include_zstd () {
+    local greenplum_install_dir="${1}"
+    local platform
+    platform="$(python -mplatform)"
+
+    local libdir
+    case "${platform}" in
+    *centos*) libdir=/usr/lib64 ;;
+    *Ubuntu*) libdir=/usr/lib ;;
+    *) return ;;
+    esac
+
+    cp --archive ${libdir}/libzstd.so.1* "${greenplum_install_dir}/lib"
+}
+
+export_gpdb () {
+    local greenplum_install_dir="${1}"
+    local tarball="${2}"
+
+    pushd "${greenplum_install_dir}"
+      (source greenplum_path.sh; python -m compileall -q -x test .)
+      tar -czf "${tarball}" ./*
+    popd
+}
+
+_main () {
+    fetch_orca_src "${ORCA_TAG}"
+
+    if [ -e /opt/gcc_env.sh ]; then
+        . /opt/gcc_env.sh
+    fi
+
+    build_xerces
+    build_orca
+
+    install_python
+
+    generate_build_number
+
+    local greenplum_install_dir="/usr/local/greenplum-db-oss"
+    build_gpdb "${greenplum_install_dir}"
+
+    include_xerces "${greenplum_install_dir}"
+    include_gporca "${greenplum_install_dir}"
+    include_python "${greenplum_install_dir}"
+    include_libstdcxx "${greenplum_install_dir}"
+    include_zstd "${greenplum_install_dir}"
+    export_gpdb "${greenplum_install_dir}" "${PWD}/gpdb_artifacts/bin_gpdb.tar.gz"
+
+}
+
+_main "${@}"

concourse/scripts/verify_gpdb_versions.bash

@@ -3,18 +3,18 @@ set -xeo pipefail
 
 install_greenplum() {
   local bin_gpdb=$1
-  local install_dir=/usr/local/greenplum-db-devel
+  local install_dir="${2}"
 
   rm -rf "$install_dir"
   mkdir -p "$install_dir"
   tar -xf "${bin_gpdb}/bin_gpdb.tar.gz" -C "$install_dir"
-  source "${install_dir}/greenplum_path.sh"
 }
 
 assert_postgres_version_matches() {
-  local gpdb_src_sha=$1
+  local gpdb_src_sha="${1}"
+  local install_dir="${2}"
 
-  if ! readelf --string-dump=.rodata $(command -v postgres) | grep -c "commit:$gpdb_src_sha" > /dev/null ; then
+  if ! readelf --string-dump=.rodata "${install_dir}/bin/postgres" | grep -c "commit:$gpdb_src_sha" > /dev/null ; then
     echo "bin_gpdb version: gpdb_src commit '${gpdb_src_sha}' not found in binary '$(command -v postgres)'. Exiting..."
     exit 1
   fi
@@ -22,10 +22,12 @@ assert_postgres_version_matches() {
 
 yum -d0 -y install git
 
+GREENPLUM_INSTALL_DIR=/usr/local/greenplum-db-devel
 GPDB_SRC_SHA=$(cd gpdb_src && git rev-parse HEAD)
+
 for bin_gpdb in bin_gpdb_{centos{6,7},ubuntu18.04}; do
-  install_greenplum "$bin_gpdb"
-  assert_postgres_version_matches "$GPDB_SRC_SHA"
+  install_greenplum "$bin_gpdb" "${GREENPLUM_INSTALL_DIR}"
+  assert_postgres_version_matches "$GPDB_SRC_SHA" "${GREENPLUM_INSTALL_DIR}"
 done
 
 echo "Release Candidate SHA: ${GPDB_SRC_SHA}"

concourse/tasks/compile_gpdb_oss.yml

+platform: linux
+image_resource:
+  type: docker-image
+inputs:
+  - name: gpdb_src  
+  - name: python-tarball
+outputs:
+  - name: gpdb_artifacts
+run:
+  path: gpdb_src/concourse/scripts/compile_gpdb_oss.bash
+params:  
+  ORCA_TAG: v3.50.0