Modify CI infrastructure and remove no longer needed pipelines/scripts/files

The main change here is that we completely get rid of the ORCA prod
pipeline, as there's no need to create a tag/publish artifacts anymore.

We also now build and test the orca unit tests during the compile step.

Additionally, I've refactored the orca-dev "slack command" pipeline to
use the canonical compile_gpdb.bash script and based the pipeline off
the existing PR pipeline. This simplifies the `test_orca_pipeline.yml`
file significantly and it now doesn't rely on custom scripts.

There are still quite a few files in the concourse directory that are
used by the `test_explain_pipeline.yml` and a couple of pipelines that
are external to this repo. Some of these aren't actively being used and
need to be cleaned up/removed, but I'll leave that for a later time.

Also, uses https for xerces-c download and checks hash to ensure correct
download.

concourse/scripts/compile_gpdb.bash

@@ -209,28 +209,23 @@ function export_gpdb_clients() {
   popd
 }
 
-function fetch_orca_src {
-  local orca_tag="${1}"
-
-  mkdir orca_src
-  wget --quiet --output-document=- "https://github.com/greenplum-db/gporca/archive/${orca_tag}.tar.gz" \
-    | tar xzf - --strip-components=1 --directory=orca_src
-}
-
 function build_xerces()
 {
     OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}"
     mkdir -p xerces_patch/concourse
-    cp -r orca_src/concourse/xerces-c xerces_patch/concourse
-    cp -r orca_src/patches/ xerces_patch
+    cp -r gpdb_src/src/backend/gporca/concourse/xerces-c xerces_patch/concourse
+    cp -r gpdb_src/src/backend/gporca/patches/ xerces_patch
     /usr/bin/python xerces_patch/concourse/xerces-c/build_xerces.py --output_dir=${OUTPUT_DIR}
     rm -rf build
 }
 
-function build_and_test_orca()
+function test_orca()
 {
-    OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}"
-    orca_src/concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR}
+    OUTPUT_DIR="../../../../gpAux/ext/${BLD_ARCH}"
+    pushd ${GPDB_SRC_PATH}/src/backend/gporca
+    concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR}
+    concourse/build_and_test.py --build_type=Debug --output_dir=${OUTPUT_DIR}
+    popd
 }
 
 function _main() {
@@ -239,9 +234,8 @@ function _main() {
   case "${TARGET_OS}" in
     centos|ubuntu|sles)
       prep_env
-      fetch_orca_src "${ORCA_TAG}"
       build_xerces
-      build_and_test_orca
+      test_orca
       install_deps
       link_python
       ;;

concourse/scripts/package_tarball.bash

-#!/bin/bash
-
-set -eux
-
-main() {
-  tar -czf "$dst_tarball" -C "$src_root" .
-}
-
-main "$@"

concourse/scripts/package_tarball.bats

-#!/usr/bin/env bats
-
-TEST_DIR="test_dir"
-
-setup() {
-  mkdir "${TEST_DIR}"
-  cd "${TEST_DIR}"
-
-  export dst_tarball="dst.tar.gz"
-  export src_root="src_root"
-
-  mkdir "${src_root}"
-  touch "${src_root}/"{a,b,c}
-}
-
-teardown() {
-  cd ..
-  rm -rf "${TEST_DIR}"
-}
-
-main() {
-  ../scripts/package_tarball.bash
-}
-
-@test "it fails if any variables are unbound" {
-  unset src_root dst_tarball
-
-  run main
-  [ "${status}" -ne 0 ]
-}
-
-@test "it fails when src_root is a non-existent directory" {
-  export src_root="not-a-file"
-
-  run main
-  [ "${status}" -ne 0 ]
-}
-
-@test "it works when src_root is empty" {
-  rm -rf "${src_root}/*"
-
-  run main
-  [ "${status}" -eq 0 ]
-}
-
-@test "it properly tars file in src_root/" {
-  run main
-  [ "${status}" -eq 0 ]
-
-  run tar tf "$dst_tarball"
-  [ "${status}" -eq 0 ]
-  [[ "${output}" =~ a ]]
-  [[ "${output}" =~ b ]]
-  [[ "${output}" =~ c ]]
-}

concourse/scripts/untar_and_build_gpdb.py

-#!/usr/bin/env python
-import subprocess
-import sys
-import os
-
-action = os.environ['ACTION']
-mode = os.environ['MODE']
-configure_option=os.environ['CONFIGURE_OPTION']
-
-def exec_command(cmd):
-  print "Executing command: {0}".format(cmd)
-  retcode = subprocess.call(cmd, shell=True)
-  if retcode != 0:
-    sys.exit(retcode)
-
-def get_assert_mode():
-    with open('gporca-commits-to-test/gpdb_assert_mode.txt') as fp:
-        assert_mode = fp.read().strip()
-        return assert_mode 
-
-untar_gpdb_cmd = "mkdir -p gpdb_src && tar -xf gpdb_tarball/gpdb*_src.tar.gz -C gpdb_src --strip 1"
-exec_command(untar_gpdb_cmd)
-
-# Default build mode is to enable assert. If the user explicitly specifies
-# 'disable-assert' option, only then don't enable the assert build.
-assert_mode = get_assert_mode()
-if assert_mode == 'enable-cassert':
-    configure_option += ' --enable-cassert'
-elif assert_mode != 'disable-cassert':
-    raise Exception('Unknown build type: (%s). Possible options are \'enable-cassert\' or \'disable-cassert\'')
-
-print "Beginning to {0}".format(action)
-if action == 'build':
-    build_gpdb_cmd = "gpdb_main_src/concourse/scripts/build_gpdb.py --mode={0} --output_dir=gpdb_binary/install --action={1} --configure-option='{2}' --orca-in-gpdb-install-location bin_orca bin_xerces".format(mode, action, configure_option)
-    exec_command(build_gpdb_cmd)
-
-    package_tarball_cmd = "env src_root=gpdb_binary/install dst_tarball=package_tarball/bin_gpdb.tar.gz gpdb_main_src/concourse/scripts/package_tarball.bash"
-    exec_command(package_tarball_cmd)
-
-elif action.startswith('test'):
-    environment_variables = os.environ['BLDWRAP_POSTGRES_CONF_ADDONS']
-
-    test_gpdb_cmd = "env {0} gpdb_main_src/concourse/scripts/build_gpdb.py --mode={1} --gpdb_name=bin_gpdb --action={2} --configure-option='{3}'".format(environment_variables, mode, action, configure_option) 
-    exec_command(test_gpdb_cmd)
-

concourse/tasks/build_with_orca.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: pivotaldata/qp-gpdbdev
-inputs:
-  - name: bin_orca
-  - name: bin_xerces
-  - name: gpdb_src
-outputs:
-  - name: bin_gpdb
-params:
-  MODE:
-  OUTPUTDIR:
-  ACTION:
-  CONFIGURE_OPTION:
-  OTHER_CONFIGS:
-run:
-  path: sh
-  args:
-  - -exc
-  - |
-    gpdb_src/concourse/scripts/build_gpdb.py ${MODE} ${OUTPUTDIR} ${ACTION} ${CONFIGURE_OPTION} ${OTHER_CONFIGS}

concourse/tasks/compile_gpdb.yml

@@ -17,5 +17,4 @@ params:
   BLD_TARGETS:
   OUTPUT_ARTIFACT_DIR: gpdb_artifacts
   CONFIGURE_FLAGS:
-  ORCA_TAG: v3.97.0
   RC_BUILD_TYPE_GCS:

concourse/tasks/package_tarball.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos6
-inputs:
-  - name: gpdb_src
-  - name: bin_gpdb
-outputs:
-  - name: package_tarball
-run:
-  path: gpdb_src/concourse/scripts/package_tarball.bash
-params:
-  src_root: bin_gpdb/install
-  dst_tarball: package_tarball/bin_gpdb.tar.gz

concourse/tasks/test_icg.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: pivotaldata/qp-gpdbdev
-inputs:
-  - name: gpdb_src
-  - name: bin_orca
-  - name: bin_xerces
-  - name: bin_gpdb
-outputs:
-  - name: icg_output
-params:
-  BLDWRAP_POSTGRES_CONF_ADDONS:
-  MODE:
-  ACTION:
-  CONFIGURE_OPTION:
-  GPDB_BINARY:
-run:
-  path: sh
-  args:
-  - -exc
-  - |
-    gpdb_src/concourse/scripts/build_gpdb.py ${MODE} ${ACTION} ${CONFIGURE_OPTION} ${GPDB_BINARY}

concourse/tasks/untar_build_with_orca.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: pivotaldata/qp-gpdbdev
-inputs:
-  - name: gporca-commits-to-test
-  - name: bin_orca
-  - name: bin_xerces
-  - name: bin_gpdb
-    optional: true
-  - name: gpdb_main_src
-  - name: gpdb_tarball
-  - name: explain_test_suite
-    optional: true
-outputs:
-  - name: gpdb_binary
-  - name: package_tarball
-  - name: icg_output
-run:
-  path: gpdb_main_src/concourse/scripts/untar_and_build_gpdb.py
-params:
-  ACTION:
-  MODE:
-  CONFIGURE_OPTION:
-  BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB

concourse/tasks/untar_test_with_planner.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: pivotaldata/qp-gpdbdev
-inputs:
-  - name: gporca-commits-to-test
-  - name: bin_orca
-  - name: bin_xerces
-  - name: bin_gpdb
-  - name: gpdb_main_src
-  - name: gpdb_tarball
-outputs:
-  - name: icg_output
-run:
-  path: gpdb_main_src/concourse/scripts/untar_and_build_gpdb.py
-params:
-  MODE:
-  ACTION:
-  BLDWRAP_POSTGRES_CONF_ADDONS:
-  CONFIGURE_OPTION:

src/backend/gporca/concourse/build_and_test.py

@@ -2,6 +2,7 @@
 
 import optparse
 import os
+import shutil
 import subprocess
 import sys
 
@@ -31,22 +32,19 @@ def install_dependency(dependency_name, output_dir):
     return subprocess.call(
         ["tar -xzf " + dependency_name + "/*.tar.gz -C " + output_dir], shell=True)
 
-def cmake_configure(src_dir, build_type, output_dir, cxx_compiler = None, cxxflags = None, thirty_two_bit = False):
+def cmake_configure(src_dir, build_type, output_dir, cxx_compiler = None, cxxflags = None):
     if os.path.exists("build"):
-        os.removedirs("build")
+        shutil.rmtree("build")
     os.mkdir("build")
     cmake_args = ["cmake",
-                  "-D", "CMAKE_BUILD_TYPE=" + build_type,
-                  "-D", "CMAKE_INSTALL_PREFIX=../" + output_dir]
+                  "-D", "CMAKE_INSTALL_PREFIX=" + output_dir,
+                  "-D", "CMAKE_BUILD_TYPE=" + build_type]
     if cxx_compiler:
         cmake_args.append("-D")
         cmake_args.append("CMAKE_CXX_COMPILER=" + cxx_compiler)
     if cxxflags:
         cmake_args.append("-D")
         cmake_args.append("CMAKE_CXX_FLAGS=" + cxxflags)
-    if thirty_two_bit:
-        cmake_args.append("-D")
-        cmake_args.append("CMAKE_TOOLCHAIN_FILE=../" + src_dir + "/cmake/i386.toolchain.cmake")
 
     cmake_args.append("../" + src_dir)
     cmake_command = " ".join(cmake_args)
@@ -79,16 +77,12 @@ def run_tests():
                             ],
                             cwd="build")
 
-def install():
-    return subprocess.call(["make", "install"], cwd="build")
-
 def main():
     parser = optparse.OptionParser()
     parser.add_option("--build_type", dest="build_type", default="RELEASE")
     parser.add_option("--compiler", dest="compiler")
     parser.add_option("--cxxflags", dest="cxxflags")
     parser.add_option("--output_dir", dest="output_dir", default="install")
-    parser.add_option("--32", dest="thirty_two_bit", default=False)
     parser.add_option("--skiptests", dest="skiptests", action="store_true", default=False)
 
     (options, args) = parser.parse_args()
@@ -96,12 +90,11 @@ def main():
     status = install_dependencies(args, "/usr/local")
     if status:
         return status
-    status = cmake_configure("orca_src",
+    status = cmake_configure("",
                              options.build_type,
                              options.output_dir,
                              options.compiler,
-                             options.cxxflags,
-                             options.thirty_two_bit)
+                             options.cxxflags)
     if status:
         return status
     status = make()
@@ -111,12 +104,6 @@ def main():
         status = run_tests()
         if status:
             return status
-    status = install()
-    if status:
-        return status
-    status = install_dependencies(args, options.output_dir)
-    if status:
-        return status
     return 0
 
 if __name__ == "__main__":

src/backend/gporca/concourse/build_and_test.yml

-platform: linux
-image_resource:
-  type: docker-image
-inputs:
-  - name: bin_xerces
-  - name: orca_src
-
-caches:
-  - path: .ccache
-
-outputs:
-  - name: build_and_test
-params:
-  BUILD_TYPE:
-  OUTPUT_DIR:
-  SKIP_TESTS:
-run:
-  path: sh
-  args:
-  - -exc
-  - |
-    orca_src/concourse/build_and_test.py ${BUILD_TYPE} ${OUTPUT_DIR} ${SKIP_TESTS} bin_xerces

src/backend/gporca/concourse/check_existing_tag.py

-#!/usr/bin/env python
-import subprocess
-import sys
-from packaging import version
-
-
-if __name__ == "__main__":
-  cmd = subprocess.Popen(['env', 'GIT_DIR=orca_src/.git', 'git', 'describe', '--abbrev=0', '--tags'], stdout=subprocess.PIPE)
-  existing_tag = cmd.stdout.read()
-  print ("Latest Tag of ORCA: {0}".format(existing_tag.strip()))
-
-  tag_to_be_published = ''
-  with open('orca_github_release_stage/tag.txt', 'r') as fh:
-    tag_to_be_published = fh.read().strip()
-
-  if tag_to_be_published == '':
-     sys.exit('Unable to read the tag from tag.txt')
-
-  print ("Tag from this commit: {0}".format(tag_to_be_published))
-
-  if (version.parse(existing_tag) >= version.parse(tag_to_be_published)):
-    print ("Tag {0} already present on ORCA repository".format(tag_to_be_published))
-    print ("Please BUMP the ORCA version")
-    sys.exit(1)

src/backend/gporca/concourse/check_existing_tag.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos5
-inputs:
-  - name: orca_src
-  - name: orca_github_release_stage
-outputs:
-run:
-  path: orca_src/concourse/check_existing_tag.py

src/backend/gporca/concourse/clone_remote_repo.py

-#!/usr/bin/env python
-import optparse
-import subprocess
-import sys
-
-def read_commit_info(filename):
-  with open(filename, 'r') as fp:
-    data = fp.readline().strip('\n')
-
-  return data.split(',')[0], data.split(',')[1]
-
-def create_tarball(directory):
-  tar_cmd = 'mkdir -p package_tarball && tar -cvzf package_tarball/{0}.tar.gz {0}'.format(directory)
-  exec_command(tar_cmd)
-
-def exec_command(cmd):
-  print "Executing command: {0}".format(cmd)
-  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
-  output = p.communicate()[0]
-  if p.returncode != 0:
-    sys.exit(p.returncode)
-
-def remote_checkout(branch_or_commit, remote_url, dirname):
-  cmd = "git clone {0} {1}".format(remote_url, dirname)
-  exec_command(cmd)
-  cmd = "pushd {0} && git checkout {1} && popd".format(dirname, branch_or_commit)
-  exec_command(cmd)
-
-
-if __name__ == "__main__":
-  parser = optparse.OptionParser()
-  parser.add_option("--orcaRemoteInfo", dest="orcaRemoteInfo", default="gporca-commits-to-test/orca_remote_info.txt")
-  parser.add_option("--gpdbRemoteInfo", dest="gpdbRemoteInfo", default="gporca-commits-to-test/gpdb_remote_info.txt")
-
-  (options, args) = parser.parse_args()
-
-  # read commit / branch info and repo url
-  orca_branch, orca_remote = read_commit_info(options.orcaRemoteInfo)
-  gpdb_branch, gpdb_remote = read_commit_info(options.gpdbRemoteInfo)
-
-  print "orca commit: {0}, orca remote: {1}".format(orca_branch, orca_remote)
-  print "gpdb commit: {0}, gpdb remote: {1}".format(gpdb_branch, gpdb_remote)
-
-  # checkout the branch from the given repo
-  remote_checkout(orca_branch, orca_remote, 'orca_src')
-  remote_checkout(gpdb_branch, gpdb_remote, 'gpdb_src')
-
-  # create tarballs for orca and gpdb
-  create_tarball('orca_src')
-  create_tarball('gpdb_src')

src/backend/gporca/concourse/clone_remote_repo.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos6
-inputs:
-  - name: orca_main_src
-  - name: gporca-commits-to-test
-outputs:
-  - name: package_tarball
-run:
-  path: orca_main_src/concourse/clone_remote_repo.py

src/backend/gporca/concourse/conanfile.py

-from conans import ConanFile, CMake
-import os
-import subprocess
-
-class OrcaConan(ConanFile):
-    name = "orca"
-    version = os.getenv('orca_version')
-    license = "Apache License v2.0"
-    url = "http://github.com/greenplum-db/conan"
-    settings = "os", "compiler", "build_type", "arch"
-    options = {"shared": [True, False]}
-    default_options = "shared=True"
-    generators = "cmake"
-    description = "Conan file to build orca"
-    # directories to be exported from source
-    exports_sources = (
-            ".gitignore",
-            "CMakeLists.txt",
-            "COPYRIGHT",
-            "LICENSE",
-            "README.md",
-            "cmake/*",
-            "libgpdbcost/*",
-            "libgpopt/*",
-            "libgpos/*",
-            "libnaucrates/*",
-            "server/*"
-            )
-
-    def build_requirements(self):
-    # Normally this would refer to some packages much like requirements
-    # but we overload this to ensure that CMake is present
-        try:
-            vers = subprocess.check_output(["cmake", "--version"]).split()[2]
-            if int(vers.split(".")[0]) < 3:
-                if int(vers.split(".")[1]) < 1:
-                    raise Exception("CMake version 3.1 or higher is required")
-        except OSError as e:
-            if e.errno == os.errno.ENOENT:
-                raise Exception("CMake is not found.  Please ensure the CMake 3.1 or later is installed")
-            else:
-                raise
-
-    def build(self):
-        cmake = CMake(self)
-        cmake_defines = {
-                        }
-        cmake.configure(defs=cmake_defines)
-        cmake.build(target="install")
-
-    def package(self):
-        return

src/backend/gporca/concourse/package_tarball.bash

-#!/bin/bash
-
-set -u -e -x
-
-main() {
-  tar czf "$dst_tarball" -C "$src_root" .
-}
-
-main "$@"

src/backend/gporca/concourse/package_tarball.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos5
-inputs:
-  - name: orca_src
-  - name: build_and_test
-outputs:
-  - name: package_tarball
-run:
-  path: orca_src/concourse/package_tarball.bash
-params:
-  src_root:
-  dst_tarball:

src/backend/gporca/concourse/pipeline.yml

-groups:
-  - name: all
-    jobs:
-    - xerces_centos5
-    - orca_centos5_release
-    - orca_centos5_debug
-    - orca_centos6_debug
-    - orca_centos6_release
-    - orca_centos7_debug
-    - orca_centos7_release
-    - orca_publish_tag
-    - gpdb_icg_with_orca
-    - gpdb_icg_with_planner
-    - build_gpdb
-    - xerces_ubuntu18
-    - orca_ubuntu18_release
-    - orca_ubuntu18_debug
-
-  - name: ReleaseBuilds
-    jobs:
-    - orca_centos5_release
-    - orca_centos6_release
-    - orca_centos7_release
-    - orca_ubuntu18_release
-
-  - name: DebugBuilds
-    jobs:
-    - orca_centos5_debug
-    - orca_centos6_debug
-    - orca_centos7_debug
-    - orca_ubuntu18_debug
-
-  - name: ICG
-    jobs:
-    - gpdb_icg_with_orca
-    - gpdb_icg_with_planner
-
-#############
-# RESOURCES #
-#############
-resources:
-
-- name: gpdb_src
-  type: git
-  source:
-    branch: 5X_STABLE
-    uri: https://github.com/greenplum-db/gpdb
-    paths:
-      - config/orca.m4
-
-- name: centos5-build
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos5
-
-- name: centos6-build
-  type: docker-image
-  source:
-    repository: pivotaldata/gpdb6-centos6-build
-    tag: latest
-
-- name: centos7-build
-  type: docker-image
-  source:
-    repository: pivotaldata/gpdb6-centos7-build
-    tag: latest
-
-- name: ubuntu18.04-build
-  type: docker-image
-  source:
-    repository: pivotaldata/gpdb6-ubuntu18.04-build
-    tag: latest
-
-- name: orca_github_release
-  type: github-release
-  source:
-    user: greenplum-db
-    repository: gporca
-    access_token: {{gporcabot_access_token}}
-
-- name: xerces_patch
-  type: git
-  source:
-    branch: master
-    paths:
-    - patches/xerces-c-gpdb.patch
-    - concourse/xerces-c
-    - concourse/package_tarball.bash
-    uri: https://github.com/greenplum-db/gporca
-
-- name: orca_src
-  type: git
-  source:
-    branch: master
-    uri: https://github.com/greenplum-db/gporca
-
-- name: bin_xerces_centos5
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_xerces_centos5.tar.gz
-
-- name: bin_xerces_ubuntu18
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_xerces_ubuntu18.tar.gz
-
-- name: bin_orca_ubuntu18_debug
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_ubuntu18_debug.tar.gz
-
-- name: bin_orca_ubuntu18_release
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_ubuntu18_release.tar.gz
-
-- name: bin_orca_centos5_debug
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos5_debug.tar.gz
-
-- name: bin_orca_centos5_release
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos5_release.tar.gz
-
-- name: bin_orca_centos6_debug
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos6_debug.tar.gz
-
-- name: bin_orca_centos6_release
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos6_release.tar.gz
-
-- name: bin_orca_centos7_debug
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos7_debug.tar.gz
-
-- name: bin_orca_centos7_release
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos7_release.tar.gz
-
-- name: regression_diffs
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: regression.diffs
-
-- name: regression_out
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: regression.out
-
-- name: bin_gpdb_centos6
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_gpdb_with_orca_centos6.tar.gz
-
-########
-# JOBS #
-########
-jobs:
-- name: orca_publish_tag
-  max_in_flight: 1
-  plan:
-  - in_parallel:
-    - get: orca_src
-      passed:
-      - orca_centos5_release
-      - orca_centos5_debug
-      - orca_centos6_release
-      - orca_centos6_debug
-      - orca_centos7_release
-      - orca_centos7_debug
-      - orca_ubuntu18_release
-      - orca_ubuntu18_debug
-      trigger: true
-    - get: bin_orca_centos5_release
-      passed:
-      - orca_centos5_release
-    - get: bin_orca_centos5_debug
-      passed:
-      - orca_centos5_debug
-  - task: orca_publish_tag
-    file: orca_src/concourse/publish_tag.yml
-  - task: check_existing_tag
-    file: orca_src/concourse/check_existing_tag.yml
-  - task: upload_orca_source_bintray
-    file: orca_src/concourse/upload_orca_source_bintray.yml
-    params:
-      BINTRAY_USER: {{bintray_user}}
-      BINTRAY_REMOTE: {{bintray_remote}}
-      BINTRAY_USER_KEY: {{bintray_user_key}}
-  - put: orca_github_release
-    params:
-      name: orca_github_release_stage/tag.txt
-      tag: orca_github_release_stage/tag.txt
-      commitish: orca_github_release_stage/commit.txt
-      globs:
-        - orca_github_release_stage/bin_orca_centos5_release.tar.gz
-        - orca_github_release_stage/bin_orca_centos5_debug.tar.gz
-
-- name: xerces_centos5
-  max_in_flight: 2
-  plan:
-  - get: xerces_patch
-    trigger: true
-  - get: centos5-build
-  - in_parallel:
-    - task: build
-      file: xerces_patch/concourse/xerces-c/build_xerces.yml
-      image: centos5-build
-  - in_parallel:
-    - task: package_tarball
-      file: xerces_patch/concourse/xerces-c/package_tarball.yml
-      params:
-        dst_tarball: package_tarball/bin_xerces_centos5.tar.gz
-        src_root: build_xerces/install
-  - in_parallel:
-    - put: bin_xerces_centos5
-      params:
-        from: package_tarball/bin_xerces_centos5.tar.gz
-
-- name: xerces_ubuntu18
-  max_in_flight: 2
-  plan:
-  - get: xerces_patch
-    trigger: true
-  - get: ubuntu18.04-build
-  - in_parallel:
-    - task: build
-      file: xerces_patch/concourse/xerces-c/build_xerces.yml
-      image: ubuntu18.04-build
-  - in_parallel:
-    - task: package_tarball
-      file: xerces_patch/concourse/xerces-c/package_tarball.yml
-      params:
-        dst_tarball: package_tarball/bin_xerces_ubuntu18.tar.gz
-        src_root: build_xerces/install
-  - in_parallel:
-    - put: bin_xerces_ubuntu18
-      params:
-        from: package_tarball/bin_xerces_ubuntu18.tar.gz
-
- ## CentOS5 build
-- name: orca_centos5_release
-  max_in_flight: 2
-  plan:
-  - get: centos5-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - task: build_and_test_release
-    image: centos5-build
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: ''
-  - task: package_tarball_release
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos5_release.tar.gz
-  - put: bin_orca_centos5_release
-    params:
-      from: package_tarball/bin_orca_centos5_release.tar.gz
-
-- name: orca_centos5_debug
-  max_in_flight: 2
-  plan:
-  - get: centos5-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - task: build_and_test_debug
-    image: centos5-build
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=Debug
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: ''
-  - task: package_tarball_debug
-    image: centos5-build
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos5_debug.tar.gz
-  - put: bin_orca_centos5_debug
-    params:
-      from: package_tarball/bin_orca_centos5_debug.tar.gz
-
-## CentOS6 build
-- name: orca_centos6_release
-  max_in_flight: 2
-  plan:
-  - get: centos6-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - task: build_and_test_release
-    image: centos6-build
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-  - task: package_tarball_release
-    image: centos6-build
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos6_release.tar.gz
-  - put: bin_orca_centos6_release
-    params:
-      from: package_tarball/bin_orca_centos6_release.tar.gz
-
-- name: orca_centos6_debug
-  max_in_flight: 2
-  plan:
-  - get: centos6-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - task: build_and_test_debug
-    image: centos6-build
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=Debug
-      OUTPUT_DIR: --output_dir=build_and_test/install
-  - task: package_tarball_debug
-    image: centos6-build
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos6_debug.tar.gz
-  - put: bin_orca_centos6_debug
-    params:
-      from: package_tarball/bin_orca_centos6_debug.tar.gz
-
-#CentOS7 build
-- name: orca_centos7_release
-  max_in_flight: 2
-  plan:
-  - get: centos7-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - task: build_and_test_release
-    image: centos7-build
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-  - task: package_tarball_release
-    image: centos7-build
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos7_release.tar.gz
-  - put: bin_orca_centos7_release
-    params:
-      from: package_tarball/bin_orca_centos7_release.tar.gz
-
-- name: orca_centos7_debug
-  max_in_flight: 2
-  plan:
-  - get: centos7-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - task: build_and_test_debug
-    image: centos7-build
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=Debug
-      OUTPUT_DIR: --output_dir=build_and_test/install
-  - task: package_tarball_debug
-    image: centos7-build
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos7_debug.tar.gz
-  - put: bin_orca_centos7_debug
-    params:
-      from: package_tarball/bin_orca_centos7_debug.tar.gz
-
-## Ubuntu18 build
-- name: orca_ubuntu18_release
-  max_in_flight: 2
-  plan:
-  - get: ubuntu18.04-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_ubuntu18
-    passed:
-    - xerces_ubuntu18
-  - task: build_and_test_release
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: ""
-    image: ubuntu18.04-build
-  - task: package_tarball_release
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      dst_tarball: package_tarball/bin_orca_ubuntu18_release.tar.gz
-      src_root: build_and_test/install
-  - put: bin_orca_ubuntu18_release
-    params:
-      from: package_tarball/bin_orca_ubuntu18_release.tar.gz
-
-- name: orca_ubuntu18_debug
-  max_in_flight: 2
-  plan:
-  - get: ubuntu18.04-build
-  - get: orca_src
-    trigger: true
-  - get: bin_xerces
-    resource: bin_xerces_ubuntu18
-    passed:
-    - xerces_ubuntu18
-  - task: build_and_test_debug
-    file: orca_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=Debug
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: ""
-    image: ubuntu18.04-build
-  - task: package_tarball_debug
-    file: orca_src/concourse/package_tarball.yml
-    params:
-      dst_tarball: package_tarball/bin_orca_ubuntu18_debug.tar.gz
-      src_root: build_and_test/install
-  - put: bin_orca_ubuntu18_debug
-    params:
-      from: package_tarball/bin_orca_ubuntu18_debug.tar.gz
-
-- name: build_gpdb
-  plan:
-  - in_parallel:
-    - &get_released_orca
-      get: bin_orca
-      resource: orca_github_release
-      params:
-        globs:
-          - bin_orca_centos5_release.tar.gz
-      passed:
-      - orca_publish_tag
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - xerces_centos5
-    - get: gpdb_src
-      params:
-        submodules: none
-      trigger: true
-  - task: build_with_orca
-    file: gpdb_src/concourse/tasks/build_with_orca.yml
-    params:
-      MODE: "--mode=orca"
-      OUTPUTDIR: "--output_dir=bin_gpdb/install"
-      ACTION: "--action=build"
-      CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions"
-      OTHER_CONFIGS: "--orca-in-gpdb-install-location  bin_orca bin_xerces"
-  - task: package_tarball
-    file: gpdb_src/concourse/tasks/package_tarball.yml
-    params:
-      src_root: bin_gpdb/install
-      dst_tarball: package_tarball/bin_gpdb.tar.gz
-  - put: bin_gpdb_centos6
-    params:
-      file: package_tarball/bin_gpdb.tar.gz
-
-- name: gpdb_icg_with_orca
-  plan:
-  - in_parallel:
-    - <<: *get_released_orca
-      passed:
-      - build_gpdb
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - build_gpdb
-    - get: bin_gpdb
-      resource: bin_gpdb_centos6
-      passed:
-      - build_gpdb
-      trigger: true
-    - get: gpdb_src
-      passed:
-      - build_gpdb
-      params:
-        submodules: none
-  - task: icg_with_orca
-    timeout: 64m
-    file: gpdb_src/concourse/tasks/test_icg.yml
-    params:
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-      MODE: "--mode=orca"
-      ACTION: "--action=test"
-      CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions"
-      GPDB_BINARY: "--gpdb_name=bin_gpdb"
-    on_failure:
-      in_parallel:
-      - put: regression_diffs
-        params:
-          file: icg_output/regression.diffs
-      - put: regression_out
-        params:
-          file: icg_output/regression.out
-
-- name: gpdb_icg_with_planner
-  plan:
-  - in_parallel:
-    - <<: *get_released_orca
-      passed:
-      - build_gpdb
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - build_gpdb
-    - get: bin_gpdb
-      resource: bin_gpdb_centos6
-      passed:
-      - build_gpdb
-      trigger: true
-    - get: gpdb_src
-      passed:
-      - build_gpdb
-      params:
-        submodules: none
-  - task: icg_with_planner
-    timeout: 45m
-    file: gpdb_src/concourse/tasks/test_icg.yml
-    params:
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-      MODE: "--mode=planner"
-      ACTION: "--action=test"
-      CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions"
-      GPDB_BINARY: "--gpdb_name=bin_gpdb"
-    on_failure:
-      in_parallel:
-      - put: regression_diffs
-        params:
-          file: icg_output/regression.diffs
-      - put: regression_out
-        params:
-          file: icg_output/regression.out

src/backend/gporca/concourse/pr_pipeline.yml

-resource_types:
-  - name: pull_request
-    type: docker-image
-    source:
-      repository: jtarchie/pr
-
-resources:
-  - name: orca_pr
-    type: pull_request
-    source:
-      repo: greenplum-db/gporca
-      access_token: {{gporcabot_access_token}}
-      ignore_paths:
-      - README.md
-      - LICENSE
-      - COPYRIGHT
-      - .gitignore
-
-  - name: xerces_patch
-    type: git
-    source:
-      branch: master
-      paths:
-      - patches/xerces-c-gpdb.patch
-      - concourse/xerces-c
-      - concourse/package_tarball.bash
-      uri: https://github.com/greenplum-db/gporca
-  
-  - name: bin_xerces_centos5
-    type: s3
-    source:
-      access_key_id: {{s3_access_key_id}}
-      bucket: {{bucket-name}}
-      region_name: {{aws-region}}
-      secret_access_key: {{s3_secret_access_key}}
-      versioned_file: bin_xerces_centos5.tar.gz
-
-  - name: centos5-build
-    type: docker-image
-    source:
-      repository: yolo/orcadev
-      tag: centos5
-
-
-jobs:
-- name: xerces_centos5
-  public: true
-  max_in_flight: 2
-  plan:
-  - get: xerces_patch
-    trigger: true
-  - get: centos5-build
-  - in_parallel:
-    - task: build
-      file: xerces_patch/concourse/xerces-c/build_xerces.yml
-      image: centos5-build
-  - in_parallel:
-    - task: package_tarball
-      file: xerces_patch/concourse/xerces-c/package_tarball.yml
-      params:
-        dst_tarball: package_tarball/bin_xerces_centos5.tar.gz
-        src_root: build_xerces/install
-  - in_parallel:
-    - put: bin_xerces_centos5
-      params:
-        from: package_tarball/bin_xerces_centos5.tar.gz
-
-- name: compile_and_test_gporca
-  public: true
-  max_in_flight: 2
-  plan:
-  - get: orca_src
-    resource: orca_pr
-    trigger: true
-    version: every
-  - get: bin_xerces
-    resource: bin_xerces_centos5
-    passed:
-    - xerces_centos5
-  - get: centos5-build
-  - put: orca_pr
-    params:
-      path: orca_src
-      status: pending
-  - in_parallel:
-    - task: build_and_test_release
-      file: orca_src/concourse/build_and_test.yml
-      image: centos5-build
-      params:
-        BUILD_TYPE: --build_type=RelWithDebInfo
-        OUTPUT_DIR: --output_dir=build_and_test/install_release
-      timeout: 30m
-    - task: build_and_test_debug
-      file: orca_src/concourse/build_and_test.yml
-      image: centos5-build
-      params:
-        BUILD_TYPE: --build_type=Debug
-        OUTPUT_DIR: --output_dir=build_and_test/install_debug
-      timeout: 30m
-    on_failure: &pr_failure
-      put: orca_pr
-      params:
-        path: orca_src
-        status: failure
-  - put: report_pr_success
-    resource: orca_pr
-    params:
-      path: orca_src
-      status: success

src/backend/gporca/concourse/publish_tag.bash

-#!/bin/bash
-
-
-set -o pipefail
-set -u -e -x
-
-main() {
-  env LC_ALL=C tar tf bin_orca_centos5_release/*.tar.gz | egrep "libgpopt\.so\.[0-9]+.[0-9]+.[0-9]+" | sed 's/\.\/lib\/libgpopt\.so\./v/' > orca_github_release_stage/tag.txt
-  cp -v bin_orca_centos5_release/*.tar.gz orca_github_release_stage/
-  cp -v bin_orca_centos5_debug/*.tar.gz orca_github_release_stage/
-  env GIT_DIR=orca_src/.git git rev-parse HEAD > orca_github_release_stage/commit.txt
-}
-
-main "$@"

src/backend/gporca/concourse/publish_tag.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos5
-inputs:
-  - name: orca_src
-  - name: bin_orca_centos5_release
-  - name: bin_orca_centos5_debug
-outputs:
-  - name: orca_github_release_stage
-run:
-  path: orca_src/concourse/publish_tag.bash

src/backend/gporca/concourse/test_explain_pipeline.yml

-groups: []
-resources:
-- name: centos5-build
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos5
-- name: gporca-commits-to-test
-  type: git
-  source:
-    branch: {{branch_name}}
-    uri: https://github.com/greenplum-db/gporca-pipeline-misc
-- name: orca_main_src  # used for yml task files
-  type: git
-  source:
-    branch: master
-    paths: null
-    uri: https://github.com/greenplum-db/gporca
-- name: orca_baseline_src # used for baseline build
-  type: git
-  source:
-    branch: master
-    paths: null
-    uri: https://github.com/greenplum-db/gporca
-- name: xerces_patch
-  type: git
-  source:
-    branch: master
-    paths:
-    - patches/xerces-c-gpdb.patch
-    - concourse/xerces-c
-    - concourse/package_tarball.bash
-    uri: https://github.com/greenplum-db/gporca
-- name: bin_gpdb_centos6
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_gpdb_with_orca_centos6.tar.gz
-- name: bin_gpdb_baseline_centos6
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_gpdb_baseline_with_orca_centos6.tar.gz
-- name: gpdb_main_src  # used for yml task files
-  type: git
-  source:
-    branch: 5X_STABLE
-    ignore_paths:
-    - README.md
-    - LICENSE
-    - COPYRIGHT
-    - .gitignore
-    uri: https://github.com/greenplum-db/gpdb
-- name: gpdb_baseline_src  # used for baseline build
-  type: git
-  source:
-    branch: 5X_STABLE
-    paths:
-    - config/orca.m4  #trigger only when 5X_STABLE has bumped ORCA versions
-    uri: https://github.com/greenplum-db/gpdb
-- name: gpdb_tarball
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: gpdb_src.tar.gz
-- name: gpdb_baseline_tarball
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: gpdb_baseline_src.tar.gz
-- name: bin_orca_centos5_release
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos5_release.tar.gz
-- name: bin_orca_baseline_centos5_release
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_baseline_centos5_release.tar.gz
-- name: bin_xerces_centos5
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_xerces_centos5.tar.gz
-- name: orca_tarball
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: orca_src.tar.gz
-- name: explain_test_suite
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: {{explain_workload_tar}}
-- name: explain_output
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: explain_output.tar.gz
-- name: explain_output_baseline
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: explain_output_baseline.tar.gz
-- name: explain_test_results
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: {{explain_workload_results}}
-
-resource_types: []
-jobs:
-- name: xerces_centos5
-  max_in_flight: 2
-  plan:
-  - get: xerces_patch
-    trigger: true
-  - get: centos5-build
-  - in_parallel:
-    - task: build
-      file: xerces_patch/concourse/xerces-c/build_xerces.yml
-      image: centos5-build
-  - in_parallel:
-    - task: package_tarball
-      file: xerces_patch/concourse/xerces-c/package_tarball.yml
-      params:
-        dst_tarball: package_tarball/bin_xerces_centos5.tar.gz
-        src_root: build_xerces/install
-  - in_parallel:
-    - put: bin_xerces_centos5
-      params:
-        from: package_tarball/bin_xerces_centos5.tar.gz
-- name: orca_centos5_release
-  max_in_flight: 2
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - clone_repos_to_test
-      version: every
-    - get: orca_main_src
-      passed:
-      - clone_repos_to_test
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - xerces_centos5
-    - get: orca_tarball
-      passed:
-      - clone_repos_to_test
-      trigger: true
-    - get: gpdb_tarball
-      passed:
-      - clone_repos_to_test
-  - task: untar_build_and_test_release
-    file: orca_main_src/concourse/untar_build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: --skiptests
-  - put: bin_orca_centos5_release
-    params:
-      from: package_tarball/bin_orca_centos5_release.tar.gz
-- name: clone_repos_to_test
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      trigger: true
-      version: every
-    - get: orca_main_src
-  - task: clone_repos_to_test
-    file: orca_main_src/concourse/clone_remote_repo.yml
-  - in_parallel:
-    - put: orca_tarball
-      params:
-        from: package_tarball/orca_src.tar.gz
-    - put: gpdb_tarball
-      params:
-        from: package_tarball/gpdb_src.tar.gz
-- name: build_gpdb
-  max_in_flight: 1
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - orca_centos5_release
-    - get: bin_orca
-      passed:
-      - orca_centos5_release
-      resource: bin_orca_centos5_release
-      trigger: true
-      version: every
-    - get: bin_xerces
-      passed:
-      - xerces_centos5
-      resource: bin_xerces_centos5
-    - get: gpdb_tarball
-      passed:
-      - orca_centos5_release
-    - get: gpdb_main_src
-  - task: build_with_orca
-    file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml
-    params:
-      ACTION: build
-      MODE: orca
-      CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-  - put: bin_gpdb_centos6
-    params:
-      from: package_tarball/bin_gpdb.tar.gz
-
-- name: orca_baseline_centos5_release
-  max_in_flight: 2
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - clone_repos_to_test
-      version: every
-    - get: orca_main_src
-      passed:
-      - clone_repos_to_test
-    - get: orca_src
-      resource: orca_baseline_src
-      trigger: true
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - xerces_centos5
-    - get: centos5-build
-  - task: build_and_test_release
-    image: centos5-build
-    file: orca_main_src/concourse/build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: --skiptests
-  - task: package_tarball_release
-    file: orca_main_src/concourse/package_tarball.yml
-    params:
-      src_root: build_and_test/install
-      dst_tarball: package_tarball/bin_orca_centos5_release.tar.gz
-  - put: bin_orca_baseline_centos5_release
-    params:
-      from: package_tarball/bin_orca_centos5_release.tar.gz
-
-- name: build_gpdb_baseline
-  plan:
-  - in_parallel:
-    - get: bin_orca
-      resource: bin_orca_baseline_centos5_release
-      passed:
-      - orca_baseline_centos5_release
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - xerces_centos5
-    - get: gpdb_main_src
-    - get: gpdb_src
-      resource: gpdb_baseline_src
-      trigger: true
-  - task: build_with_orca
-    file: gpdb_main_src/concourse/tasks/build_with_orca.yml
-    params:
-      MODE: "--mode=orca"
-      OUTPUTDIR: "--output_dir=bin_gpdb/install"
-      ACTION: "--action=build"
-      CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions"
-      OTHER_CONFIGS: "--orca-in-gpdb-install-location bin_orca bin_xerces"
-  - task: package_tarball
-    file: gpdb_main_src/concourse/tasks/package_tarball.yml
-    params:
-      src_root: bin_gpdb/install
-      dst_tarball: package_tarball/bin_gpdb.tar.gz
-  - task: package_gpdb_baseline_src
-    config:
-      platform: linux
-      image_resource:
-          type: docker-image
-          source:
-            repository: yolo/orcadev
-            tag: centos6
-      inputs:
-        - name: gpdb_src
-      outputs:
-        - name: package_gpdb_baseline_src
-      run:
-        path: bash
-        args:
-          - -ec
-          - |
-            tar czf "package_gpdb_baseline_src/gpdb_src.tar.gz" -C gpdb_src .
-  - put: bin_gpdb_baseline_centos6
-    params:
-      file: package_tarball/bin_gpdb.tar.gz
-  - put: gpdb_baseline_tarball
-    params:
-      file: package_gpdb_baseline_src/gpdb_src.tar.gz
-- name: test_explain_suite
-  max_in_flight: 1
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - build_gpdb
-    - get: bin_orca
-      passed:
-      - build_gpdb
-      resource: bin_orca_centos5_release
-    - get: bin_xerces
-      passed:
-      - build_gpdb
-      resource: bin_xerces_centos5
-    - get: bin_gpdb
-      passed:
-      - build_gpdb
-      resource: bin_gpdb_centos6
-      trigger: true
-    - get: gpdb_main_src
-      passed:
-      - build_gpdb
-      params:
-        submodules: none
-    - get: gpdb_tarball
-      passed:
-      - build_gpdb
-    - get: explain_test_suite
-  - task: run_explain_suite
-    file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml
-    params:
-      MODE: orca
-      ACTION: test_explain_suite
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-      CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions
-    timeout: 90m
-  - put: explain_output
-    params:
-      file: icg_output/explain_ouput.tar.gz
-- name: test_explain_suite_baseline
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - build_gpdb
-    - get: bin_orca
-      resource: bin_orca_baseline_centos5_release
-      params:
-        globs:
-          - bin_orca_baseline_centos5_release.tar.gz
-      passed:
-      - build_gpdb_baseline
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - build_gpdb_baseline
-    - get: bin_gpdb
-      resource: bin_gpdb_baseline_centos6
-      passed:
-      - build_gpdb_baseline
-      trigger: true
-    - get: gpdb_main_src
-      passed:
-      - build_gpdb_baseline
-      params:
-        submodules: none
-    - get: gpdb_tarball
-      resource: gpdb_baseline_tarball
-      passed:
-      - build_gpdb_baseline
-    - get: explain_test_suite
-  - task: run_explain_suite_baseline
-    timeout: 64m
-    file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml
-    params:
-      MODE: orca
-      ACTION: test_explain_suite
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-      CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions
-    timeout: 90m
-  - put: explain_output_baseline
-    params:
-      file: icg_output/explain_ouput.tar.gz
-- name: diff_explain_results_with_baseline
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - test_explain_suite
-    - get: gpdb_main_src
-      passed:
-      - test_explain_suite
-      params:
-        submodules: none
-    - get: explain_output
-      passed:
-      - test_explain_suite
-      trigger: true
-    - get: explain_output_baseline
-      passed:
-      - test_explain_suite_baseline
-  - task: diff_explain_results_with_baseline
-    timeout: 64m
-    file: gpdb_main_src/concourse/tasks/diff_explain_results_with_baseline.yml
-    ensure:
-      put: explain_test_results
-      params:
-        file: diffs/explain_test_results.tar.gz

src/backend/gporca/concourse/test_orca_pipeline.yml

-groups: []
+## ======================================================================
+## resources
+## ======================================================================
+
+resource_types:
+- name: gcs
+  type: docker-image
+  source:
+    repository: frodenas/gcs-resource
+
 resources:
 - name: gporca-commits-to-test
   type: git
   source:
     branch: master
     uri: https://github.com/greenplum-db/gporca-pipeline-misc
-- name: regression_diffs
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: regression.diffs
-- name: orca_main_src
-  type: git
-  source:
-    branch: master
-    paths: null
-    uri: https://github.com/greenplum-db/gporca
-- name: xerces_patch
-  type: git
-  source:
-    branch: master
-    paths:
-    - patches/xerces-c-gpdb.patch
-    - concourse/xerces-c
-    - concourse/package_tarball.bash
-    uri: https://github.com/greenplum-db/gporca
-- name: regression_out
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: regression.out
-- name: bin_gpdb_centos6
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_gpdb_with_orca_centos6.tar.gz
-- name: gpdb_main_src
-  type: git
-  source:
-    branch: 5X_STABLE
-    ignore_paths:
-    - README.md
-    - LICENSE
-    - COPYRIGHT
-    - .gitignore
-    uri: https://github.com/greenplum-db/gpdb
-- name: gpdb_tarball
-  type: s3
-  source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: gpdb_src.tar.gz
-- name: bin_orca_centos5_debug
-  type: s3
+
+- name: gpdb6-centos7-build
+  type: docker-image
   source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos5_debug.tar.gz
-- name: bin_orca_centos5_release
-  type: s3
+    repository: pivotaldata/gpdb6-centos7-build
+
+- name: gpdb6-centos7-test
+  type: docker-image
   source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_orca_centos5_release.tar.gz
-- name: bin_xerces_centos5
-  type: s3
+    repository: pivotaldata/gpdb6-centos7-test
+
+- name: libquicklz-centos7
+  type: gcs
   source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: bin_xerces_centos5.tar.gz
-- name: orca_tarball
-  type: s3
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: gp-internal-artifacts/centos7/libquicklz-(1\.5\.0-.*)-1.el7.x86_64.rpm
+
+- name: libquicklz-devel-centos7
+  type: gcs
   source:
-    access_key_id: {{s3_access_key_id}}
-    bucket: {{gporca-concourse-bucket-dev}}
-    region_name: us-west-2
-    secret_access_key: {{s3_secret_access_key}}
-    versioned_file: orca_src.tar.gz
-- name: centos5-build
-  type: docker-image
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: gp-internal-artifacts/centos7/libquicklz-devel-(1\.5\.0-.*)-1.el7.x86_64.rpm
+
+- name: python-centos7
+  type: gcs
   source:
-    repository: yolo/orcadev
-    tag: centos5
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: gp-internal-artifacts/centos7/python-(2\.7\.12-.*).tar.gz
+
 
-resource_types: []
 jobs:
-- name: xerces_centos5
-  max_in_flight: 2
-  plan:
-  - get: xerces_patch
-    trigger: true
-  - get: centos5-build
-  - in_parallel:
-    - task: build
-      file: xerces_patch/concourse/xerces-c/build_xerces.yml
-      image: centos5-build
-  - in_parallel:
-    - task: package_tarball
-      file: xerces_patch/concourse/xerces-c/package_tarball.yml
-      params:
-        dst_tarball: package_tarball/bin_xerces_centos5.tar.gz
-        src_root: build_xerces/install
-  - in_parallel:
-    - put: bin_xerces_centos5
-      params:
-        from: package_tarball/bin_xerces_centos5.tar.gz
-- name: orca_centos5_release
-  max_in_flight: 1
+- name: compile_and_test_gpdb
+  max_in_flight: 10
   plan:
   - in_parallel:
     - get: gporca-commits-to-test
-      passed:
-      - clone_repos_to_test
       trigger: true
       version: every
-    - get: orca_main_src
-      passed:
-      - clone_repos_to_test
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - xerces_centos5
-    - get: orca_tarball
-      passed:
-      - clone_repos_to_test
-    - get: gpdb_tarball
-      passed:
-      - clone_repos_to_test
-  - task: untar_build_and_test_release
-    file: orca_main_src/concourse/untar_build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=RelWithDebInfo
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: ''
-  - put: bin_orca_centos5_release
-    params:
-      from: package_tarball/bin_orca_centos5_release.tar.gz
-- name: orca_centos5_debug
-  max_in_flight: 1
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - clone_repos_to_test
-      trigger: true
-      version: every
-    - get: orca_main_src
-      passed:
-      - clone_repos_to_test
-    - get: bin_xerces_centos5
-    - get: bin_xerces
-      resource: bin_xerces_centos5
-      passed:
-      - xerces_centos5
-    - get: orca_tarball
-      passed:
-      - clone_repos_to_test
-  - task: untar_build_and_test_debug
-    file: orca_main_src/concourse/untar_build_and_test.yml
-    params:
-      BUILD_TYPE: --build_type=Debug
-      OUTPUT_DIR: --output_dir=build_and_test/install
-      SKIP_TESTS: ''
-  - in_parallel:
-    - put: bin_orca_centos5_debug
-      params:
-        from: package_tarball/bin_orca_centos5_debug.tar.gz
-- name: clone_repos_to_test
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      trigger: true
-      version: every
-    - get: orca_main_src
-  - task: clone_repos_to_test
-    file: orca_main_src/concourse/clone_remote_repo.yml
-  - in_parallel:
-    - put: orca_tarball
-      params:
-        from: package_tarball/orca_src.tar.gz
-    - put: gpdb_tarball
-      params:
-        from: package_tarball/gpdb_src.tar.gz
-- name: build_gpdb
-  max_in_flight: 1
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - orca_centos5_release
-      - orca_centos5_debug
-      trigger: true
-      version: every
-    - get: bin_orca
-      passed:
-      - orca_centos5_release
-      resource: bin_orca_centos5_release
-    - get: bin_orca_centos5_debug
-      passed:
-      - orca_centos5_debug
-    - get: bin_xerces
-      passed:
-      - xerces_centos5
-      resource: bin_xerces_centos5
-    - get: gpdb_tarball
-      passed:
-      - orca_centos5_release
-    - get: gpdb_main_src
-  - task: build_with_orca
-    file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml
-    params:
-      ACTION: build
-      MODE: orca
-      CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions
-  - put: bin_gpdb_centos6
-    params:
-      from: package_tarball/bin_gpdb.tar.gz
-- name: gpdb_icg_with_orca
-  max_in_flight: 3
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - build_gpdb
-      trigger: true
-    - get: bin_orca
-      passed:
-      - build_gpdb
-      resource: bin_orca_centos5_release
-    - get: bin_xerces
-      passed:
-      - build_gpdb
-      resource: bin_xerces_centos5
-    - get: bin_gpdb
-      passed:
-      - build_gpdb
-      resource: bin_gpdb_centos6
-    - get: gpdb_main_src
-      passed:
-      - build_gpdb
-      params:
-        submodules: none
-    - get: gpdb_tarball
-      passed:
-      - build_gpdb
-  - task: icg_with_orca
-    file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml
-    params:
-      MODE: orca
-      ACTION: test
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-      CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions
-    on_failure:
-      in_parallel:
-      - put: regression_diffs
-        params:
-          file: icg_output/regression.diffs
-      - put: regression_out
-        params:
-          file: icg_output/regression.out
-    timeout: 180m
-- name: gpdb_icg_with_planner
-  max_in_flight: 3
-  plan:
-  - in_parallel:
-    - get: gporca-commits-to-test
-      passed:
-      - build_gpdb
-      trigger: true
-    - get: bin_orca
-      passed:
-      - build_gpdb
-      resource: bin_orca_centos5_release
-    - get: bin_xerces
-      passed:
-      - build_gpdb
-      resource: bin_xerces_centos5
-    - get: bin_gpdb
-      passed:
-      - build_gpdb
-      resource: bin_gpdb_centos6
-    - get: gpdb_main_src
-      passed:
-      - build_gpdb
+    - get: gpdb6-centos7-build
+    - get: gpdb6-centos7-test
+    - get: libquicklz-installer
+      resource: libquicklz-centos7
+    - get: libquicklz-devel-installer
+      resource: libquicklz-devel-centos7
+    - get: python-tarball
+      resource: python-centos7
+
+  - do:
+    - task: init gpdb_src
+      image: gpdb6-centos7-build
+      config:
+        platform: linux
+        run:
+          path: bash
+          args:
+          - -c
+          - |
+            BRANCH=$(cut -d, -f1 gporca-commits-to-test/gpdb_remote_info.txt)
+            REMOTE=$(cut -d, -f2 gporca-commits-to-test/gpdb_remote_info.txt)
+            git clone -b $BRANCH --single-branch $REMOTE gpdb_src &&
+            cd gpdb_src &&
+            git fetch https://github.com/greenplum-db/gpdb.git --tags &&
+            git submodule update --init --recursive
+        inputs: [{ name: gporca-commits-to-test }]
+        outputs: [{ name: gpdb_src }]
+    - task: compile_gpdb_centos7
+      file: gpdb_src/concourse/tasks/compile_gpdb.yml
+      image: gpdb6-centos7-build
       params:
-        submodules: none
-    - get: gpdb_tarball
-      passed:
-      - build_gpdb
-  - task: icg_with_planner
-    file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml
-    params:
-      MODE: planner
-      ACTION: test
-      BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB
-      CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions
+        CONFIGURE_FLAGS: {{configure_flags_with_extensions}}
+        TARGET_OS: centos
+        TARGET_OS_VERSION: 7
+        BLD_TARGETS: "clients loaders"
+      timeout: 30m
 
-    on_failure:
-      in_parallel:
-      - put: regression_diffs
+
+    - in_parallel:
+      - task: icw_planner_centos7
+        file: gpdb_src/concourse/tasks/ic_gpdb.yml
+        image: gpdb6-centos7-test
+        input_mapping:
+          bin_gpdb: gpdb_artifacts
         params:
-          file: icg_output/regression.diffs
-      - put: regression_out
+          MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off' installcheck-world
+          TEST_OS: centos
+          CONFIGURE_FLAGS: {{configure_flags}}
+        timeout: 3h
+
+      - task: icw_gporca_centos7
+        file: gpdb_src/concourse/tasks/ic_gpdb.yml
+        image: gpdb6-centos7-test
+        input_mapping:
+          bin_gpdb: gpdb_artifacts
         params:
-          file: icg_output/regression.out
-    timeout: 180m
+          MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=on' installcheck-world
+          TEST_OS: centos
+          CONFIGURE_FLAGS: {{configure_flags}}
+        timeout: 5h

src/backend/gporca/concourse/untar_build_and_test.py

-#!/usr/bin/env python
-import subprocess
-import os
-import sys
-
-BUILD_TYPE=os.environ['BUILD_TYPE']
-OUTPUT_DIR=os.environ['OUTPUT_DIR']
-SKIP_TESTS=os.environ['SKIP_TESTS']
-if 'Debug' in BUILD_TYPE:
-    path_identifier = 'debug'
-else:
-    path_identifier = 'release'
-
-def exec_command(cmd):
-  print "Executing command: {0}".format(cmd)
-  retcode = subprocess.call(cmd, shell=True)
-  if retcode != 0:
-    sys.exit(retcode)
-
-untar_orca_cmd = "mkdir -p orca_src && tar -xf orca_tarball/orca_src.tar.gz -C orca_src --strip 1"
-exec_command(untar_orca_cmd)
-build_and_test_orca_cmd = "orca_main_src/concourse/build_and_test.py {0} {1} {2} bin_xerces".format(BUILD_TYPE, OUTPUT_DIR, SKIP_TESTS)
-exec_command(build_and_test_orca_cmd)
-
-SRC_DIR=OUTPUT_DIR.split('=')[1]
-package_tarball_cmd = "env dst_tarball=package_tarball/bin_orca_centos5_{0}.tar.gz src_root={1} orca_main_src/concourse/package_tarball.bash".format(path_identifier, SRC_DIR)
-exec_command(package_tarball_cmd)

src/backend/gporca/concourse/untar_build_and_test.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: yolo/orcadev
-    tag: centos5
-inputs:
-  - name: orca_main_src
-  - name: orca_tarball
-  - name: bin_xerces
-
-caches:
-  - path: .ccache
-
-outputs:
-  - name: build_and_test
-  - name: package_tarball
-params:
-  BUILD_TYPE:
-  OUTPUT_DIR:
-  SKIP_TESTS:
-run:
-  path: sh 
-  args:
-  - -exc
-  - |
-    orca_main_src/concourse/untar_build_and_test.py ${BUILD_TYPE} ${OUTPUT_DIR} ${SKIP_TESTS} bin_xerces

src/backend/gporca/concourse/upload_orca_source_bintray.py

-#!/usr/bin/env python
-import subprocess
-import os
-import sys
-import argparse
-import shutil
-from functools import wraps
-
-BASE_WD = os.getcwd()
-ORCA_SRC_DIR=os.path.join(BASE_WD, 'orca_src')
-GPORCA_TAG_FILE='orca_github_release_stage/tag.txt'
-
-def fail_on_error(func):
-  def _check_status(request, *args, **kwargs):
-    response = func(request, *args, **kwargs)
-    if response:
-      return sys.exit(1)
-  return wraps(func)(_check_status)
-
-def exec_command(cmd):
-  print "Executing command: {0}".format(' '.join(cmd))
-  p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-  while True:
-    nextline = p.stdout.readline()
-    if nextline == '' and p.poll() is not None:
-      break
-    sys.stdout.write(nextline)
-    sys.stdout.flush()
-
-  output = p.communicate()[0]
-  exitCode = p.returncode
-  print "Exit Code: {0}\n".format(p.returncode)
-  return p.returncode == 1
-
-def read_orca_version():
-  # read orca version
-  with open(GPORCA_TAG_FILE, 'r') as fh:
-      return fh.read().strip();
-
-@fail_on_error
-def export_orca_src(orca_version):
-  cmd = ["conan", "export", "orca/{0}@gpdb/stable".format(orca_version)]
-  return exec_command(cmd)
-
-@fail_on_error
-def add_conan_remote(bintray_remote_name):
-  # add conan remote
-  cmd = ["conan", "remote",  "add", bintray_remote_name, "https://api.bintray.com/conan/greenplum-db/gpdb-oss"]
-  return exec_command(cmd)
-
-@fail_on_error
-def set_bintray_user(bintray_user, bintray_user_key, bintray_remote):
-  # set bintray account to be used to upload package
-  cmd = ["conan", "user", "-p", bintray_user_key, "-r={0}".format(bintray_remote), bintray_user]
-  return exec_command(cmd)
-
-@fail_on_error
-def upload_orca_src(orca_version):
-  # upload orca source to bintray
-  cmd = ["conan", "upload", "orca/{0}@gpdb/stable".format(orca_version), "--all", "-r=conan-gpdb"]
-  return exec_command(cmd)
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description='Main drived to build and install ORCA using conan')
-  required_arguments = parser.add_argument_group('required arguments')
-  required_arguments.add_argument('--bintrayUser', help='Bintray user name to upload packages', type=str)
-  required_arguments.add_argument('--bintrayUserKey', help='Bintray user key', type=str)
-  required_arguments.add_argument('--bintrayRemote', help='Name of conan remote refering to bintray', type=str)
-
-  args = parser.parse_args()
-  if args.bintrayRemote is None or args.bintrayUser is None or args.bintrayUserKey is None:
-    print "Values for --bintrayRemote, --bintrayUser and --bintrayUserKey argument values are required, some are missing, exiting!"
-    sys.exit(1)
-
-  # setup
-  orca_version = read_orca_version()
-  print "Tag: {0}".format(orca_version)
-  os.chdir(ORCA_SRC_DIR)
-  shutil.copy("concourse/conanfile.py", "conanfile.py")
-
-  # main conan driver
-  export_orca_src(orca_version)
-  add_conan_remote(args.bintrayRemote)
-  set_bintray_user(args.bintrayUser, args.bintrayUserKey, args.bintrayRemote)
-  upload_orca_src(orca_version)

src/backend/gporca/concourse/upload_orca_source_bintray.yml

-platform: linux
-image_resource:
-  type: docker-image
-  source:
-    repository: pivotaldata/conan
-    tag: latest
-inputs:
-  - name: orca_src
-  - name: orca_github_release_stage
-outputs:
-run:
-  path: sh
-  args:
-  - -exc
-  - |
-    python orca_src/concourse/upload_orca_source_bintray.py --bintrayUser=${BINTRAY_USER} --bintrayRemote=${BINTRAY_REMOTE} --bintrayUserKey=${BINTRAY_USER_KEY}
-params:
-  BINTRAY_USER: required
-  BINTRAY_REMOTE: required
-  BINTRAY_USER_KEY: required

src/backend/gporca/concourse/xerces-c/build_xerces.bash

-#!/bin/bash
-
-set -u -e -x
-
-main() {
-        wget http://www.trieuvan.com/apache//xerces/c/3/sources/xerces-c-3.1.2.tar.gz
-        tar -xzf xerces-c-3.1.2.tar.gz
-        cd xerces-c-3.1.2
-        patch -p1 < ../xerces_patch/patches/xerces-c-gpdb.patch
-        ./configure --prefix=$PWD/../install
-        make -j$(nproc_wrapper)
-        make install
-}
-
-nproc_wrapper() {
-        if ! type nproc &> /dev/null; then
-                echo 4
-        else
-                command nproc
-        fi
-}
-
-main "$@"

src/backend/gporca/concourse/xerces-c/build_xerces.py

@@ -7,6 +7,7 @@ import subprocess
 import sys
 import tarfile
 import urllib2
+import hashlib
 
 XERCES_SOURCE_URL = "http://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.2.tar.gz"
 XERCES_SOURCE_DIR = "xerces-c-3.1.2"
@@ -32,6 +33,12 @@ def get_xerces_source():
     local_src = open("xerces_src.tar.gz", "w")
     local_src.write(remote_src.read())
     local_src.close()
+    file_hash = hashlib.sha256(open('xerces_src.tar.gz','rb').read()).hexdigest()
+    actual_hash = ""
+    with open('xerces_patch/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256', 'r') as f:
+        actual_hash = f.read().strip()
+    if file_hash != actual_hash:
+        return 1
     tarball = tarfile.open("xerces_src.tar.gz", "r:gz")
     for item in tarball:
         tarball.extract(item, ".")

src/backend/gporca/concourse/xerces-c/build_xerces.yml

-platform: linux
-image_resource:
-  type: docker-image
-inputs:
-  - name: xerces_patch
-outputs:
-  - name: build_xerces
-run:
-  path: /usr/bin/python
-  args:
-    - xerces_patch/concourse/xerces-c/build_xerces.py
-    - --output_dir=build_xerces/install

src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256

+743bd0a029bf8de56a587c270d97031e0099fe2b7142cef03e0da16e282655a0