diff --git a/concourse/scripts/compile_gpdb.bash b/concourse/scripts/compile_gpdb.bash index c43ae1224830dba34c180ad4f81c2e98c4590987..395de50ac44664b3e2aa3aa54f0dba8886dc961d 100755 --- a/concourse/scripts/compile_gpdb.bash +++ b/concourse/scripts/compile_gpdb.bash @@ -209,28 +209,23 @@ function export_gpdb_clients() { popd } -function fetch_orca_src { - local orca_tag="${1}" - - mkdir orca_src - wget --quiet --output-document=- "https://github.com/greenplum-db/gporca/archive/${orca_tag}.tar.gz" \ - | tar xzf - --strip-components=1 --directory=orca_src -} - function build_xerces() { OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}" mkdir -p xerces_patch/concourse - cp -r orca_src/concourse/xerces-c xerces_patch/concourse - cp -r orca_src/patches/ xerces_patch + cp -r gpdb_src/src/backend/gporca/concourse/xerces-c xerces_patch/concourse + cp -r gpdb_src/src/backend/gporca/patches/ xerces_patch /usr/bin/python xerces_patch/concourse/xerces-c/build_xerces.py --output_dir=${OUTPUT_DIR} rm -rf build } -function build_and_test_orca() +function test_orca() { - OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}" - orca_src/concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR} + OUTPUT_DIR="../../../../gpAux/ext/${BLD_ARCH}" + pushd ${GPDB_SRC_PATH}/src/backend/gporca + concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR} + concourse/build_and_test.py --build_type=Debug --output_dir=${OUTPUT_DIR} + popd } function _main() { @@ -239,9 +234,8 @@ function _main() { case "${TARGET_OS}" in centos|ubuntu|sles) prep_env - fetch_orca_src "${ORCA_TAG}" build_xerces - build_and_test_orca + test_orca install_deps link_python ;; diff --git a/concourse/scripts/package_tarball.bash b/concourse/scripts/package_tarball.bash deleted file mode 100755 index f46170553d1718fb4421b9219c5200eb1729b0ba..0000000000000000000000000000000000000000 --- a/concourse/scripts/package_tarball.bash +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -eux - -main() { - tar -czf "$dst_tarball" -C "$src_root" . -} - -main "$@" diff --git a/concourse/scripts/package_tarball.bats b/concourse/scripts/package_tarball.bats deleted file mode 100755 index 2297c5ccafa02cd62ba77e55959661df82a2e1e7..0000000000000000000000000000000000000000 --- a/concourse/scripts/package_tarball.bats +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bats - -TEST_DIR="test_dir" - -setup() { - mkdir "${TEST_DIR}" - cd "${TEST_DIR}" - - export dst_tarball="dst.tar.gz" - export src_root="src_root" - - mkdir "${src_root}" - touch "${src_root}/"{a,b,c} -} - -teardown() { - cd .. - rm -rf "${TEST_DIR}" -} - -main() { - ../scripts/package_tarball.bash -} - -@test "it fails if any variables are unbound" { - unset src_root dst_tarball - - run main - [ "${status}" -ne 0 ] -} - -@test "it fails when src_root is a non-existent directory" { - export src_root="not-a-file" - - run main - [ "${status}" -ne 0 ] -} - -@test "it works when src_root is empty" { - rm -rf "${src_root}/*" - - run main - [ "${status}" -eq 0 ] -} - -@test "it properly tars file in src_root/" { - run main - [ "${status}" -eq 0 ] - - run tar tf "$dst_tarball" - [ "${status}" -eq 0 ] - [[ "${output}" =~ a ]] - [[ "${output}" =~ b ]] - [[ "${output}" =~ c ]] -} diff --git a/concourse/scripts/untar_and_build_gpdb.py b/concourse/scripts/untar_and_build_gpdb.py deleted file mode 100755 index 90b2aed6ad0756bf9d60d5dbbe65ef943a03331d..0000000000000000000000000000000000000000 --- a/concourse/scripts/untar_and_build_gpdb.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import subprocess -import sys -import os - -action = os.environ['ACTION'] -mode = os.environ['MODE'] -configure_option=os.environ['CONFIGURE_OPTION'] - -def exec_command(cmd): - print "Executing command: {0}".format(cmd) - retcode = subprocess.call(cmd, shell=True) - if retcode != 0: - sys.exit(retcode) - -def get_assert_mode(): - with open('gporca-commits-to-test/gpdb_assert_mode.txt') as fp: - assert_mode = fp.read().strip() - return assert_mode - -untar_gpdb_cmd = "mkdir -p gpdb_src && tar -xf gpdb_tarball/gpdb*_src.tar.gz -C gpdb_src --strip 1" -exec_command(untar_gpdb_cmd) - -# Default build mode is to enable assert. If the user explicitly specifies -# 'disable-assert' option, only then don't enable the assert build. -assert_mode = get_assert_mode() -if assert_mode == 'enable-cassert': - configure_option += ' --enable-cassert' -elif assert_mode != 'disable-cassert': - raise Exception('Unknown build type: (%s). Possible options are \'enable-cassert\' or \'disable-cassert\'') - -print "Beginning to {0}".format(action) -if action == 'build': - build_gpdb_cmd = "gpdb_main_src/concourse/scripts/build_gpdb.py --mode={0} --output_dir=gpdb_binary/install --action={1} --configure-option='{2}' --orca-in-gpdb-install-location bin_orca bin_xerces".format(mode, action, configure_option) - exec_command(build_gpdb_cmd) - - package_tarball_cmd = "env src_root=gpdb_binary/install dst_tarball=package_tarball/bin_gpdb.tar.gz gpdb_main_src/concourse/scripts/package_tarball.bash" - exec_command(package_tarball_cmd) - -elif action.startswith('test'): - environment_variables = os.environ['BLDWRAP_POSTGRES_CONF_ADDONS'] - - test_gpdb_cmd = "env {0} gpdb_main_src/concourse/scripts/build_gpdb.py --mode={1} --gpdb_name=bin_gpdb --action={2} --configure-option='{3}'".format(environment_variables, mode, action, configure_option) - exec_command(test_gpdb_cmd) - diff --git a/concourse/tasks/build_with_orca.yml b/concourse/tasks/build_with_orca.yml deleted file mode 100644 index 276450de5a1cd1df44aa27f58fa7af641719666a..0000000000000000000000000000000000000000 --- a/concourse/tasks/build_with_orca.yml +++ /dev/null @@ -1,23 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: pivotaldata/qp-gpdbdev -inputs: - - name: bin_orca - - name: bin_xerces - - name: gpdb_src -outputs: - - name: bin_gpdb -params: - MODE: - OUTPUTDIR: - ACTION: - CONFIGURE_OPTION: - OTHER_CONFIGS: -run: - path: sh - args: - - -exc - - | - gpdb_src/concourse/scripts/build_gpdb.py ${MODE} ${OUTPUTDIR} ${ACTION} ${CONFIGURE_OPTION} ${OTHER_CONFIGS} diff --git a/concourse/tasks/compile_gpdb.yml b/concourse/tasks/compile_gpdb.yml index 263dd50916d523a77a2f2e34e15bbfebb6e4081e..0673abefaef2b3e71d114c96ad7a5b7bca084a84 100644 --- a/concourse/tasks/compile_gpdb.yml +++ b/concourse/tasks/compile_gpdb.yml @@ -17,5 +17,4 @@ params: BLD_TARGETS: OUTPUT_ARTIFACT_DIR: gpdb_artifacts CONFIGURE_FLAGS: - ORCA_TAG: v3.97.0 RC_BUILD_TYPE_GCS: diff --git a/concourse/tasks/package_tarball.yml b/concourse/tasks/package_tarball.yml deleted file mode 100644 index a4b06d112090dbb2f174b1a9e5f4f39a0d1cc752..0000000000000000000000000000000000000000 --- a/concourse/tasks/package_tarball.yml +++ /dev/null @@ -1,16 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos6 -inputs: - - name: gpdb_src - - name: bin_gpdb -outputs: - - name: package_tarball -run: - path: gpdb_src/concourse/scripts/package_tarball.bash -params: - src_root: bin_gpdb/install - dst_tarball: package_tarball/bin_gpdb.tar.gz diff --git a/concourse/tasks/test_icg.yml b/concourse/tasks/test_icg.yml deleted file mode 100644 index b9123fa202d1176196ba8d068f33b196b11f8429..0000000000000000000000000000000000000000 --- a/concourse/tasks/test_icg.yml +++ /dev/null @@ -1,24 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: pivotaldata/qp-gpdbdev -inputs: - - name: gpdb_src - - name: bin_orca - - name: bin_xerces - - name: bin_gpdb -outputs: - - name: icg_output -params: - BLDWRAP_POSTGRES_CONF_ADDONS: - MODE: - ACTION: - CONFIGURE_OPTION: - GPDB_BINARY: -run: - path: sh - args: - - -exc - - | - gpdb_src/concourse/scripts/build_gpdb.py ${MODE} ${ACTION} ${CONFIGURE_OPTION} ${GPDB_BINARY} diff --git a/concourse/tasks/untar_build_with_orca.yml b/concourse/tasks/untar_build_with_orca.yml deleted file mode 100644 index 7b65b90861796e6f5a21357a6692428c691c373c..0000000000000000000000000000000000000000 --- a/concourse/tasks/untar_build_with_orca.yml +++ /dev/null @@ -1,26 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: pivotaldata/qp-gpdbdev -inputs: - - name: gporca-commits-to-test - - name: bin_orca - - name: bin_xerces - - name: bin_gpdb - optional: true - - name: gpdb_main_src - - name: gpdb_tarball - - name: explain_test_suite - optional: true -outputs: - - name: gpdb_binary - - name: package_tarball - - name: icg_output -run: - path: gpdb_main_src/concourse/scripts/untar_and_build_gpdb.py -params: - ACTION: - MODE: - CONFIGURE_OPTION: - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB diff --git a/concourse/tasks/untar_test_with_planner.yml b/concourse/tasks/untar_test_with_planner.yml deleted file mode 100644 index fa67493da3b00072b4a47214e31e17ff766c5b16..0000000000000000000000000000000000000000 --- a/concourse/tasks/untar_test_with_planner.yml +++ /dev/null @@ -1,21 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: pivotaldata/qp-gpdbdev -inputs: - - name: gporca-commits-to-test - - name: bin_orca - - name: bin_xerces - - name: bin_gpdb - - name: gpdb_main_src - - name: gpdb_tarball -outputs: - - name: icg_output -run: - path: gpdb_main_src/concourse/scripts/untar_and_build_gpdb.py -params: - MODE: - ACTION: - BLDWRAP_POSTGRES_CONF_ADDONS: - CONFIGURE_OPTION: diff --git a/src/backend/gporca/concourse/build_and_test.py b/src/backend/gporca/concourse/build_and_test.py index 587c6bfecb0ee4840d96231ebde64d91255663d4..bd81aaeb439933cabb713ec4eb183e6f4e0a41f5 100755 --- a/src/backend/gporca/concourse/build_and_test.py +++ b/src/backend/gporca/concourse/build_and_test.py @@ -2,6 +2,7 @@ import optparse import os +import shutil import subprocess import sys @@ -31,22 +32,19 @@ def install_dependency(dependency_name, output_dir): return subprocess.call( ["tar -xzf " + dependency_name + "/*.tar.gz -C " + output_dir], shell=True) -def cmake_configure(src_dir, build_type, output_dir, cxx_compiler = None, cxxflags = None, thirty_two_bit = False): +def cmake_configure(src_dir, build_type, output_dir, cxx_compiler = None, cxxflags = None): if os.path.exists("build"): - os.removedirs("build") + shutil.rmtree("build") os.mkdir("build") cmake_args = ["cmake", - "-D", "CMAKE_BUILD_TYPE=" + build_type, - "-D", "CMAKE_INSTALL_PREFIX=../" + output_dir] + "-D", "CMAKE_INSTALL_PREFIX=" + output_dir, + "-D", "CMAKE_BUILD_TYPE=" + build_type] if cxx_compiler: cmake_args.append("-D") cmake_args.append("CMAKE_CXX_COMPILER=" + cxx_compiler) if cxxflags: cmake_args.append("-D") cmake_args.append("CMAKE_CXX_FLAGS=" + cxxflags) - if thirty_two_bit: - cmake_args.append("-D") - cmake_args.append("CMAKE_TOOLCHAIN_FILE=../" + src_dir + "/cmake/i386.toolchain.cmake") cmake_args.append("../" + src_dir) cmake_command = " ".join(cmake_args) @@ -79,16 +77,12 @@ def run_tests(): ], cwd="build") -def install(): - return subprocess.call(["make", "install"], cwd="build") - def main(): parser = optparse.OptionParser() parser.add_option("--build_type", dest="build_type", default="RELEASE") parser.add_option("--compiler", dest="compiler") parser.add_option("--cxxflags", dest="cxxflags") parser.add_option("--output_dir", dest="output_dir", default="install") - parser.add_option("--32", dest="thirty_two_bit", default=False) parser.add_option("--skiptests", dest="skiptests", action="store_true", default=False) (options, args) = parser.parse_args() @@ -96,12 +90,11 @@ def main(): status = install_dependencies(args, "/usr/local") if status: return status - status = cmake_configure("orca_src", + status = cmake_configure("", options.build_type, options.output_dir, options.compiler, - options.cxxflags, - options.thirty_two_bit) + options.cxxflags) if status: return status status = make() @@ -111,12 +104,6 @@ def main(): status = run_tests() if status: return status - status = install() - if status: - return status - status = install_dependencies(args, options.output_dir) - if status: - return status return 0 if __name__ == "__main__": diff --git a/src/backend/gporca/concourse/build_and_test.yml b/src/backend/gporca/concourse/build_and_test.yml deleted file mode 100644 index fb92bb43f55535356b3812683daab7bbadd85aaa..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/build_and_test.yml +++ /dev/null @@ -1,22 +0,0 @@ -platform: linux -image_resource: - type: docker-image -inputs: - - name: bin_xerces - - name: orca_src - -caches: - - path: .ccache - -outputs: - - name: build_and_test -params: - BUILD_TYPE: - OUTPUT_DIR: - SKIP_TESTS: -run: - path: sh - args: - - -exc - - | - orca_src/concourse/build_and_test.py ${BUILD_TYPE} ${OUTPUT_DIR} ${SKIP_TESTS} bin_xerces diff --git a/src/backend/gporca/concourse/check_existing_tag.py b/src/backend/gporca/concourse/check_existing_tag.py deleted file mode 100755 index af1b386221bfa72dbbe7d3954b1fa101d9bb944f..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/check_existing_tag.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -import subprocess -import sys -from packaging import version - - -if __name__ == "__main__": - cmd = subprocess.Popen(['env', 'GIT_DIR=orca_src/.git', 'git', 'describe', '--abbrev=0', '--tags'], stdout=subprocess.PIPE) - existing_tag = cmd.stdout.read() - print ("Latest Tag of ORCA: {0}".format(existing_tag.strip())) - - tag_to_be_published = '' - with open('orca_github_release_stage/tag.txt', 'r') as fh: - tag_to_be_published = fh.read().strip() - - if tag_to_be_published == '': - sys.exit('Unable to read the tag from tag.txt') - - print ("Tag from this commit: {0}".format(tag_to_be_published)) - - if (version.parse(existing_tag) >= version.parse(tag_to_be_published)): - print ("Tag {0} already present on ORCA repository".format(tag_to_be_published)) - print ("Please BUMP the ORCA version") - sys.exit(1) diff --git a/src/backend/gporca/concourse/check_existing_tag.yml b/src/backend/gporca/concourse/check_existing_tag.yml deleted file mode 100644 index c696f626216a0db4cbf78f6161e4f9f5b9f21aad..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/check_existing_tag.yml +++ /dev/null @@ -1,12 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 -inputs: - - name: orca_src - - name: orca_github_release_stage -outputs: -run: - path: orca_src/concourse/check_existing_tag.py diff --git a/src/backend/gporca/concourse/clone_remote_repo.py b/src/backend/gporca/concourse/clone_remote_repo.py deleted file mode 100755 index e65364206ecee2fb69dbb18c29b311f4a4cef66f..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/clone_remote_repo.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -import optparse -import subprocess -import sys - -def read_commit_info(filename): - with open(filename, 'r') as fp: - data = fp.readline().strip('\n') - - return data.split(',')[0], data.split(',')[1] - -def create_tarball(directory): - tar_cmd = 'mkdir -p package_tarball && tar -cvzf package_tarball/{0}.tar.gz {0}'.format(directory) - exec_command(tar_cmd) - -def exec_command(cmd): - print "Executing command: {0}".format(cmd) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) - output = p.communicate()[0] - if p.returncode != 0: - sys.exit(p.returncode) - -def remote_checkout(branch_or_commit, remote_url, dirname): - cmd = "git clone {0} {1}".format(remote_url, dirname) - exec_command(cmd) - cmd = "pushd {0} && git checkout {1} && popd".format(dirname, branch_or_commit) - exec_command(cmd) - - -if __name__ == "__main__": - parser = optparse.OptionParser() - parser.add_option("--orcaRemoteInfo", dest="orcaRemoteInfo", default="gporca-commits-to-test/orca_remote_info.txt") - parser.add_option("--gpdbRemoteInfo", dest="gpdbRemoteInfo", default="gporca-commits-to-test/gpdb_remote_info.txt") - - (options, args) = parser.parse_args() - - # read commit / branch info and repo url - orca_branch, orca_remote = read_commit_info(options.orcaRemoteInfo) - gpdb_branch, gpdb_remote = read_commit_info(options.gpdbRemoteInfo) - - print "orca commit: {0}, orca remote: {1}".format(orca_branch, orca_remote) - print "gpdb commit: {0}, gpdb remote: {1}".format(gpdb_branch, gpdb_remote) - - # checkout the branch from the given repo - remote_checkout(orca_branch, orca_remote, 'orca_src') - remote_checkout(gpdb_branch, gpdb_remote, 'gpdb_src') - - # create tarballs for orca and gpdb - create_tarball('orca_src') - create_tarball('gpdb_src') diff --git a/src/backend/gporca/concourse/clone_remote_repo.yml b/src/backend/gporca/concourse/clone_remote_repo.yml deleted file mode 100644 index 11ac5d94c035b861850edd60636a5a1a82f8c3e5..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/clone_remote_repo.yml +++ /dev/null @@ -1,13 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos6 -inputs: - - name: orca_main_src - - name: gporca-commits-to-test -outputs: - - name: package_tarball -run: - path: orca_main_src/concourse/clone_remote_repo.py diff --git a/src/backend/gporca/concourse/conanfile.py b/src/backend/gporca/concourse/conanfile.py deleted file mode 100644 index 21be7588dc7c9a292540dd695d11524c6042f382..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/conanfile.py +++ /dev/null @@ -1,52 +0,0 @@ -from conans import ConanFile, CMake -import os -import subprocess - -class OrcaConan(ConanFile): - name = "orca" - version = os.getenv('orca_version') - license = "Apache License v2.0" - url = "http://github.com/greenplum-db/conan" - settings = "os", "compiler", "build_type", "arch" - options = {"shared": [True, False]} - default_options = "shared=True" - generators = "cmake" - description = "Conan file to build orca" - # directories to be exported from source - exports_sources = ( - ".gitignore", - "CMakeLists.txt", - "COPYRIGHT", - "LICENSE", - "README.md", - "cmake/*", - "libgpdbcost/*", - "libgpopt/*", - "libgpos/*", - "libnaucrates/*", - "server/*" - ) - - def build_requirements(self): - # Normally this would refer to some packages much like requirements - # but we overload this to ensure that CMake is present - try: - vers = subprocess.check_output(["cmake", "--version"]).split()[2] - if int(vers.split(".")[0]) < 3: - if int(vers.split(".")[1]) < 1: - raise Exception("CMake version 3.1 or higher is required") - except OSError as e: - if e.errno == os.errno.ENOENT: - raise Exception("CMake is not found. Please ensure the CMake 3.1 or later is installed") - else: - raise - - def build(self): - cmake = CMake(self) - cmake_defines = { - } - cmake.configure(defs=cmake_defines) - cmake.build(target="install") - - def package(self): - return diff --git a/src/backend/gporca/concourse/package_tarball.bash b/src/backend/gporca/concourse/package_tarball.bash deleted file mode 100755 index f5ea4f670ef1499d87068d05f8728805e522a3bf..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/package_tarball.bash +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -u -e -x - -main() { - tar czf "$dst_tarball" -C "$src_root" . -} - -main "$@" diff --git a/src/backend/gporca/concourse/package_tarball.yml b/src/backend/gporca/concourse/package_tarball.yml deleted file mode 100644 index 562cdd95b90156fcbc19c525deb24c44782b11a9..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/package_tarball.yml +++ /dev/null @@ -1,16 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 -inputs: - - name: orca_src - - name: build_and_test -outputs: - - name: package_tarball -run: - path: orca_src/concourse/package_tarball.bash -params: - src_root: - dst_tarball: diff --git a/src/backend/gporca/concourse/pipeline.yml b/src/backend/gporca/concourse/pipeline.yml deleted file mode 100644 index 82fcc86a292638c0aa0b0a92283bc08e0c28f9b8..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/pipeline.yml +++ /dev/null @@ -1,624 +0,0 @@ -groups: - - name: all - jobs: - - xerces_centos5 - - orca_centos5_release - - orca_centos5_debug - - orca_centos6_debug - - orca_centos6_release - - orca_centos7_debug - - orca_centos7_release - - orca_publish_tag - - gpdb_icg_with_orca - - gpdb_icg_with_planner - - build_gpdb - - xerces_ubuntu18 - - orca_ubuntu18_release - - orca_ubuntu18_debug - - - name: ReleaseBuilds - jobs: - - orca_centos5_release - - orca_centos6_release - - orca_centos7_release - - orca_ubuntu18_release - - - name: DebugBuilds - jobs: - - orca_centos5_debug - - orca_centos6_debug - - orca_centos7_debug - - orca_ubuntu18_debug - - - name: ICG - jobs: - - gpdb_icg_with_orca - - gpdb_icg_with_planner - -############# -# RESOURCES # -############# -resources: - -- name: gpdb_src - type: git - source: - branch: 5X_STABLE - uri: https://github.com/greenplum-db/gpdb - paths: - - config/orca.m4 - -- name: centos5-build - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 - -- name: centos6-build - type: docker-image - source: - repository: pivotaldata/gpdb6-centos6-build - tag: latest - -- name: centos7-build - type: docker-image - source: - repository: pivotaldata/gpdb6-centos7-build - tag: latest - -- name: ubuntu18.04-build - type: docker-image - source: - repository: pivotaldata/gpdb6-ubuntu18.04-build - tag: latest - -- name: orca_github_release - type: github-release - source: - user: greenplum-db - repository: gporca - access_token: {{gporcabot_access_token}} - -- name: xerces_patch - type: git - source: - branch: master - paths: - - patches/xerces-c-gpdb.patch - - concourse/xerces-c - - concourse/package_tarball.bash - uri: https://github.com/greenplum-db/gporca - -- name: orca_src - type: git - source: - branch: master - uri: https://github.com/greenplum-db/gporca - -- name: bin_xerces_centos5 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_xerces_centos5.tar.gz - -- name: bin_xerces_ubuntu18 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_xerces_ubuntu18.tar.gz - -- name: bin_orca_ubuntu18_debug - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_ubuntu18_debug.tar.gz - -- name: bin_orca_ubuntu18_release - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_ubuntu18_release.tar.gz - -- name: bin_orca_centos5_debug - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos5_debug.tar.gz - -- name: bin_orca_centos5_release - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos5_release.tar.gz - -- name: bin_orca_centos6_debug - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos6_debug.tar.gz - -- name: bin_orca_centos6_release - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos6_release.tar.gz - -- name: bin_orca_centos7_debug - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos7_debug.tar.gz - -- name: bin_orca_centos7_release - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos7_release.tar.gz - -- name: regression_diffs - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: regression.diffs - -- name: regression_out - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: regression.out - -- name: bin_gpdb_centos6 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_gpdb_with_orca_centos6.tar.gz - -######## -# JOBS # -######## -jobs: -- name: orca_publish_tag - max_in_flight: 1 - plan: - - in_parallel: - - get: orca_src - passed: - - orca_centos5_release - - orca_centos5_debug - - orca_centos6_release - - orca_centos6_debug - - orca_centos7_release - - orca_centos7_debug - - orca_ubuntu18_release - - orca_ubuntu18_debug - trigger: true - - get: bin_orca_centos5_release - passed: - - orca_centos5_release - - get: bin_orca_centos5_debug - passed: - - orca_centos5_debug - - task: orca_publish_tag - file: orca_src/concourse/publish_tag.yml - - task: check_existing_tag - file: orca_src/concourse/check_existing_tag.yml - - task: upload_orca_source_bintray - file: orca_src/concourse/upload_orca_source_bintray.yml - params: - BINTRAY_USER: {{bintray_user}} - BINTRAY_REMOTE: {{bintray_remote}} - BINTRAY_USER_KEY: {{bintray_user_key}} - - put: orca_github_release - params: - name: orca_github_release_stage/tag.txt - tag: orca_github_release_stage/tag.txt - commitish: orca_github_release_stage/commit.txt - globs: - - orca_github_release_stage/bin_orca_centos5_release.tar.gz - - orca_github_release_stage/bin_orca_centos5_debug.tar.gz - -- name: xerces_centos5 - max_in_flight: 2 - plan: - - get: xerces_patch - trigger: true - - get: centos5-build - - in_parallel: - - task: build - file: xerces_patch/concourse/xerces-c/build_xerces.yml - image: centos5-build - - in_parallel: - - task: package_tarball - file: xerces_patch/concourse/xerces-c/package_tarball.yml - params: - dst_tarball: package_tarball/bin_xerces_centos5.tar.gz - src_root: build_xerces/install - - in_parallel: - - put: bin_xerces_centos5 - params: - from: package_tarball/bin_xerces_centos5.tar.gz - -- name: xerces_ubuntu18 - max_in_flight: 2 - plan: - - get: xerces_patch - trigger: true - - get: ubuntu18.04-build - - in_parallel: - - task: build - file: xerces_patch/concourse/xerces-c/build_xerces.yml - image: ubuntu18.04-build - - in_parallel: - - task: package_tarball - file: xerces_patch/concourse/xerces-c/package_tarball.yml - params: - dst_tarball: package_tarball/bin_xerces_ubuntu18.tar.gz - src_root: build_xerces/install - - in_parallel: - - put: bin_xerces_ubuntu18 - params: - from: package_tarball/bin_xerces_ubuntu18.tar.gz - - ## CentOS5 build -- name: orca_centos5_release - max_in_flight: 2 - plan: - - get: centos5-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - task: build_and_test_release - image: centos5-build - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: '' - - task: package_tarball_release - file: orca_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos5_release.tar.gz - - put: bin_orca_centos5_release - params: - from: package_tarball/bin_orca_centos5_release.tar.gz - -- name: orca_centos5_debug - max_in_flight: 2 - plan: - - get: centos5-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - task: build_and_test_debug - image: centos5-build - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=Debug - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: '' - - task: package_tarball_debug - image: centos5-build - file: orca_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos5_debug.tar.gz - - put: bin_orca_centos5_debug - params: - from: package_tarball/bin_orca_centos5_debug.tar.gz - -## CentOS6 build -- name: orca_centos6_release - max_in_flight: 2 - plan: - - get: centos6-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - task: build_and_test_release - image: centos6-build - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - - task: package_tarball_release - image: centos6-build - file: orca_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos6_release.tar.gz - - put: bin_orca_centos6_release - params: - from: package_tarball/bin_orca_centos6_release.tar.gz - -- name: orca_centos6_debug - max_in_flight: 2 - plan: - - get: centos6-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - task: build_and_test_debug - image: centos6-build - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=Debug - OUTPUT_DIR: --output_dir=build_and_test/install - - task: package_tarball_debug - image: centos6-build - file: orca_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos6_debug.tar.gz - - put: bin_orca_centos6_debug - params: - from: package_tarball/bin_orca_centos6_debug.tar.gz - -#CentOS7 build -- name: orca_centos7_release - max_in_flight: 2 - plan: - - get: centos7-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - task: build_and_test_release - image: centos7-build - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - - task: package_tarball_release - image: centos7-build - file: orca_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos7_release.tar.gz - - put: bin_orca_centos7_release - params: - from: package_tarball/bin_orca_centos7_release.tar.gz - -- name: orca_centos7_debug - max_in_flight: 2 - plan: - - get: centos7-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - task: build_and_test_debug - image: centos7-build - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=Debug - OUTPUT_DIR: --output_dir=build_and_test/install - - task: package_tarball_debug - image: centos7-build - file: orca_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos7_debug.tar.gz - - put: bin_orca_centos7_debug - params: - from: package_tarball/bin_orca_centos7_debug.tar.gz - -## Ubuntu18 build -- name: orca_ubuntu18_release - max_in_flight: 2 - plan: - - get: ubuntu18.04-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_ubuntu18 - passed: - - xerces_ubuntu18 - - task: build_and_test_release - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: "" - image: ubuntu18.04-build - - task: package_tarball_release - file: orca_src/concourse/package_tarball.yml - params: - dst_tarball: package_tarball/bin_orca_ubuntu18_release.tar.gz - src_root: build_and_test/install - - put: bin_orca_ubuntu18_release - params: - from: package_tarball/bin_orca_ubuntu18_release.tar.gz - -- name: orca_ubuntu18_debug - max_in_flight: 2 - plan: - - get: ubuntu18.04-build - - get: orca_src - trigger: true - - get: bin_xerces - resource: bin_xerces_ubuntu18 - passed: - - xerces_ubuntu18 - - task: build_and_test_debug - file: orca_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=Debug - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: "" - image: ubuntu18.04-build - - task: package_tarball_debug - file: orca_src/concourse/package_tarball.yml - params: - dst_tarball: package_tarball/bin_orca_ubuntu18_debug.tar.gz - src_root: build_and_test/install - - put: bin_orca_ubuntu18_debug - params: - from: package_tarball/bin_orca_ubuntu18_debug.tar.gz - -- name: build_gpdb - plan: - - in_parallel: - - &get_released_orca - get: bin_orca - resource: orca_github_release - params: - globs: - - bin_orca_centos5_release.tar.gz - passed: - - orca_publish_tag - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: gpdb_src - params: - submodules: none - trigger: true - - task: build_with_orca - file: gpdb_src/concourse/tasks/build_with_orca.yml - params: - MODE: "--mode=orca" - OUTPUTDIR: "--output_dir=bin_gpdb/install" - ACTION: "--action=build" - CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions" - OTHER_CONFIGS: "--orca-in-gpdb-install-location bin_orca bin_xerces" - - task: package_tarball - file: gpdb_src/concourse/tasks/package_tarball.yml - params: - src_root: bin_gpdb/install - dst_tarball: package_tarball/bin_gpdb.tar.gz - - put: bin_gpdb_centos6 - params: - file: package_tarball/bin_gpdb.tar.gz - -- name: gpdb_icg_with_orca - plan: - - in_parallel: - - <<: *get_released_orca - passed: - - build_gpdb - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - build_gpdb - - get: bin_gpdb - resource: bin_gpdb_centos6 - passed: - - build_gpdb - trigger: true - - get: gpdb_src - passed: - - build_gpdb - params: - submodules: none - - task: icg_with_orca - timeout: 64m - file: gpdb_src/concourse/tasks/test_icg.yml - params: - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - MODE: "--mode=orca" - ACTION: "--action=test" - CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions" - GPDB_BINARY: "--gpdb_name=bin_gpdb" - on_failure: - in_parallel: - - put: regression_diffs - params: - file: icg_output/regression.diffs - - put: regression_out - params: - file: icg_output/regression.out - -- name: gpdb_icg_with_planner - plan: - - in_parallel: - - <<: *get_released_orca - passed: - - build_gpdb - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - build_gpdb - - get: bin_gpdb - resource: bin_gpdb_centos6 - passed: - - build_gpdb - trigger: true - - get: gpdb_src - passed: - - build_gpdb - params: - submodules: none - - task: icg_with_planner - timeout: 45m - file: gpdb_src/concourse/tasks/test_icg.yml - params: - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - MODE: "--mode=planner" - ACTION: "--action=test" - CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions" - GPDB_BINARY: "--gpdb_name=bin_gpdb" - on_failure: - in_parallel: - - put: regression_diffs - params: - file: icg_output/regression.diffs - - put: regression_out - params: - file: icg_output/regression.out diff --git a/src/backend/gporca/concourse/pr_pipeline.yml b/src/backend/gporca/concourse/pr_pipeline.yml deleted file mode 100644 index 8d2536d80af2d0ddc6d8653b4846a0476eaf823a..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/pr_pipeline.yml +++ /dev/null @@ -1,109 +0,0 @@ -resource_types: - - name: pull_request - type: docker-image - source: - repository: jtarchie/pr - -resources: - - name: orca_pr - type: pull_request - source: - repo: greenplum-db/gporca - access_token: {{gporcabot_access_token}} - ignore_paths: - - README.md - - LICENSE - - COPYRIGHT - - .gitignore - - - name: xerces_patch - type: git - source: - branch: master - paths: - - patches/xerces-c-gpdb.patch - - concourse/xerces-c - - concourse/package_tarball.bash - uri: https://github.com/greenplum-db/gporca - - - name: bin_xerces_centos5 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{bucket-name}} - region_name: {{aws-region}} - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_xerces_centos5.tar.gz - - - name: centos5-build - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 - - -jobs: -- name: xerces_centos5 - public: true - max_in_flight: 2 - plan: - - get: xerces_patch - trigger: true - - get: centos5-build - - in_parallel: - - task: build - file: xerces_patch/concourse/xerces-c/build_xerces.yml - image: centos5-build - - in_parallel: - - task: package_tarball - file: xerces_patch/concourse/xerces-c/package_tarball.yml - params: - dst_tarball: package_tarball/bin_xerces_centos5.tar.gz - src_root: build_xerces/install - - in_parallel: - - put: bin_xerces_centos5 - params: - from: package_tarball/bin_xerces_centos5.tar.gz - -- name: compile_and_test_gporca - public: true - max_in_flight: 2 - plan: - - get: orca_src - resource: orca_pr - trigger: true - version: every - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: centos5-build - - put: orca_pr - params: - path: orca_src - status: pending - - in_parallel: - - task: build_and_test_release - file: orca_src/concourse/build_and_test.yml - image: centos5-build - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install_release - timeout: 30m - - task: build_and_test_debug - file: orca_src/concourse/build_and_test.yml - image: centos5-build - params: - BUILD_TYPE: --build_type=Debug - OUTPUT_DIR: --output_dir=build_and_test/install_debug - timeout: 30m - on_failure: &pr_failure - put: orca_pr - params: - path: orca_src - status: failure - - put: report_pr_success - resource: orca_pr - params: - path: orca_src - status: success diff --git a/src/backend/gporca/concourse/publish_tag.bash b/src/backend/gporca/concourse/publish_tag.bash deleted file mode 100755 index 76b7da146653925fa438a389b92ca782f2fa9fab..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/publish_tag.bash +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - - -set -o pipefail -set -u -e -x - -main() { - env LC_ALL=C tar tf bin_orca_centos5_release/*.tar.gz | egrep "libgpopt\.so\.[0-9]+.[0-9]+.[0-9]+" | sed 's/\.\/lib\/libgpopt\.so\./v/' > orca_github_release_stage/tag.txt - cp -v bin_orca_centos5_release/*.tar.gz orca_github_release_stage/ - cp -v bin_orca_centos5_debug/*.tar.gz orca_github_release_stage/ - env GIT_DIR=orca_src/.git git rev-parse HEAD > orca_github_release_stage/commit.txt -} - -main "$@" diff --git a/src/backend/gporca/concourse/publish_tag.yml b/src/backend/gporca/concourse/publish_tag.yml deleted file mode 100644 index ad155920bf97fe2680469e9574732c92750666c2..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/publish_tag.yml +++ /dev/null @@ -1,14 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 -inputs: - - name: orca_src - - name: bin_orca_centos5_release - - name: bin_orca_centos5_debug -outputs: - - name: orca_github_release_stage -run: - path: orca_src/concourse/publish_tag.bash diff --git a/src/backend/gporca/concourse/test_explain_pipeline.yml b/src/backend/gporca/concourse/test_explain_pipeline.yml deleted file mode 100644 index 04cbd5d19e444ec5905142afa264fe8afe2f6341..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/test_explain_pipeline.yml +++ /dev/null @@ -1,444 +0,0 @@ -groups: [] -resources: -- name: centos5-build - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 -- name: gporca-commits-to-test - type: git - source: - branch: {{branch_name}} - uri: https://github.com/greenplum-db/gporca-pipeline-misc -- name: orca_main_src # used for yml task files - type: git - source: - branch: master - paths: null - uri: https://github.com/greenplum-db/gporca -- name: orca_baseline_src # used for baseline build - type: git - source: - branch: master - paths: null - uri: https://github.com/greenplum-db/gporca -- name: xerces_patch - type: git - source: - branch: master - paths: - - patches/xerces-c-gpdb.patch - - concourse/xerces-c - - concourse/package_tarball.bash - uri: https://github.com/greenplum-db/gporca -- name: bin_gpdb_centos6 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_gpdb_with_orca_centos6.tar.gz -- name: bin_gpdb_baseline_centos6 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_gpdb_baseline_with_orca_centos6.tar.gz -- name: gpdb_main_src # used for yml task files - type: git - source: - branch: 5X_STABLE - ignore_paths: - - README.md - - LICENSE - - COPYRIGHT - - .gitignore - uri: https://github.com/greenplum-db/gpdb -- name: gpdb_baseline_src # used for baseline build - type: git - source: - branch: 5X_STABLE - paths: - - config/orca.m4 #trigger only when 5X_STABLE has bumped ORCA versions - uri: https://github.com/greenplum-db/gpdb -- name: gpdb_tarball - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: gpdb_src.tar.gz -- name: gpdb_baseline_tarball - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: gpdb_baseline_src.tar.gz -- name: bin_orca_centos5_release - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos5_release.tar.gz -- name: bin_orca_baseline_centos5_release - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_baseline_centos5_release.tar.gz -- name: bin_xerces_centos5 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_xerces_centos5.tar.gz -- name: orca_tarball - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: orca_src.tar.gz -- name: explain_test_suite - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: {{explain_workload_tar}} -- name: explain_output - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: explain_output.tar.gz -- name: explain_output_baseline - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: explain_output_baseline.tar.gz -- name: explain_test_results - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: {{explain_workload_results}} - -resource_types: [] -jobs: -- name: xerces_centos5 - max_in_flight: 2 - plan: - - get: xerces_patch - trigger: true - - get: centos5-build - - in_parallel: - - task: build - file: xerces_patch/concourse/xerces-c/build_xerces.yml - image: centos5-build - - in_parallel: - - task: package_tarball - file: xerces_patch/concourse/xerces-c/package_tarball.yml - params: - dst_tarball: package_tarball/bin_xerces_centos5.tar.gz - src_root: build_xerces/install - - in_parallel: - - put: bin_xerces_centos5 - params: - from: package_tarball/bin_xerces_centos5.tar.gz -- name: orca_centos5_release - max_in_flight: 2 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - clone_repos_to_test - version: every - - get: orca_main_src - passed: - - clone_repos_to_test - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: orca_tarball - passed: - - clone_repos_to_test - trigger: true - - get: gpdb_tarball - passed: - - clone_repos_to_test - - task: untar_build_and_test_release - file: orca_main_src/concourse/untar_build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: --skiptests - - put: bin_orca_centos5_release - params: - from: package_tarball/bin_orca_centos5_release.tar.gz -- name: clone_repos_to_test - plan: - - in_parallel: - - get: gporca-commits-to-test - trigger: true - version: every - - get: orca_main_src - - task: clone_repos_to_test - file: orca_main_src/concourse/clone_remote_repo.yml - - in_parallel: - - put: orca_tarball - params: - from: package_tarball/orca_src.tar.gz - - put: gpdb_tarball - params: - from: package_tarball/gpdb_src.tar.gz -- name: build_gpdb - max_in_flight: 1 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - orca_centos5_release - - get: bin_orca - passed: - - orca_centos5_release - resource: bin_orca_centos5_release - trigger: true - version: every - - get: bin_xerces - passed: - - xerces_centos5 - resource: bin_xerces_centos5 - - get: gpdb_tarball - passed: - - orca_centos5_release - - get: gpdb_main_src - - task: build_with_orca - file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml - params: - ACTION: build - MODE: orca - CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - - put: bin_gpdb_centos6 - params: - from: package_tarball/bin_gpdb.tar.gz - -- name: orca_baseline_centos5_release - max_in_flight: 2 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - clone_repos_to_test - version: every - - get: orca_main_src - passed: - - clone_repos_to_test - - get: orca_src - resource: orca_baseline_src - trigger: true - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: centos5-build - - task: build_and_test_release - image: centos5-build - file: orca_main_src/concourse/build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: --skiptests - - task: package_tarball_release - file: orca_main_src/concourse/package_tarball.yml - params: - src_root: build_and_test/install - dst_tarball: package_tarball/bin_orca_centos5_release.tar.gz - - put: bin_orca_baseline_centos5_release - params: - from: package_tarball/bin_orca_centos5_release.tar.gz - -- name: build_gpdb_baseline - plan: - - in_parallel: - - get: bin_orca - resource: bin_orca_baseline_centos5_release - passed: - - orca_baseline_centos5_release - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: gpdb_main_src - - get: gpdb_src - resource: gpdb_baseline_src - trigger: true - - task: build_with_orca - file: gpdb_main_src/concourse/tasks/build_with_orca.yml - params: - MODE: "--mode=orca" - OUTPUTDIR: "--output_dir=bin_gpdb/install" - ACTION: "--action=build" - CONFIGURE_OPTION: "--configure-option=--disable-gpcloud --configure-option=--enable-debug-extensions" - OTHER_CONFIGS: "--orca-in-gpdb-install-location bin_orca bin_xerces" - - task: package_tarball - file: gpdb_main_src/concourse/tasks/package_tarball.yml - params: - src_root: bin_gpdb/install - dst_tarball: package_tarball/bin_gpdb.tar.gz - - task: package_gpdb_baseline_src - config: - platform: linux - image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos6 - inputs: - - name: gpdb_src - outputs: - - name: package_gpdb_baseline_src - run: - path: bash - args: - - -ec - - | - tar czf "package_gpdb_baseline_src/gpdb_src.tar.gz" -C gpdb_src . - - put: bin_gpdb_baseline_centos6 - params: - file: package_tarball/bin_gpdb.tar.gz - - put: gpdb_baseline_tarball - params: - file: package_gpdb_baseline_src/gpdb_src.tar.gz -- name: test_explain_suite - max_in_flight: 1 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - build_gpdb - - get: bin_orca - passed: - - build_gpdb - resource: bin_orca_centos5_release - - get: bin_xerces - passed: - - build_gpdb - resource: bin_xerces_centos5 - - get: bin_gpdb - passed: - - build_gpdb - resource: bin_gpdb_centos6 - trigger: true - - get: gpdb_main_src - passed: - - build_gpdb - params: - submodules: none - - get: gpdb_tarball - passed: - - build_gpdb - - get: explain_test_suite - - task: run_explain_suite - file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml - params: - MODE: orca - ACTION: test_explain_suite - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions - timeout: 90m - - put: explain_output - params: - file: icg_output/explain_ouput.tar.gz -- name: test_explain_suite_baseline - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - build_gpdb - - get: bin_orca - resource: bin_orca_baseline_centos5_release - params: - globs: - - bin_orca_baseline_centos5_release.tar.gz - passed: - - build_gpdb_baseline - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - build_gpdb_baseline - - get: bin_gpdb - resource: bin_gpdb_baseline_centos6 - passed: - - build_gpdb_baseline - trigger: true - - get: gpdb_main_src - passed: - - build_gpdb_baseline - params: - submodules: none - - get: gpdb_tarball - resource: gpdb_baseline_tarball - passed: - - build_gpdb_baseline - - get: explain_test_suite - - task: run_explain_suite_baseline - timeout: 64m - file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml - params: - MODE: orca - ACTION: test_explain_suite - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions - timeout: 90m - - put: explain_output_baseline - params: - file: icg_output/explain_ouput.tar.gz -- name: diff_explain_results_with_baseline - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - test_explain_suite - - get: gpdb_main_src - passed: - - test_explain_suite - params: - submodules: none - - get: explain_output - passed: - - test_explain_suite - trigger: true - - get: explain_output_baseline - passed: - - test_explain_suite_baseline - - task: diff_explain_results_with_baseline - timeout: 64m - file: gpdb_main_src/concourse/tasks/diff_explain_results_with_baseline.yml - ensure: - put: explain_test_results - params: - file: diffs/explain_test_results.tar.gz diff --git a/src/backend/gporca/concourse/test_orca_pipeline.yml b/src/backend/gporca/concourse/test_orca_pipeline.yml index ac40191380c3954d42d92331984ef2dedbbf6e81..91b1c7bb502292f78a6fa6e448eddf9d0ce0ce02 100644 --- a/src/backend/gporca/concourse/test_orca_pipeline.yml +++ b/src/backend/gporca/concourse/test_orca_pipeline.yml @@ -1,324 +1,117 @@ -groups: [] +## ====================================================================== +## resources +## ====================================================================== + +resource_types: +- name: gcs + type: docker-image + source: + repository: frodenas/gcs-resource + resources: - name: gporca-commits-to-test type: git source: branch: master uri: https://github.com/greenplum-db/gporca-pipeline-misc -- name: regression_diffs - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: regression.diffs -- name: orca_main_src - type: git - source: - branch: master - paths: null - uri: https://github.com/greenplum-db/gporca -- name: xerces_patch - type: git - source: - branch: master - paths: - - patches/xerces-c-gpdb.patch - - concourse/xerces-c - - concourse/package_tarball.bash - uri: https://github.com/greenplum-db/gporca -- name: regression_out - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: regression.out -- name: bin_gpdb_centos6 - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_gpdb_with_orca_centos6.tar.gz -- name: gpdb_main_src - type: git - source: - branch: 5X_STABLE - ignore_paths: - - README.md - - LICENSE - - COPYRIGHT - - .gitignore - uri: https://github.com/greenplum-db/gpdb -- name: gpdb_tarball - type: s3 - source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: gpdb_src.tar.gz -- name: bin_orca_centos5_debug - type: s3 + +- name: gpdb6-centos7-build + type: docker-image source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos5_debug.tar.gz -- name: bin_orca_centos5_release - type: s3 + repository: pivotaldata/gpdb6-centos7-build + +- name: gpdb6-centos7-test + type: docker-image source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_orca_centos5_release.tar.gz -- name: bin_xerces_centos5 - type: s3 + repository: pivotaldata/gpdb6-centos7-test + +- name: libquicklz-centos7 + type: gcs source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: bin_xerces_centos5.tar.gz -- name: orca_tarball - type: s3 + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/centos7/libquicklz-(1\.5\.0-.*)-1.el7.x86_64.rpm + +- name: libquicklz-devel-centos7 + type: gcs source: - access_key_id: {{s3_access_key_id}} - bucket: {{gporca-concourse-bucket-dev}} - region_name: us-west-2 - secret_access_key: {{s3_secret_access_key}} - versioned_file: orca_src.tar.gz -- name: centos5-build - type: docker-image + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/centos7/libquicklz-devel-(1\.5\.0-.*)-1.el7.x86_64.rpm + +- name: python-centos7 + type: gcs source: - repository: yolo/orcadev - tag: centos5 + bucket: ((gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/centos7/python-(2\.7\.12-.*).tar.gz + -resource_types: [] jobs: -- name: xerces_centos5 - max_in_flight: 2 - plan: - - get: xerces_patch - trigger: true - - get: centos5-build - - in_parallel: - - task: build - file: xerces_patch/concourse/xerces-c/build_xerces.yml - image: centos5-build - - in_parallel: - - task: package_tarball - file: xerces_patch/concourse/xerces-c/package_tarball.yml - params: - dst_tarball: package_tarball/bin_xerces_centos5.tar.gz - src_root: build_xerces/install - - in_parallel: - - put: bin_xerces_centos5 - params: - from: package_tarball/bin_xerces_centos5.tar.gz -- name: orca_centos5_release - max_in_flight: 1 +- name: compile_and_test_gpdb + max_in_flight: 10 plan: - in_parallel: - get: gporca-commits-to-test - passed: - - clone_repos_to_test trigger: true version: every - - get: orca_main_src - passed: - - clone_repos_to_test - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: orca_tarball - passed: - - clone_repos_to_test - - get: gpdb_tarball - passed: - - clone_repos_to_test - - task: untar_build_and_test_release - file: orca_main_src/concourse/untar_build_and_test.yml - params: - BUILD_TYPE: --build_type=RelWithDebInfo - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: '' - - put: bin_orca_centos5_release - params: - from: package_tarball/bin_orca_centos5_release.tar.gz -- name: orca_centos5_debug - max_in_flight: 1 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - clone_repos_to_test - trigger: true - version: every - - get: orca_main_src - passed: - - clone_repos_to_test - - get: bin_xerces_centos5 - - get: bin_xerces - resource: bin_xerces_centos5 - passed: - - xerces_centos5 - - get: orca_tarball - passed: - - clone_repos_to_test - - task: untar_build_and_test_debug - file: orca_main_src/concourse/untar_build_and_test.yml - params: - BUILD_TYPE: --build_type=Debug - OUTPUT_DIR: --output_dir=build_and_test/install - SKIP_TESTS: '' - - in_parallel: - - put: bin_orca_centos5_debug - params: - from: package_tarball/bin_orca_centos5_debug.tar.gz -- name: clone_repos_to_test - plan: - - in_parallel: - - get: gporca-commits-to-test - trigger: true - version: every - - get: orca_main_src - - task: clone_repos_to_test - file: orca_main_src/concourse/clone_remote_repo.yml - - in_parallel: - - put: orca_tarball - params: - from: package_tarball/orca_src.tar.gz - - put: gpdb_tarball - params: - from: package_tarball/gpdb_src.tar.gz -- name: build_gpdb - max_in_flight: 1 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - orca_centos5_release - - orca_centos5_debug - trigger: true - version: every - - get: bin_orca - passed: - - orca_centos5_release - resource: bin_orca_centos5_release - - get: bin_orca_centos5_debug - passed: - - orca_centos5_debug - - get: bin_xerces - passed: - - xerces_centos5 - resource: bin_xerces_centos5 - - get: gpdb_tarball - passed: - - orca_centos5_release - - get: gpdb_main_src - - task: build_with_orca - file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml - params: - ACTION: build - MODE: orca - CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions - - put: bin_gpdb_centos6 - params: - from: package_tarball/bin_gpdb.tar.gz -- name: gpdb_icg_with_orca - max_in_flight: 3 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - build_gpdb - trigger: true - - get: bin_orca - passed: - - build_gpdb - resource: bin_orca_centos5_release - - get: bin_xerces - passed: - - build_gpdb - resource: bin_xerces_centos5 - - get: bin_gpdb - passed: - - build_gpdb - resource: bin_gpdb_centos6 - - get: gpdb_main_src - passed: - - build_gpdb - params: - submodules: none - - get: gpdb_tarball - passed: - - build_gpdb - - task: icg_with_orca - file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml - params: - MODE: orca - ACTION: test - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions - on_failure: - in_parallel: - - put: regression_diffs - params: - file: icg_output/regression.diffs - - put: regression_out - params: - file: icg_output/regression.out - timeout: 180m -- name: gpdb_icg_with_planner - max_in_flight: 3 - plan: - - in_parallel: - - get: gporca-commits-to-test - passed: - - build_gpdb - trigger: true - - get: bin_orca - passed: - - build_gpdb - resource: bin_orca_centos5_release - - get: bin_xerces - passed: - - build_gpdb - resource: bin_xerces_centos5 - - get: bin_gpdb - passed: - - build_gpdb - resource: bin_gpdb_centos6 - - get: gpdb_main_src - passed: - - build_gpdb + - get: gpdb6-centos7-build + - get: gpdb6-centos7-test + - get: libquicklz-installer + resource: libquicklz-centos7 + - get: libquicklz-devel-installer + resource: libquicklz-devel-centos7 + - get: python-tarball + resource: python-centos7 + + - do: + - task: init gpdb_src + image: gpdb6-centos7-build + config: + platform: linux + run: + path: bash + args: + - -c + - | + BRANCH=$(cut -d, -f1 gporca-commits-to-test/gpdb_remote_info.txt) + REMOTE=$(cut -d, -f2 gporca-commits-to-test/gpdb_remote_info.txt) + git clone -b $BRANCH --single-branch $REMOTE gpdb_src && + cd gpdb_src && + git fetch https://github.com/greenplum-db/gpdb.git --tags && + git submodule update --init --recursive + inputs: [{ name: gporca-commits-to-test }] + outputs: [{ name: gpdb_src }] + - task: compile_gpdb_centos7 + file: gpdb_src/concourse/tasks/compile_gpdb.yml + image: gpdb6-centos7-build params: - submodules: none - - get: gpdb_tarball - passed: - - build_gpdb - - task: icg_with_planner - file: gpdb_main_src/concourse/tasks/untar_build_with_orca.yml - params: - MODE: planner - ACTION: test - BLDWRAP_POSTGRES_CONF_ADDONS: statement_mem=250MB - CONFIGURE_OPTION: --disable-gpcloud --enable-debug-extensions + CONFIGURE_FLAGS: {{configure_flags_with_extensions}} + TARGET_OS: centos + TARGET_OS_VERSION: 7 + BLD_TARGETS: "clients loaders" + timeout: 30m - on_failure: - in_parallel: - - put: regression_diffs + + - in_parallel: + - task: icw_planner_centos7 + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb6-centos7-test + input_mapping: + bin_gpdb: gpdb_artifacts params: - file: icg_output/regression.diffs - - put: regression_out + MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off' installcheck-world + TEST_OS: centos + CONFIGURE_FLAGS: {{configure_flags}} + timeout: 3h + + - task: icw_gporca_centos7 + file: gpdb_src/concourse/tasks/ic_gpdb.yml + image: gpdb6-centos7-test + input_mapping: + bin_gpdb: gpdb_artifacts params: - file: icg_output/regression.out - timeout: 180m + MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=on' installcheck-world + TEST_OS: centos + CONFIGURE_FLAGS: {{configure_flags}} + timeout: 5h diff --git a/src/backend/gporca/concourse/untar_build_and_test.py b/src/backend/gporca/concourse/untar_build_and_test.py deleted file mode 100755 index 458c57a5944a256f4c052354e739603dbb4daa00..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/untar_build_and_test.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -import subprocess -import os -import sys - -BUILD_TYPE=os.environ['BUILD_TYPE'] -OUTPUT_DIR=os.environ['OUTPUT_DIR'] -SKIP_TESTS=os.environ['SKIP_TESTS'] -if 'Debug' in BUILD_TYPE: - path_identifier = 'debug' -else: - path_identifier = 'release' - -def exec_command(cmd): - print "Executing command: {0}".format(cmd) - retcode = subprocess.call(cmd, shell=True) - if retcode != 0: - sys.exit(retcode) - -untar_orca_cmd = "mkdir -p orca_src && tar -xf orca_tarball/orca_src.tar.gz -C orca_src --strip 1" -exec_command(untar_orca_cmd) -build_and_test_orca_cmd = "orca_main_src/concourse/build_and_test.py {0} {1} {2} bin_xerces".format(BUILD_TYPE, OUTPUT_DIR, SKIP_TESTS) -exec_command(build_and_test_orca_cmd) - -SRC_DIR=OUTPUT_DIR.split('=')[1] -package_tarball_cmd = "env dst_tarball=package_tarball/bin_orca_centos5_{0}.tar.gz src_root={1} orca_main_src/concourse/package_tarball.bash".format(path_identifier, SRC_DIR) -exec_command(package_tarball_cmd) diff --git a/src/backend/gporca/concourse/untar_build_and_test.yml b/src/backend/gporca/concourse/untar_build_and_test.yml deleted file mode 100644 index 66eea87d4c903ed808c9d63bb867ff4642d0ef48..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/untar_build_and_test.yml +++ /dev/null @@ -1,27 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: yolo/orcadev - tag: centos5 -inputs: - - name: orca_main_src - - name: orca_tarball - - name: bin_xerces - -caches: - - path: .ccache - -outputs: - - name: build_and_test - - name: package_tarball -params: - BUILD_TYPE: - OUTPUT_DIR: - SKIP_TESTS: -run: - path: sh - args: - - -exc - - | - orca_main_src/concourse/untar_build_and_test.py ${BUILD_TYPE} ${OUTPUT_DIR} ${SKIP_TESTS} bin_xerces diff --git a/src/backend/gporca/concourse/upload_orca_source_bintray.py b/src/backend/gporca/concourse/upload_orca_source_bintray.py deleted file mode 100755 index 3ed9bbd27a568151fbac051b6ca21b3cbbdadcb3..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/upload_orca_source_bintray.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -import subprocess -import os -import sys -import argparse -import shutil -from functools import wraps - -BASE_WD = os.getcwd() -ORCA_SRC_DIR=os.path.join(BASE_WD, 'orca_src') -GPORCA_TAG_FILE='orca_github_release_stage/tag.txt' - -def fail_on_error(func): - def _check_status(request, *args, **kwargs): - response = func(request, *args, **kwargs) - if response: - return sys.exit(1) - return wraps(func)(_check_status) - -def exec_command(cmd): - print "Executing command: {0}".format(' '.join(cmd)) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) - while True: - nextline = p.stdout.readline() - if nextline == '' and p.poll() is not None: - break - sys.stdout.write(nextline) - sys.stdout.flush() - - output = p.communicate()[0] - exitCode = p.returncode - print "Exit Code: {0}\n".format(p.returncode) - return p.returncode == 1 - -def read_orca_version(): - # read orca version - with open(GPORCA_TAG_FILE, 'r') as fh: - return fh.read().strip(); - -@fail_on_error -def export_orca_src(orca_version): - cmd = ["conan", "export", "orca/{0}@gpdb/stable".format(orca_version)] - return exec_command(cmd) - -@fail_on_error -def add_conan_remote(bintray_remote_name): - # add conan remote - cmd = ["conan", "remote", "add", bintray_remote_name, "https://api.bintray.com/conan/greenplum-db/gpdb-oss"] - return exec_command(cmd) - -@fail_on_error -def set_bintray_user(bintray_user, bintray_user_key, bintray_remote): - # set bintray account to be used to upload package - cmd = ["conan", "user", "-p", bintray_user_key, "-r={0}".format(bintray_remote), bintray_user] - return exec_command(cmd) - -@fail_on_error -def upload_orca_src(orca_version): - # upload orca source to bintray - cmd = ["conan", "upload", "orca/{0}@gpdb/stable".format(orca_version), "--all", "-r=conan-gpdb"] - return exec_command(cmd) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Main drived to build and install ORCA using conan') - required_arguments = parser.add_argument_group('required arguments') - required_arguments.add_argument('--bintrayUser', help='Bintray user name to upload packages', type=str) - required_arguments.add_argument('--bintrayUserKey', help='Bintray user key', type=str) - required_arguments.add_argument('--bintrayRemote', help='Name of conan remote refering to bintray', type=str) - - args = parser.parse_args() - if args.bintrayRemote is None or args.bintrayUser is None or args.bintrayUserKey is None: - print "Values for --bintrayRemote, --bintrayUser and --bintrayUserKey argument values are required, some are missing, exiting!" - sys.exit(1) - - # setup - orca_version = read_orca_version() - print "Tag: {0}".format(orca_version) - os.chdir(ORCA_SRC_DIR) - shutil.copy("concourse/conanfile.py", "conanfile.py") - - # main conan driver - export_orca_src(orca_version) - add_conan_remote(args.bintrayRemote) - set_bintray_user(args.bintrayUser, args.bintrayUserKey, args.bintrayRemote) - upload_orca_src(orca_version) diff --git a/src/backend/gporca/concourse/upload_orca_source_bintray.yml b/src/backend/gporca/concourse/upload_orca_source_bintray.yml deleted file mode 100644 index 28585c9469b76813b631da1d56f5dcf44cd41aca..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/upload_orca_source_bintray.yml +++ /dev/null @@ -1,20 +0,0 @@ -platform: linux -image_resource: - type: docker-image - source: - repository: pivotaldata/conan - tag: latest -inputs: - - name: orca_src - - name: orca_github_release_stage -outputs: -run: - path: sh - args: - - -exc - - | - python orca_src/concourse/upload_orca_source_bintray.py --bintrayUser=${BINTRAY_USER} --bintrayRemote=${BINTRAY_REMOTE} --bintrayUserKey=${BINTRAY_USER_KEY} -params: - BINTRAY_USER: required - BINTRAY_REMOTE: required - BINTRAY_USER_KEY: required diff --git a/src/backend/gporca/concourse/xerces-c/build_xerces.bash b/src/backend/gporca/concourse/xerces-c/build_xerces.bash deleted file mode 100644 index f8baf30ffd09ce654889c34e36c590b593a7f62c..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/xerces-c/build_xerces.bash +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -u -e -x - -main() { - wget http://www.trieuvan.com/apache//xerces/c/3/sources/xerces-c-3.1.2.tar.gz - tar -xzf xerces-c-3.1.2.tar.gz - cd xerces-c-3.1.2 - patch -p1 < ../xerces_patch/patches/xerces-c-gpdb.patch - ./configure --prefix=$PWD/../install - make -j$(nproc_wrapper) - make install -} - -nproc_wrapper() { - if ! type nproc &> /dev/null; then - echo 4 - else - command nproc - fi -} - -main "$@" diff --git a/src/backend/gporca/concourse/xerces-c/build_xerces.py b/src/backend/gporca/concourse/xerces-c/build_xerces.py index 716db9461233c973621518c353b202ef5bb38eb3..d1dcbf5b8798b809f8f88a48866ec2f5eee25737 100644 --- a/src/backend/gporca/concourse/xerces-c/build_xerces.py +++ b/src/backend/gporca/concourse/xerces-c/build_xerces.py @@ -7,6 +7,7 @@ import subprocess import sys import tarfile import urllib2 +import hashlib XERCES_SOURCE_URL = "http://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.2.tar.gz" XERCES_SOURCE_DIR = "xerces-c-3.1.2" @@ -32,6 +33,12 @@ def get_xerces_source(): local_src = open("xerces_src.tar.gz", "w") local_src.write(remote_src.read()) local_src.close() + file_hash = hashlib.sha256(open('xerces_src.tar.gz','rb').read()).hexdigest() + actual_hash = "" + with open('xerces_patch/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256', 'r') as f: + actual_hash = f.read().strip() + if file_hash != actual_hash: + return 1 tarball = tarfile.open("xerces_src.tar.gz", "r:gz") for item in tarball: tarball.extract(item, ".") diff --git a/src/backend/gporca/concourse/xerces-c/build_xerces.yml b/src/backend/gporca/concourse/xerces-c/build_xerces.yml deleted file mode 100644 index 06c529864a3302fd5097efd002cb830d7472c39d..0000000000000000000000000000000000000000 --- a/src/backend/gporca/concourse/xerces-c/build_xerces.yml +++ /dev/null @@ -1,12 +0,0 @@ -platform: linux -image_resource: - type: docker-image -inputs: - - name: xerces_patch -outputs: - - name: build_xerces -run: - path: /usr/bin/python - args: - - xerces_patch/concourse/xerces-c/build_xerces.py - - --output_dir=build_xerces/install diff --git a/src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 b/src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 new file mode 100644 index 0000000000000000000000000000000000000000..7fbcc8000ec21e8bdb7e27abc9224c6eb06e5c7f --- /dev/null +++ b/src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 @@ -0,0 +1 @@ +743bd0a029bf8de56a587c270d97031e0099fe2b7142cef03e0da16e282655a0