From fe8e51b96eeb57595dd892654bc1f84160df67bb Mon Sep 17 00:00:00 2001 From: Goutam Tadi Date: Fri, 29 Sep 2017 18:27:36 -0700 Subject: [PATCH] Test ubuntu with ICW: configure, Make demo cluster - We receive compiled GPDB tar output through a shared dir - Separate Compile and ICW jobs for ubuntu - Increase number of parallel client connections Signed-off-by: C.J. Jameson --- concourse/pipelines/pipeline.yml | 46 ++++++++++- .../compile_gpdb_open_source_ubuntu.bash | 25 +++--- concourse/scripts/ic_gpdb_ubuntu.bash | 82 +++++++++++++++++++ .../tasks/compile_gpdb_open_source_ubuntu.yml | 4 + concourse/tasks/ic_gpdb_ubuntu.yml | 15 ++++ 5 files changed, 161 insertions(+), 11 deletions(-) create mode 100755 concourse/scripts/ic_gpdb_ubuntu.bash create mode 100644 concourse/tasks/ic_gpdb_ubuntu.yml diff --git a/concourse/pipelines/pipeline.yml b/concourse/pipelines/pipeline.yml index b590463c9c..1293b6cf98 100644 --- a/concourse/pipelines/pipeline.yml +++ b/concourse/pipelines/pipeline.yml @@ -25,6 +25,7 @@ groups: - icw_gporca_centos6 - icw_gporca_centos7 - icw_gporca_sles11 + - icw_planner_ubuntu16 - icw_planner_ictcp_centos6 - icw_gporca_centos6_gpos_memory - fts @@ -355,6 +356,15 @@ resources: secret_access_key: {{bucket-secret-access-key}} versioned_file: {{bin_gpdb_centos_versioned_file}} +- name: compiled_bits_ubuntu16 + type: s3 + source: + access_key_id: {{bucket-access-key-id}} + bucket: {{bucket-name}} + region_name: {{aws-region}} + secret_access_key: {{bucket-secret-access-key}} + versioned_file: {{compiled_bits_ubuntu16_versioned_file}} + - name: binary_swap_gpdb_centos6 type: s3 source: @@ -750,6 +760,24 @@ jobs: - put: installer_rhel7_gpdb_loaders params: file: gpdb_artifacts/greenplum-loaders-*-rhel7-x86_64.zip + +- name: compile_gpdb_ubuntu16 + plan: + - aggregate: + - get: reduced-frequency-trigger + trigger: {{reduced-frequency-trigger-flag}} + - get: gpdb_src + trigger: {{gpdb_src-trigger-flag}} + - get: ubuntu-gpdb-dev-16 + - task: compile_gpdb + image: ubuntu-gpdb-dev-16 + file: gpdb_src/concourse/tasks/compile_gpdb_open_source_ubuntu.yml + params: + CONFIGURE_FLAGS: {{configure_flags}} + - put: compiled_bits_ubuntu16 + params: + file: compiled_bits_ubuntu16/compiled_bits_ubuntu16.tar.gz + - name: compile_gpdb_sles11 plan: - aggregate: @@ -1179,7 +1207,22 @@ jobs: BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off" TEST_OS: sles CONFIGURE_FLAGS: {{configure_flags}} -- name: icw_gporca_centos6_gpos_memory + +- name: icw_planner_ubuntu16 + plan: + - aggregate: + - get: gpdb_src + passed: [compile_gpdb_ubuntu16] + - get: compiled_bits_ubuntu16 + passed: [compile_gpdb_ubuntu16] + trigger: true + - get: ubuntu-gpdb-dev-16 + - task: ic_gpdb + file: gpdb_src/concourse/tasks/ic_gpdb_ubuntu.yml + image: ubuntu-gpdb-dev-16 + params: + MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off' installcheck-world + CONFIGURE_FLAGS: {{configure_flags}}- name: icw_gporca_centos6_gpos_memory plan: - aggregate: - get: gpdb_src @@ -2640,6 +2683,7 @@ jobs: - icw_gporca_centos6 - icw_gporca_centos7 - icw_gporca_sles11 + - icw_planner_ubuntu16 - icw_planner_ictcp_centos6 - client_loader_remote_test_aix - mpp_resource_group_centos6 diff --git a/concourse/scripts/compile_gpdb_open_source_ubuntu.bash b/concourse/scripts/compile_gpdb_open_source_ubuntu.bash index 184a647927..cc7882578f 100755 --- a/concourse/scripts/compile_gpdb_open_source_ubuntu.bash +++ b/concourse/scripts/compile_gpdb_open_source_ubuntu.bash @@ -2,19 +2,13 @@ set -exo pipefail GREENPLUM_INSTALL_DIR=/usr/local/gpdb - -# This is a canonical way to build GPDB. The intent is to validate that GPDB compiles -# with a fairly basic build. It is not meant to be exhaustive or include all features -# and components available in GPDB. +TRANSFER_DIR_ABSOLUTE_PATH="$(pwd)/${TRANSFER_DIR}" +COMPILED_BITS_FILENAME=${COMPILED_BITS_FILENAME:="compiled_bits_ubuntu16.tar.gz"} function build_gpdb() { pushd gpdb_src - CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-perl --with-libxml \ - --disable-orca --with-python --disable-gpfdist --prefix=${GREENPLUM_INSTALL_DIR} - # Use -j4 to speed up the build. (Doesn't seem worth trying to guess a better - # value based on number of CPUs or anything like that. Going above -j4 wouldn't - # make it much faster, and -j4 is small enough to not hurt too badly even on - # a single-CPU system + CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-gssapi --with-perl --with-libxml \ + --disable-orca --with-python --prefix=${GREENPLUM_INSTALL_DIR} make -j4 make install popd @@ -24,9 +18,20 @@ function unittest_check_gpdb() { make -C gpdb_src/src/backend -s unittest-check } +function export_gpdb() { + TARBALL="$TRANSFER_DIR_ABSOLUTE_PATH"/$COMPILED_BITS_FILENAME + pushd $GREENPLUM_INSTALL_DIR + source greenplum_path.sh + python -m compileall -x test . + chmod -R 755 . + tar -czf "${TARBALL}" ./* + popd +} + function _main() { build_gpdb unittest_check_gpdb + export_gpdb } _main "$@" diff --git a/concourse/scripts/ic_gpdb_ubuntu.bash b/concourse/scripts/ic_gpdb_ubuntu.bash new file mode 100755 index 0000000000..50ac6440d5 --- /dev/null +++ b/concourse/scripts/ic_gpdb_ubuntu.bash @@ -0,0 +1,82 @@ +#!/bin/bash -l + +set -eox pipefail + +CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +GREENPLUM_INSTALL_DIR=/usr/local/gpdb + +function load_transfered_bits_into_install_dir() { + mkdir -p $GREENPLUM_INSTALL_DIR + tar xzf $TRANSFER_DIR/$COMPILED_BITS_FILENAME -C $GREENPLUM_INSTALL_DIR +} + +function configure() { + pushd gpdb_src + ./configure --prefix=${GREENPLUM_INSTALL_DIR} --with-gssapi --with-perl --with-python --with-libxml --enable-mapreduce --disable-orca --enable-pxf ${CONFIGURE_FLAGS} + popd +} + +function setup_gpadmin_user() { + ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash ubuntu +} + +function make_cluster() { + export BLDWRAP_POSTGRES_CONF_ADDONS="fsync=off" + source "${GREENPLUM_INSTALL_DIR}/greenplum_path.sh" + export DEFAULT_QD_MAX_CONNECT=150 + pushd gpdb_src/gpAux/gpdemo + su gpadmin -c "make create-demo-cluster" + popd +} + +function gen_env_for_gpadmin_to_run(){ + cat > /opt/run_test.sh <<-EOF + trap look4diffs ERR + + function look4diffs() { + + diff_files=\`find .. -name regression.diffs\` + + for diff_file in \${diff_files}; do + if [ -f "\${diff_file}" ]; then + cat <<-FEOF + + ====================================================================== + DIFF FILE: \${diff_file} + ---------------------------------------------------------------------- + + \$(cat "\${diff_file}") + + FEOF + fi + done + exit 1 + } + source ${GREENPLUM_INSTALL_DIR}/greenplum_path.sh + cd "\${1}/gpdb_src" + source gpAux/gpdemo/gpdemo-env.sh + make ${MAKE_TEST_COMMAND} + EOF + + chmod a+x /opt/run_test.sh +} + +function run_test() { + su - gpadmin -c "bash /opt/run_test.sh $(pwd)" +} + +function _main() { + if [ -z "${MAKE_TEST_COMMAND}" ]; then + echo "FATAL: MAKE_TEST_COMMAND is not set" + exit 1 + fi + + time load_transfered_bits_into_install_dir + time configure + time setup_gpadmin_user + time make_cluster + time gen_env_for_gpadmin_to_run + time run_test +} + +_main "$@" diff --git a/concourse/tasks/compile_gpdb_open_source_ubuntu.yml b/concourse/tasks/compile_gpdb_open_source_ubuntu.yml index 11bba3e3c3..d9fd83c255 100644 --- a/concourse/tasks/compile_gpdb_open_source_ubuntu.yml +++ b/concourse/tasks/compile_gpdb_open_source_ubuntu.yml @@ -3,7 +3,11 @@ image_resource: type: docker-image inputs: - name: gpdb_src +outputs: + - name: compiled_bits_ubuntu16 run: path: gpdb_src/concourse/scripts/compile_gpdb_open_source_ubuntu.bash params: CONFIGURE_FLAGS: "" + TRANSFER_DIR: compiled_bits_ubuntu16 + COMPILED_BITS_FILENAME: compiled_bits_ubuntu16.tar.gz diff --git a/concourse/tasks/ic_gpdb_ubuntu.yml b/concourse/tasks/ic_gpdb_ubuntu.yml new file mode 100644 index 0000000000..6ac164218b --- /dev/null +++ b/concourse/tasks/ic_gpdb_ubuntu.yml @@ -0,0 +1,15 @@ +platform: linux +image_resource: + type: docker-image +inputs: + - name: gpdb_src + - name: compiled_bits_ubuntu16 +outputs: +params: + MAKE_TEST_COMMAND: "" + CONFIGURE_FLAGS: "" + WITH_MIRRORS: + TRANSFER_DIR: compiled_bits_ubuntu16 + COMPILED_BITS_FILENAME: compiled_bits_ubuntu16.tar.gz +run: + path: gpdb_src/concourse/scripts/ic_gpdb_ubuntu.bash -- GitLab