Test ubuntu with ICW: configure, Make demo cluster

- We receive compiled GPDB tar output through a shared dir
- Separate Compile and ICW jobs for ubuntu
- Increase number of parallel client connections
Signed-off-by: NC.J. Jameson <cjameson@pivotal.io>

concourse/pipelines/pipeline.yml

@@ -25,6 +25,7 @@ groups:
   - icw_gporca_centos6
   - icw_gporca_centos7
   - icw_gporca_sles11
+  - icw_planner_ubuntu16
   - icw_planner_ictcp_centos6
   - icw_gporca_centos6_gpos_memory
   - fts
@@ -355,6 +356,15 @@ resources:
     secret_access_key: {{bucket-secret-access-key}}
     versioned_file: {{bin_gpdb_centos_versioned_file}}
 
+- name: compiled_bits_ubuntu16
+  type: s3
+  source:
+    access_key_id: {{bucket-access-key-id}}
+    bucket: {{bucket-name}}
+    region_name: {{aws-region}}
+    secret_access_key: {{bucket-secret-access-key}}
+    versioned_file: {{compiled_bits_ubuntu16_versioned_file}}
+
 - name: binary_swap_gpdb_centos6
   type: s3
   source:
@@ -750,6 +760,24 @@ jobs:
     - put: installer_rhel7_gpdb_loaders
       params:
         file: gpdb_artifacts/greenplum-loaders-*-rhel7-x86_64.zip
+
+- name: compile_gpdb_ubuntu16
+  plan:
+  - aggregate:
+    - get: reduced-frequency-trigger
+      trigger: {{reduced-frequency-trigger-flag}}
+    - get: gpdb_src
+      trigger: {{gpdb_src-trigger-flag}}
+    - get: ubuntu-gpdb-dev-16
+  - task: compile_gpdb
+    image: ubuntu-gpdb-dev-16
+    file: gpdb_src/concourse/tasks/compile_gpdb_open_source_ubuntu.yml
+    params:
+      CONFIGURE_FLAGS: {{configure_flags}}
+  - put: compiled_bits_ubuntu16
+    params:
+      file: compiled_bits_ubuntu16/compiled_bits_ubuntu16.tar.gz
+
 - name: compile_gpdb_sles11
   plan:
   - aggregate:
@@ -1179,7 +1207,22 @@ jobs:
       BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
       TEST_OS: sles
       CONFIGURE_FLAGS: {{configure_flags}}
-- name: icw_gporca_centos6_gpos_memory
+
+- name: icw_planner_ubuntu16
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_ubuntu16]
+    - get: compiled_bits_ubuntu16
+      passed: [compile_gpdb_ubuntu16]
+      trigger: true
+    - get: ubuntu-gpdb-dev-16
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb_ubuntu.yml
+    image: ubuntu-gpdb-dev-16
+    params:
+      MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off' installcheck-world
+      CONFIGURE_FLAGS: {{configure_flags}}- name: icw_gporca_centos6_gpos_memory
   plan:
   - aggregate:
     - get: gpdb_src
@@ -2640,6 +2683,7 @@ jobs:
     - icw_gporca_centos6
     - icw_gporca_centos7
     - icw_gporca_sles11
+    - icw_planner_ubuntu16
     - icw_planner_ictcp_centos6
     - client_loader_remote_test_aix
     - mpp_resource_group_centos6

concourse/scripts/compile_gpdb_open_source_ubuntu.bash

@@ -2,19 +2,13 @@
 set -exo pipefail
 
 GREENPLUM_INSTALL_DIR=/usr/local/gpdb
-
-# This is a canonical way to build GPDB. The intent is to validate that GPDB compiles
-# with a fairly basic build. It is not meant to be exhaustive or include all features
-# and components available in GPDB.
+TRANSFER_DIR_ABSOLUTE_PATH="$(pwd)/${TRANSFER_DIR}"
+COMPILED_BITS_FILENAME=${COMPILED_BITS_FILENAME:="compiled_bits_ubuntu16.tar.gz"}
 
 function build_gpdb() {
   pushd gpdb_src
-    CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-perl --with-libxml \
-	--disable-orca --with-python --disable-gpfdist --prefix=${GREENPLUM_INSTALL_DIR}
-    # Use -j4 to speed up the build. (Doesn't seem worth trying to guess a better
-    # value based on number of CPUs or anything like that. Going above -j4 wouldn't
-    # make it much faster, and -j4 is small enough to not hurt too badly even on
-    # a single-CPU system
+    CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-gssapi --with-perl --with-libxml \
+	--disable-orca --with-python --prefix=${GREENPLUM_INSTALL_DIR}
     make -j4
     make install
   popd
@@ -24,9 +18,20 @@ function unittest_check_gpdb() {
   make -C gpdb_src/src/backend -s unittest-check
 }
 
+function export_gpdb() {
+  TARBALL="$TRANSFER_DIR_ABSOLUTE_PATH"/$COMPILED_BITS_FILENAME
+  pushd $GREENPLUM_INSTALL_DIR
+    source greenplum_path.sh
+    python -m compileall -x test .
+    chmod -R 755 .
+    tar -czf "${TARBALL}" ./*
+  popd
+}
+
 function _main() {
   build_gpdb
   unittest_check_gpdb
+  export_gpdb
 }
 
 _main "$@"

concourse/scripts/ic_gpdb_ubuntu.bash

+#!/bin/bash -l
+
+set -eox pipefail
+
+CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+GREENPLUM_INSTALL_DIR=/usr/local/gpdb
+
+function load_transfered_bits_into_install_dir() {
+  mkdir -p $GREENPLUM_INSTALL_DIR
+  tar xzf $TRANSFER_DIR/$COMPILED_BITS_FILENAME -C $GREENPLUM_INSTALL_DIR
+}
+
+function configure() {
+  pushd gpdb_src
+    ./configure --prefix=${GREENPLUM_INSTALL_DIR} --with-gssapi --with-perl --with-python --with-libxml --enable-mapreduce --disable-orca --enable-pxf ${CONFIGURE_FLAGS}
+  popd
+}
+
+function setup_gpadmin_user() {
+    ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash ubuntu
+}
+
+function make_cluster() {
+  export BLDWRAP_POSTGRES_CONF_ADDONS="fsync=off"
+  source "${GREENPLUM_INSTALL_DIR}/greenplum_path.sh"
+  export DEFAULT_QD_MAX_CONNECT=150
+  pushd gpdb_src/gpAux/gpdemo
+    su gpadmin -c "make create-demo-cluster"
+  popd
+}
+
+function gen_env_for_gpadmin_to_run(){
+  cat > /opt/run_test.sh <<-EOF
+		trap look4diffs ERR
+
+		function look4diffs() {
+
+		    diff_files=\`find .. -name regression.diffs\`
+
+		    for diff_file in \${diff_files}; do
+			if [ -f "\${diff_file}" ]; then
+			    cat <<-FEOF
+
+						======================================================================
+						DIFF FILE: \${diff_file}
+						----------------------------------------------------------------------
+
+						\$(cat "\${diff_file}")
+
+					FEOF
+			fi
+		    done
+		    exit 1
+		}
+		source ${GREENPLUM_INSTALL_DIR}/greenplum_path.sh
+		cd "\${1}/gpdb_src"
+		source gpAux/gpdemo/gpdemo-env.sh
+		make ${MAKE_TEST_COMMAND}
+	EOF
+
+	chmod a+x /opt/run_test.sh
+}
+
+function run_test() {
+  su - gpadmin -c "bash /opt/run_test.sh $(pwd)"
+}
+
+function _main() {
+    if [ -z "${MAKE_TEST_COMMAND}" ]; then
+        echo "FATAL: MAKE_TEST_COMMAND is not set"
+        exit 1
+    fi
+
+    time load_transfered_bits_into_install_dir
+    time configure
+    time setup_gpadmin_user
+    time make_cluster
+    time gen_env_for_gpadmin_to_run
+    time run_test
+}
+
+_main "$@"

concourse/tasks/compile_gpdb_open_source_ubuntu.yml

@@ -3,7 +3,11 @@ image_resource:
   type: docker-image
 inputs:
   - name: gpdb_src
+outputs:
+  - name: compiled_bits_ubuntu16
 run:
   path: gpdb_src/concourse/scripts/compile_gpdb_open_source_ubuntu.bash
 params:
   CONFIGURE_FLAGS: ""
+  TRANSFER_DIR: compiled_bits_ubuntu16
+  COMPILED_BITS_FILENAME: compiled_bits_ubuntu16.tar.gz

concourse/tasks/ic_gpdb_ubuntu.yml

+platform: linux
+image_resource:
+  type: docker-image
+inputs:
+  - name: gpdb_src
+  - name: compiled_bits_ubuntu16
+outputs:
+params:
+  MAKE_TEST_COMMAND: ""
+  CONFIGURE_FLAGS: ""
+  WITH_MIRRORS:
+  TRANSFER_DIR: compiled_bits_ubuntu16
+  COMPILED_BITS_FILENAME: compiled_bits_ubuntu16.tar.gz
+run:
+  path: gpdb_src/concourse/scripts/ic_gpdb_ubuntu.bash