Add ICW jobs to pipeline template for Ubuntu 18.04

- ubuntu jobs are using the systems compiler and the Docker build image
  does not contain the GCC 6.4.0 package; CI scripts are modified to
  conditionally source the gcc environment setup script

- when running `make demo-cluster` on ubuntu, it looks like `su` does not
  preserve the environment set by "greenplum_path.sh" so we re-source
  "greenplum_path.sh" as gpadmin before starting the demo cluster
Co-authored-by: NJesse Zhang <jzhang@pivotal.io>
Co-authored-by: NBradford D. Boyle <bboyle@pivotal.io>

concourse/pipelines/templates/gpdb-tpl.yml

@@ -73,6 +73,10 @@ groups:
   - icw_gporca_centos7
   - icw_planner_centos7
 {% endif %}
+{% if "ubuntu18.04" in os_types %}
+  - icw_gporca_ubuntu18.04
+  - icw_planner_ubuntu18.04
+{% endif %}
 {% if "sles" in os_types %}
   - icw_gporca_sles11
   - icw_gporca_sles12
@@ -185,6 +189,10 @@ groups:
   - icw_planner_centos7
   - compile_gpdb_centos7
 {% endif %}
+{% if "ubuntu18.04" in os_types %}
+  - icw_gporca_ubuntu18.04
+  - icw_planner_ubuntu18.04
+{% endif %}
 {% if "sles" in os_types %}
   - icw_gporca_sles11
   - icw_gporca_sles12
@@ -496,6 +504,13 @@ resources:
   source:
     repository: pivotaldata/gpdb6-ubuntu18.04-build
     tag: latest
+
+- name: gpdb6-ubuntu18.04-test
+  type: docker-image
+  source:
+    repository: pivotaldata/gpdb6-ubuntu18.04-test
+    tag: latest
+
 {% endif %}
 {% if "centos6" in os_types %}
 - name: bin_gpdb_centos6
@@ -1165,6 +1180,44 @@ jobs:
       TEST_OS: centos
       CONFIGURE_FLAGS: {{configure_flags}}
 
+{% endif %}
+{% if "ubuntu18.04" in os_types %}
+- name: icw_gporca_ubuntu18.04
+  plan:
+    - aggregate:
+        - get: gpdb_src
+          passed: [compile_gpdb_ubuntu18.04]
+        - get: bin_gpdb
+          resource: bin_gpdb_ubuntu18.04
+          passed: [compile_gpdb_ubuntu18.04]
+          trigger: [[ test_trigger ]]
+        - get: gpdb6-ubuntu18.04-test
+    - task: ic_gpdb
+      file: gpdb_src/concourse/tasks/ic_gpdb.yml
+      image: gpdb6-ubuntu18.04-test
+      params:
+        MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on' installcheck-world
+        TEST_OS: ubuntu
+        CONFIGURE_FLAGS: {{configure_flags}}
+
+- name: icw_planner_ubuntu18.04
+  plan:
+    - aggregate:
+        - get: gpdb_src
+          passed: [compile_gpdb_ubuntu18.04]
+        - get: bin_gpdb
+          passed: [compile_gpdb_ubuntu18.04]
+          resource: bin_gpdb_ubuntu18.04
+          trigger: [[ test_trigger ]]
+        - get: gpdb6-ubuntu18.04-test
+    - task: ic_gpdb
+      file: gpdb_src/concourse/tasks/ic_gpdb.yml
+      image: gpdb6-ubuntu18.04-test
+      params:
+        MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world
+        TEST_OS: ubuntu
+        CONFIGURE_FLAGS: {{configure_flags}}
+
 {% endif %}
 {% if "sles" in os_types %}
 - name: icw_gporca_sles11

concourse/scripts/common.bash

@@ -27,7 +27,10 @@ function setup_configure_vars() {
 }
 
 function configure() {
-  source /opt/gcc_env.sh
+  if [ -f /opt/gcc_env.sh ]; then
+    # ubuntu uses the system compiler
+    source /opt/gcc_env.sh
+  fi
   pushd gpdb_src
       # The full set of configure options which were used for building the
       # tree must be used here as well since the toplevel Makefile depends
@@ -57,7 +60,7 @@ function make_cluster() {
   export DEFAULT_QD_MAX_CONNECT=150
   export STATEMENT_MEM=250MB
   pushd gpdb_src/gpAux/gpdemo
-  su gpadmin -c "make create-demo-cluster"
+  su gpadmin -c "source /usr/local/greenplum-db-devel/greenplum_path.sh; make create-demo-cluster"
   popd
 }
 

concourse/scripts/ic_gpdb.bash

@@ -29,7 +29,9 @@ function gen_env(){
 		    exit 1
 		}
 		source /usr/local/greenplum-db-devel/greenplum_path.sh
-		source /opt/gcc_env.sh
+		if [ -f /opt/gcc_env.sh ]; then
+		    source /opt/gcc_env.sh
+		fi
 		cd "\${1}/gpdb_src"
 		source gpAux/gpdemo/gpdemo-env.sh
 		make -s ${MAKE_TEST_COMMAND}
@@ -53,11 +55,14 @@ function _main() {
         exit 1
     fi
 
-    if [ "$TEST_OS" != "centos" -a "$TEST_OS" != "sles" ]; then
-        echo "FATAL: TEST_OS is set to an invalid value: $TEST_OS"
-	echo "Configure TEST_OS to be centos or sles"
-        exit 1
-    fi
+    case "${TEST_OS}" in
+    centos|sles|ubuntu) ;; #Valid
+    *)
+      echo "FATAL: TEST_OS is set to an invalid value: $TEST_OS"
+      echo "Configure TEST_OS to be centos, sles, or ubuntu"
+      exit 1
+      ;;
+    esac
 
     # This ugly block exists since sles11 installs kerberos at a different path that is a test-only dependency
     if [ "$TEST_OS" == "sles" ]; then

concourse/scripts/setup_gpadmin_user.bash

@@ -128,7 +128,7 @@ determine_os() {
     echo "sles"
     return
   fi
-  if lsb_release -a | grep -q 'Ubuntu' ; then
+  if grep -q ID=ubuntu /etc/os-release ; then
     echo "ubuntu"
     return
   fi