sles11 compile fix; sync_tools refactor

This is a common issue with sles11 - that it doesn't
currently include support for > TLS v1.1. Many upstream endpoints over
the last 6 months have started to enforce > TLS v1.1.

We resolved this by separating the sync_tools call into a separate task,
such that we could run this task from a centos docker image, prior to
compiling on the correct OS.

These changes were backported to other compile jobs. We are pushing this
change to resolve the sles11 blocker, but we are still experiencing
difficulty with windows.
Co-authored-by: NLisa Oakley <loakley@pivotal.io>
Co-authored-by: NTrevor Yacovone <tyacovone@pivotal.io>
Co-authored-by: NEd Espino <edespino@pivotal.io>

concourse/pipelines/templates/gpdb-tpl.yml

@@ -603,13 +603,13 @@ resources:
 
 {% endif %}
 {% if "aix7" in os_types %}
-- name: nightly-trigger
-  type: time
-  source:
-    location: America/Los_Angeles
-    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
-    start: 6:00 AM
-    stop: 7:00 AM
+#- name: nightly-trigger
+#  type: time
+#  source:
+#    location: America/Los_Angeles
+#    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
+#    start: 6:00 AM
+#    stop: 7:00 AM
 
 {% endif %}
 - name: reduced-frequency-trigger
@@ -768,14 +768,22 @@ jobs:
       trigger: ((reduced-frequency-trigger-flag))
     - get: gpaddon_src
     - get: centos-gpdb-dev-6
-  - task: compile_gpdb
-    file: gpdb_src/concourse/tasks/compile_gpdb.yml
+  - task: sync_tools
+    file: gpdb_src/concourse/tasks/sync_tools.yml
     image: centos-gpdb-dev-6
     params:
       IVYREPO_HOST: {{ivyrepo_host}}
       IVYREPO_REALM: {{ivyrepo_realm}}
       IVYREPO_USER: {{ivyrepo_user}}
       IVYREPO_PASSWD: {{ivyrepo_passwd}}
+      TARGET_OS: centos
+      TARGET_OS_VERSION: 6
+      TASK_OS: centos
+      TASK_OS_VERSION: 6
+  - task: compile_gpdb
+    file: gpdb_src/concourse/tasks/compile_gpdb.yml
+    image: centos-gpdb-dev-6
+    params:
       CONFIGURE_FLAGS: {{configure_flags}}
       TARGET_OS: centos
       TARGET_OS_VERSION: 6
@@ -796,14 +804,22 @@ jobs:
       trigger: ((gpdb_src-trigger-flag))
     - get: gpaddon_src
     - get: centos-gpdb-build-7
-  - task: compile_gpdb
+  - task: sync_tools
+    file: gpdb_src/concourse/tasks/sync_tools.yml
     image: centos-gpdb-build-7
-    file: gpdb_src/concourse/tasks/compile_gpdb.yml
     params:
       IVYREPO_HOST: {{ivyrepo_host}}
       IVYREPO_REALM: {{ivyrepo_realm}}
       IVYREPO_USER: {{ivyrepo_user}}
       IVYREPO_PASSWD: {{ivyrepo_passwd}}
+      TARGET_OS: centos
+      TARGET_OS_VERSION: 7
+      TASK_OS: centos
+      TASK_OS_VERSION: 7
+  - task: compile_gpdb
+    image: centos-gpdb-build-7
+    file: gpdb_src/concourse/tasks/compile_gpdb.yml
+    params:
       CONFIGURE_FLAGS: {{configure_flags}}
       TARGET_OS: centos
       TARGET_OS_VERSION: 7
@@ -823,13 +839,22 @@ jobs:
     - get: gpdb_src
       trigger: ((gpdb_src-trigger-flag))
     - get: gpaddon_src
-  - task: compile_gpdb
-    file: gpdb_src/concourse/tasks/compile_gpdb_sles11.yml
+    - get: centos-gpdb-build-7
+  - task: sync_tools
+    file: gpdb_src/concourse/tasks/sync_tools.yml
+    image: centos-gpdb-build-7
     params:
       IVYREPO_HOST: {{ivyrepo_host}}
       IVYREPO_REALM: {{ivyrepo_realm}}
       IVYREPO_USER: {{ivyrepo_user}}
       IVYREPO_PASSWD: {{ivyrepo_passwd}}
+      TARGET_OS: sles
+      TARGET_OS_VERSION:
+      TASK_OS: centos
+      TASK_OS_VERSION: 7
+  - task: compile_gpdb
+    file: gpdb_src/concourse/tasks/compile_gpdb_sles11.yml
+    params:
       CONFIGURE_FLAGS: {{configure_flags}}
       TARGET_OS: sles
       TARGET_OS_VERSION:
@@ -931,18 +956,30 @@ jobs:
       tags: ["wix"]
     - get: centos-mingw
       tags: ["wix"]
+    - get: centos-gpdb-build-7
+      tags: ["wix"]
+  - task: sync_tools
+    tags: ["wix"]
+    file: gpdb_src/concourse/tasks/sync_tools.yml
+    image: centos-gpdb-build-7
+    params:
+      IVYREPO_HOST: {{ivyrepo_host}}
+      IVYREPO_REALM: {{ivyrepo_realm}}
+      IVYREPO_USER: {{ivyrepo_user}}
+      IVYREPO_PASSWD: {{ivyrepo_passwd}}
+      TARGET_OS: win32
+      TARGET_OS_VERSION:
+      TASK_OS: centos
+      TASK_OS_VERSION: 7
   - task: compile_gpdb
     tags: ["wix"]
     file: gpdb_src/concourse/tasks/compile_gpdb.yml
     image: centos-mingw
     params:
+      CONFIGURE_FLAGS: {{configure_flags}}
       TARGET_OS: win32
       TARGET_OS_VERSION:
-      BLD_TARGETS: clients loaders
-      IVYREPO_HOST: {{ivyrepo_host}}
-      IVYREPO_REALM: {{ivyrepo_realm}}
-      IVYREPO_USER: {{ivyrepo_user}}
-      IVYREPO_PASSWD: {{ivyrepo_passwd}}
+      BLD_TARGETS: "clients loaders"
   - aggregate:
     - put: bin_gpdb_windows_clients
       tags: ["wix"]
@@ -962,8 +999,8 @@ jobs:
   # We need to serialize this job to avoid overwhelming workload
   # on remote machine.
   - aggregate:
-    - get: nightly-trigger
-      trigger: true
+#    - get: nightly-trigger
+#      trigger: true
     - get: gpdb_src
     - get: gpaddon_src
     - get: centos-gpdb-dev-6
@@ -973,8 +1010,6 @@ jobs:
     params:
       REMOTE_HOST: {{remote_host_build}}
       REMOTE_PORT: {{remote_port_build}}
-      REMOTE_USER: {{remote_user_build}}
-      REMOTE_KEY: {{remote_key_build}}
       IVYREPO_HOST: {{ivyrepo_host}}
       IVYREPO_REALM: {{ivyrepo_realm}}
       IVYREPO_USER: {{ivyrepo_user}}

concourse/scripts/compile_gpdb.bash

@@ -18,17 +18,17 @@ function expand_glob_ensure_exists() {
 function prep_env_for_centos() {
   case "${TARGET_OS_VERSION}" in
     5)
-      BLDARCH=rhel5_x86_64
+      BLD_ARCH=rhel5_x86_64
       export JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.39.x86_64
       ;;
 
     6)
-      BLDARCH=rhel6_x86_64
+      BLD_ARCH=rhel6_x86_64
       export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64
       ;;
 
     7)
-      BLDARCH=rhel7_x86_64
+      BLD_ARCH=rhel7_x86_64
       echo "Detecting java7 path ..."
       java7_packages=$(rpm -qa | grep -F java-1.7)
       java7_bin="$(rpm -ql ${java7_packages} | grep /jre/bin/java$)"
@@ -44,7 +44,7 @@ function prep_env_for_centos() {
   esac
 
   source /opt/gcc_env.sh
-  ln -sf $(pwd)/${GPDB_SRC_PATH}/gpAux/ext/${BLDARCH}/python-2.7.12 /opt/python-2.7.12
+  ln -sf $(pwd)/${GPDB_SRC_PATH}/gpAux/ext/${BLD_ARCH}/python-2.7.12 /opt/python-2.7.12
   export PATH=${JAVA_HOME}/bin:${PATH}
 }
 
@@ -64,14 +64,6 @@ function generate_build_number() {
   popd
 }
 
-function make_sync_tools() {
-  pushd ${GPDB_SRC_PATH}/gpAux
-    # Requires these variables in the env:
-    # IVYREPO_HOST IVYREPO_REALM IVYREPO_USER IVYREPO_PASSWD
-    make sync_tools
-  popd
-}
-
 function link_tools_for_sles() {
   ln -sf "$(expand_glob_ensure_exists $(pwd)/${GPDB_SRC_PATH}/gpAux/ext/*/python-2.7.12 )" /opt
 }
@@ -152,7 +144,10 @@ function _main() {
   esac
 
   generate_build_number
-  make_sync_tools
+  
+  # Copy input ext dir; assuming ext doesnt exist
+  mv gpAux_ext/ext ${GPDB_SRC_PATH}/gpAux
+
   case "${TARGET_OS}" in
     sles) link_tools_for_sles ;;
   esac

concourse/scripts/scan_with_coverity.bash

@@ -6,7 +6,7 @@ export GPDB_ARTIFACTS_DIR
 GPDB_ARTIFACTS_DIR=$BASE_DIR/$OUTPUT_ARTIFACT_DIR
 
 function prep_env_for_centos() {
-  BLDARCH=rhel7_x86_64
+  BLD_ARCH=rhel7_x86_64
   echo "Detecting java7 path ..."
   java7_packages=($(rpm -qa | grep -F java-1.7))
   java7_bin="$(rpm -ql "${java7_packages[@]}" | grep /jre/bin/java$)"
@@ -15,7 +15,7 @@ function prep_env_for_centos() {
   ln -sf /usr/bin/xsubpp /usr/share/perl5/ExtUtils/xsubpp
   source /opt/gcc_env.sh
 
-  ln -sf "$BASE_DIR"/gpdb_src/gpAux/ext/${BLDARCH}/python-2.7.12 /opt/python-2.7.12
+  ln -sf "$BASE_DIR"/gpdb_src/gpAux/ext/${BLD_ARCH}/python-2.7.12 /opt/python-2.7.12
   export PATH=${JAVA_HOME}/bin:${PATH}
 }
 

concourse/scripts/sync_tools.bash

+#!/bin/bash -l
+set -exo pipefail
+
+CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+GPDB_SRC_PATH=${GPDB_SRC_PATH:=gpdb_src}
+
+
+function make_sync_tools() {
+  pushd ${GPDB_SRC_PATH}/gpAux
+    # Requires these variables in the env:
+    # IVYREPO_HOST IVYREPO_REALM IVYREPO_USER IVYREPO_PASSWD
+    make sync_tools
+  popd
+}
+
+function _main() {
+  case "${TARGET_OS}" in
+    centos)
+        case "${TARGET_OS_VERSION}" in
+         5)
+           BLD_ARCH=rhel5_x86_64
+           ;;
+
+         6)
+           BLD_ARCH=rhel6_x86_64
+           ;;
+
+         7)
+           BLD_ARCH=rhel7_x86_64 
+           ;;
+        esac
+        ;;
+    sles)
+        export BLD_ARCH=sles11_x86_64
+        ;;
+    win32)
+        export BLD_ARCH=win32
+        ;;
+    *)
+        echo "only centos, sles and win32 are supported TARGET_OS'es"
+        false
+        ;;
+  esac
+
+  case "${TASK_OS}" in
+    centos)
+        case "${TASK_OS_VERSION}" in
+         5)
+           export JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.39.x86_64
+           ;;
+
+         6)
+           export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64
+           ;;
+
+         7)
+           echo "Detecting java7 path ..."
+           java7_packages=$(rpm -qa | grep -F java-1.7)
+           java7_bin="$(rpm -ql ${java7_packages} | grep /jre/bin/java$)"
+           alternatives --set java "$java7_bin"
+           export JAVA_HOME="${java7_bin/jre\/bin\/java/}"
+           ln -sf /usr/bin/xsubpp /usr/share/perl5/ExtUtils/xsubpp
+           ;;
+
+        esac
+        ;;
+    *)
+        echo "only centos 6 and 7 are supported TASK_OS'es"
+        false
+        ;;
+  esac
+
+  make_sync_tools
+
+  # Move ext directory to output dir
+  mv ${GPDB_SRC_PATH}/gpAux/ext gpAux_ext/
+
+}
+
+_main "$@"

concourse/tasks/compile_gpdb.yml

@@ -4,6 +4,7 @@ image_resource:
 inputs:
   - name: gpdb_src
   - name: gpaddon_src
+  - name: gpAux_ext
 outputs:
   - name: gpdb_artifacts
 run:
@@ -13,8 +14,4 @@ params:
   TARGET_OS_VERSION:
   BLD_TARGETS:
   OUTPUT_ARTIFACT_DIR: gpdb_artifacts
-  IVYREPO_HOST:
-  IVYREPO_REALM:
-  IVYREPO_USER:
-  IVYREPO_PASSWD:
   CONFIGURE_FLAGS:

concourse/tasks/compile_gpdb_sles11.yml

@@ -10,6 +10,7 @@ image_resource:
 inputs:
 - name: gpdb_src
 - name: gpaddon_src
+- name: gpAux_ext
 
 outputs:
 - name: gpdb_artifacts
@@ -22,8 +23,4 @@ params:
   TARGET_OS_VERSION:
   BLD_TARGETS:
   OUTPUT_ARTIFACT_DIR: gpdb_artifacts
-  IVYREPO_HOST:
-  IVYREPO_REALM:
-  IVYREPO_USER:
-  IVYREPO_PASSWD:
   CONFIGURE_FLAGS:

concourse/tasks/sync_tools.yml

+---
+platform: linux
+
+image_resource:
+  type: docker-image
+
+inputs:
+- name: gpdb_src
+- name: gpaddon_src
+
+outputs:
+- name: gpAux_ext
+
+run:
+  path: gpdb_src/concourse/scripts/sync_tools.bash
+
+params:
+  TARGET_OS:
+  TARGET_OS_VERSION:
+  TASK_OS:
+  TASK_OS_VERSION:
+  IVYREPO_HOST:
+  IVYREPO_REALM:
+  IVYREPO_USER:
+  IVYREPO_PASSWD: