Extract test_orca as a separate job outside of compile step (#10819)

This is done to speed up development iterations by reducing the time the
of the "compile" job in the following pipelines, and starting ICW jobs
sooner. Runs the unit tests in parallel to ICW jobs.

Pipelines modified:
- gporca-dev
- master build pipeline
- master build pipeline without asserts
- PR pipeline
Co-authored-by: NShreedhar Hardikar <hardikar@cs.wisc.edu>
Co-authored-by: NChris Hajas <chajas@vmware.com>

concourse/pipelines/gpdb_master-generated.yml

@@ -12,7 +12,7 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2020-11-02 11:53:06.131787
+## Generated by gen_pipeline.py at: 2020-09-15 11:19:24.645167
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'ubuntu18.04', 'win']
 ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions']
@@ -181,6 +181,7 @@ groups:
   - icw_gporca_icproxy_centos7
   - icw_planner_icproxy_centos7
   - icw_gporca_ictcp_centos7
+  - unit_tests_gporca_centos7
   - gpexpand
   - pg_upgrade
   - icw_gporca_ubuntu18.04
@@ -850,6 +851,18 @@ jobs:
       MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=tcp -c optimizer=on' installcheck-world
       TEST_OS: centos
 
+- name: unit_tests_gporca_centos7
+  plan:
+  - in_parallel:
+      steps:
+      - get: gpdb_src
+        passed: [compile_gpdb_centos7]
+        trigger: true
+      - get: gpdb7-centos7-build
+  - task: unit_tests_gporca
+    file: gpdb_src/concourse/tasks/unit_tests_gporca.yml
+    image: gpdb7-centos7-build
+
 - name: icw_gporca_ubuntu18.04
   plan:
     - in_parallel:
@@ -941,6 +954,7 @@ jobs:
         - icw_gporca_icproxy_centos7
         - icw_planner_icproxy_centos7
         - icw_gporca_ictcp_centos7
+        - unit_tests_gporca_centos7
         - icw_extensions_gpcloud_centos7
       - get: icw_planner_centos7_dump
         passed:
@@ -952,6 +966,7 @@ jobs:
         - icw_gporca_icproxy_centos7
         - icw_planner_icproxy_centos7
         - icw_gporca_ictcp_centos7
+        - unit_tests_gporca_centos7
         - icw_extensions_gpcloud_centos7
         trigger: true
   - put: bin_gpdb_centos7_icw_green
@@ -3262,6 +3277,7 @@ jobs:
         - icw_gporca_icproxy_ubuntu18.04
         - icw_planner_icproxy_ubuntu18.04
         - cli_cross_subnet
+        - unit_tests_gporca_centos7
         - gpmovemirrors
         - gpmovemirrors_ubuntu18
         - gppkg
@@ -3313,6 +3329,7 @@ jobs:
         - icw_extensions_gpcloud_centos7
         - resource_group_centos7
         - cli_cross_subnet
+        - unit_tests_gporca_centos7
         - gpmovemirrors
         - gppkg
         - analyzedb

concourse/pipelines/pr_pipeline.yml

@@ -140,3 +140,10 @@ jobs:
           TEST_OS: centos
           CONFIGURE_FLAGS: ((configure_flags))
         timeout: 5h
+
+      - task: unit_tests_gporca_centos7
+        file: gpdb_pr/concourse/tasks/unit_tests_gporca.yml
+        image: gpdb7-centos7-build
+        input_mapping:
+          gpdb_src: gpdb_pr
+        timeout: 1h

concourse/pipelines/templates/gpdb-tpl.yml

@@ -208,6 +208,7 @@ groups:
   - icw_gporca_icproxy_centos7
   - icw_planner_icproxy_centos7
   - icw_gporca_ictcp_centos7
+  - unit_tests_gporca_centos7
 {% endif %}
 {% if "centos7" in os_types and "CLI" in test_sections %}
   - gpexpand
@@ -996,6 +997,18 @@ jobs:
       MAKE_TEST_COMMAND: -k PGOPTIONS='-c gp_interconnect_type=tcp -c optimizer=on' installcheck-world
       TEST_OS: centos
 
+- name: unit_tests_gporca_centos7
+  plan:
+  - in_parallel:
+      steps:
+      - get: gpdb_src
+        passed: [compile_gpdb_centos7]
+        trigger: [[ test_trigger ]]
+      - get: gpdb7-centos7-build
+  - task: unit_tests_gporca
+    file: gpdb_src/concourse/tasks/unit_tests_gporca.yml
+    image: gpdb7-centos7-build
+
 {% endif %}
 {% if "ubuntu18.04" in os_types %}
 - name: icw_gporca_ubuntu18.04
@@ -1099,6 +1112,7 @@ jobs:
         - icw_gporca_icproxy_centos7
         - icw_planner_icproxy_centos7
         - icw_gporca_ictcp_centos7
+        - unit_tests_gporca_centos7
 {% if "Extensions" in test_sections %}
         - icw_extensions_gpcloud_centos7
 {% endif %}
@@ -1114,6 +1128,7 @@ jobs:
         - icw_gporca_icproxy_centos7
         - icw_planner_icproxy_centos7
         - icw_gporca_ictcp_centos7
+        - unit_tests_gporca_centos7
 {% if "Extensions" in test_sections %}
         - icw_extensions_gpcloud_centos7
 {% endif %}
@@ -1822,6 +1837,7 @@ jobs:
         - icw_gporca_icproxy_ubuntu18.04
         - icw_planner_icproxy_ubuntu18.04
         - cli_cross_subnet
+        - unit_tests_gporca_centos7
 {% for test in CLI_BEHAVE_TESTS %}
         - [[ test.name ]]
 {% if "ubuntu18.04" in os_types %}
@@ -1847,6 +1863,7 @@ jobs:
         - icw_extensions_gpcloud_centos7
         - resource_group_centos7
         - cli_cross_subnet
+        - unit_tests_gporca_centos7
 {% for test in CLI_BEHAVE_TESTS %}
         - [[ test.name ]]
 {% endfor %}

concourse/scripts/compile_gpdb.bash

@@ -230,18 +230,6 @@ function build_xerces()
     rm -rf build
 }
 
-function test_orca()
-{
-    if [ -n "${SKIP_UNITTESTS}" ]; then
-        return
-    fi
-    OUTPUT_DIR="../../../../gpAux/ext/${BLD_ARCH}"
-    pushd ${GPDB_SRC_PATH}/src/backend/gporca
-    concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR}
-    concourse/build_and_test.py --build_type=Debug --output_dir=${OUTPUT_DIR}
-    popd
-}
-
 function _main() {
   mkdir gpdb_src/gpAux/ext
 
@@ -249,7 +237,6 @@ function _main() {
     centos|ubuntu|sles)
       prep_env
       build_xerces
-      test_orca
       install_deps
       ;;
     win32)

concourse/scripts/unit_tests_gporca.bash

+#!/bin/bash
+set -exo pipefail
+
+GPDB_SRC_PATH=${GPDB_SRC_PATH:=gpdb_src}
+
+function build_xerces
+{
+    OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}"
+    mkdir -p xerces_patch/concourse
+    cp -r gpdb_src/src/backend/gporca/concourse/xerces-c xerces_patch/concourse
+    /usr/bin/python xerces_patch/concourse/xerces-c/build_xerces.py --output_dir=${OUTPUT_DIR}
+    rm -rf build
+}
+
+function test_orca
+{
+    if [ -n "${SKIP_UNITTESTS}" ]; then
+        return
+    fi
+    OUTPUT_DIR="../../../../gpAux/ext/${BLD_ARCH}"
+    pushd ${GPDB_SRC_PATH}/src/backend/gporca
+    concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR}
+    concourse/build_and_test.py --build_type=Debug --output_dir=${OUTPUT_DIR}
+    popd
+}
+
+function _main
+{
+  mkdir gpdb_src/gpAux/ext
+  build_xerces
+  test_orca
+}
+
+_main "$@"

concourse/tasks/unit_tests_gporca.yml

+platform: linux
+image_resource:
+  type: docker-image
+inputs:
+  - name: gpdb_src
+run:
+  path: gpdb_src/concourse/scripts/unit_tests_gporca.bash

src/backend/gporca/concourse/test_orca_pipeline.yml

@@ -116,3 +116,8 @@ jobs:
           TEST_OS: centos
           CONFIGURE_FLAGS: ((configure_flags))
         timeout: 5h
+
+      - task: unit_tests_gporca_centos7
+        file: gpdb_src/concourse/tasks/unit_tests_gporca.yml
+        image: gpdb7-centos7-build
+        timeout: 1h