CLI code coverage: refactor common CLI functions into common.bash

Also update gpperfmon to collect and upload coverage.
Co-authored-by: NKalen Krempely <kkrempely@pivotal.io>

concourse/pipelines/templates/gpdb-tpl.yml

@@ -1388,11 +1388,25 @@ jobs:
       passed: [gate_gpperfmon_start]
       trigger: true
     - get: gpdb6-centos6-test
+    - get: gpdb6-centos7-test
   - task: gpperfmon
     file: gpdb_src/concourse/tasks/behave_gpdb.yml
     image: gpdb6-centos6-test
     params:
       BEHAVE_TAGS: gpperfmon
+      TEST_NAME: gpperfmon_centos6
+  - task: publish_coverage
+    image: gpdb6-centos7-test
+    config:
+      platform: linux
+      inputs:
+        - name: gpdb_src
+        - name: coverage
+      run:
+        path: gpdb_src/concourse/scripts/gsutil_sync
+        args: [ "./coverage/", "gs://((coverage-bucket-name))/((pipeline-name))/" ]
+      params:
+        JSON_KEY: ((concourse-gcs-resources-service-account-key))
 
 {% if "centos7" in os_types %}
 - name: gpperfmon_centos7
@@ -1413,6 +1427,19 @@ jobs:
     image: gpdb6-centos7-test
     params:
       BEHAVE_TAGS: gpperfmon
+      TEST_NAME: gpperfmon_centos7
+  - task: publish_coverage
+    image: gpdb6-centos7-test
+    config:
+      platform: linux
+      inputs:
+        - name: gpdb_src
+        - name: coverage
+      run:
+        path: gpdb_src/concourse/scripts/gsutil_sync
+        args: [ "./coverage/", "gs://((coverage-bucket-name))/((pipeline-name))/" ]
+      params:
+        JSON_KEY: ((concourse-gcs-resources-service-account-key))
 
 {% endif %}
 

concourse/scripts/behave_gpdb.bash

@@ -5,60 +5,6 @@ set -eox pipefail
 CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source "${CWDIR}/common.bash"
 
-function install_python_hacks() {
-    # Our Python installation doesn't run standalone; it requires
-    # LD_LIBRARY_PATH which causes virtualenv to fail (because the system and
-    # vendored libpythons collide). We'll try our best to install patchelf to
-    # fix this later, but it's not available on all platforms.
-    if which yum > /dev/null; then
-        yum install -y patchelf
-    else
-        set +x
-        echo "ERROR: install_python_hacks() doesn't support the current platform and should be modified"
-        exit 1
-    fi
-}
-
-function install_python_requirements() {
-    # virtualenv 16.0 and greater does not support python2.6, which is
-    # used on centos6
-    pip install --user virtualenv~=15.0
-    export PATH=$PATH:~/.local/bin
-
-    # create virtualenv before sourcing greenplum_path since greenplum_path
-    # modifies PYTHONHOME and PYTHONPATH
-    #
-    # XXX Patch up the vendored Python's RPATH so we can successfully run
-    # virtualenv. If we instead set LD_LIBRARY_PATH (as greenplum_path.sh
-    # does), the system Python and the vendored Python will collide and
-    # virtualenv will fail. This step requires patchelf.
-    if which patchelf > /dev/null; then
-        patchelf \
-            --set-rpath /usr/local/greenplum-db-devel/ext/python/lib \
-            /usr/local/greenplum-db-devel/ext/python/bin/python
-
-        virtualenv \
-            --python /usr/local/greenplum-db-devel/ext/python/bin/python /tmp/venv
-    else
-        # We don't have patchelf on this environment. The only workaround we
-        # currently have is to set both PYTHONHOME and LD_LIBRARY_PATH and
-        # pray that the resulting libpython collision doesn't break
-        # something too badly.
-        echo 'WARNING: about to execute a cross-linked virtualenv; here there be dragons'
-        LD_LIBRARY_PATH=/usr/local/greenplum-db-devel/ext/python/lib \
-        PYTHONHOME=/usr/local/greenplum-db-devel/ext/python \
-        virtualenv \
-            --python /usr/local/greenplum-db-devel/ext/python/bin/python /tmp/venv
-    fi
-
-    # Install requirements into the vendored Python stack.
-    mkdir -p /tmp/py-requirements
-    source /tmp/venv/bin/activate
-        pip install --prefix /tmp/py-requirements -r ./gpdb_src/gpMgmt/requirements-dev.txt
-        cp -r /tmp/py-requirements/* /usr/local/greenplum-db-devel/ext/python/
-    deactivate
-}
-
 function gen_env(){
     cat > /opt/run_test.sh <<-EOF
 		set -ex
@@ -81,38 +27,6 @@ function gen_env(){
     chmod a+x /opt/run_test.sh
 }
 
-function setup_coverage() {
-    # Enables coverage.py on this machine. Note that this function modifies
-    # greenplum_path.sh, so callers need to source that file AFTER this is done.
-    local commit_sha
-    read -r commit_sha < ./gpdb_src/.git/HEAD
-    local coverage_path="$(pwd)/coverage/$commit_sha"
-
-    # This file will be copied into GPDB's PYTHONPATH; it sets up the coverage
-    # hook for all Python source files that are executed.
-    cat > /tmp/sitecustomize.py <<SITEEOF
-import coverage
-coverage.process_startup()
-SITEEOF
-
-    # Set up coverage.py to handle analysis from multiple parallel processes.
-    cat > /tmp/coveragerc <<COVEOF
-[run]
-branch = True
-data_file = $coverage_path/coverage
-parallel = True
-COVEOF
-
-    # Now copy everything over to the installation.
-    cp /tmp/sitecustomize.py /usr/local/greenplum-db-devel/lib/python
-    cp /tmp/coveragerc /usr/local/greenplum-db-devel
-    mkdir -p $coverage_path
-    chown gpadmin:gpadmin $coverage_path
-
-    # Enable coverage instrumentation after sourcing greenplum_path.
-    echo 'export COVERAGE_PROCESS_START=/usr/local/greenplum-db-devel/coveragerc' >> /usr/local/greenplum-db-devel/greenplum_path.sh
-}
-
 function _main() {
 
     if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then
@@ -128,15 +42,23 @@ function _main() {
     time (make_cluster)
 
     time install_python_hacks
-    time install_python_requirements
+    time install_python_requirements_on_single_host ./gpdb_src/gpMgmt/requirements-dev.txt
+
     time gen_env
 
-    # Enable coverage.py.
-    time setup_coverage
+    # need to run setup_coverage as gpadmin due to scp and ssh commands
+    time su gpadmin -c "
+        source ./gpdb_src/concourse/scripts/common.bash
+        # setup hostfile_all for demo_cluster tests
+        echo localhost > /tmp/hostfile_all
+        setup_coverage ./gpdb_src
+    "
 
     time run_test
 
-    time prepare_coverage "$TEST_NAME"
+    # collect coverage
+    cp -r /tmp/coverage/* ./coverage/
+    time tar_coverage "${TEST_NAME}_demo"
 }
 
 _main "$@"

concourse/scripts/common.bash

@@ -66,16 +66,7 @@ function install_python_hacks() {
     # LD_LIBRARY_PATH which causes virtualenv to fail (because the system and
     # vendored libpythons collide). We'll try our best to install patchelf to
     # fix this later, but it's not available on all platforms.
-    if which zypper > /dev/null; then
-        zypper addrepo https://download.opensuse.org/repositories/devel:tools:building/SLE_12_SP4/devel:tools:building.repo
-        # Note that this will fail on SLES11.
-        if ! zypper --non-interactive --gpg-auto-import-keys install patchelf; then
-            set +x
-            echo 'WARNING: could not install patchelf; virtualenv may fail later'
-            echo 'WARNING: This is a known issue on SLES11.'
-            set -x
-        fi
-    elif which yum > /dev/null; then
+    if which yum > /dev/null; then
         yum install -y patchelf
     else
         set +x
@@ -84,7 +75,7 @@ function install_python_hacks() {
     fi
 }
 
-function install_python_requirements() {
+function _install_python_requirements() {
     # virtualenv 16.0 and greater does not support python2.6, which is
     # used on centos6
     pip install --user virtualenv~=15.0
@@ -115,21 +106,41 @@ function install_python_requirements() {
         virtualenv \
             --python /usr/local/greenplum-db-devel/ext/python/bin/python /tmp/venv
     fi
+}
+
+function install_python_requirements_on_single_host() {
+    local requirements_txt="$1"
+    _install_python_requirements
 
-    # Install requirements into the vendored Python stack.
+    # Install requirements into the vendored Python stack
     mkdir -p /tmp/py-requirements
     source /tmp/venv/bin/activate
-        pip install --prefix /tmp/py-requirements -r ./gpdb_src/gpMgmt/requirements-dev.txt
+        pip install --prefix /tmp/py-requirements -r ${requirements_txt}
         cp -r /tmp/py-requirements/* /usr/local/greenplum-db-devel/ext/python/
     deactivate
 }
 
+function install_python_requirements_on_multi_host() {
+    local requirements_txt="$1"
+    _install_python_requirements
+
+    # Install requirements into the vendored Python stack on all hosts.
+    mkdir -p /tmp/py-requirements
+    source /tmp/venv/bin/activate
+        pip install --prefix /tmp/py-requirements -r ${requirements_txt}
+        while read -r host; do
+            rsync -rz /tmp/py-requirements/ "$host":/usr/local/greenplum-db-devel/ext/python/
+        done < /tmp/hostfile_all
+    deactivate
+}
+
 function setup_coverage() {
-    # Enables coverage.py on this machine. Note that this function modifies
-    # greenplum_path.sh, so callers need to source that file AFTER this is done.
-    local commit_sha
-    read -r commit_sha < ./gpdb_src/.git/HEAD
-    local coverage_path="$(pwd)/coverage/$commit_sha"
+    # Enables coverage.py on all hosts in the cluster. Note that this function
+    # modifies greenplum_path.sh, so callers need to source that file AFTER this
+    # is done.
+    local gpdb_src_dir="$1"
+    local commit_sha=$(head -1 "$gpdb_src_dir/.git/HEAD")
+    local coverage_path="/tmp/coverage/$commit_sha"
 
     # This file will be copied into GPDB's PYTHONPATH; it sets up the coverage
     # hook for all Python source files that are executed.
@@ -146,17 +157,18 @@ data_file = $coverage_path/coverage
 parallel = True
 COVEOF
 
-    # Now copy everything over to the installation.
-    cp /tmp/sitecustomize.py /usr/local/greenplum-db-devel/lib/python
-    cp /tmp/coveragerc /usr/local/greenplum-db-devel
-    mkdir -p $coverage_path
-    chown gpadmin:gpadmin $coverage_path
+    # Now copy everything over to the hosts.
+    while read -r host; do
+        scp /tmp/sitecustomize.py "$host":/usr/local/greenplum-db-devel/lib/python
+        scp /tmp/coveragerc "$host":/usr/local/greenplum-db-devel
+        ssh "$host" "mkdir -p $coverage_path" < /dev/null
 
-    # Enable coverage instrumentation after sourcing greenplum_path.
-    echo 'export COVERAGE_PROCESS_START=/usr/local/greenplum-db-devel/coveragerc' >> /usr/local/greenplum-db-devel/greenplum_path.sh
+        # Enable coverage instrumentation after sourcing greenplum_path.
+        ssh "$host" "echo 'export COVERAGE_PROCESS_START=/usr/local/greenplum-db-devel/coveragerc' >> /usr/local/greenplum-db-devel/greenplum_path.sh" < /dev/null
+    done < /tmp/hostfile_all
 }
 
-function prepare_coverage() {
+function tar_coverage() {
     # Call this function after running tests under the setup_coverage
     # environment. It performs any final needed manipulation of the coverage
     # data before it is published.

concourse/scripts/gpMgmt_check_gpdb.bash

@@ -42,17 +42,24 @@ function _main() {
     setup_gpadmin_user
     (make_cluster)
 
-    # pip-install the gpMgmt requirements file.
     install_python_hacks
-    install_python_requirements
-
-    # Set up test coverage.
-    setup_coverage
+    install_python_requirements_on_single_host ./gpdb_src/gpMgmt/requirements-dev.txt
 
     gen_env
+
+    # need to run setup_coverage as gpadmin due to scp and ssh commands
+    su gpadmin -c "
+        source ./gpdb_src/concourse/scripts/common.bash
+        # setup hostfile_all for demo_cluster tests
+        echo localhost > /tmp/hostfile_all
+        setup_coverage ./gpdb_src
+    "
+
     run_test
 
-    prepare_coverage "$TEST_NAME"
+    # collect coverage
+    cp -r /tmp/coverage/* ./coverage/
+    tar_coverage "$TEST_NAME"
 }
 
 _main "$@"

concourse/scripts/run_behave_test.sh

 #!/bin/bash
 set -ex
 
-BEHAVE_FLAGS=$@
-
-function setup_coverage() {
-    # Enables coverage.py on all hosts in the cluster. Note that this function
-    # modifies greenplum_path.sh, so callers need to source that file AFTER this
-    # is done.
-    local commit_sha
-    read -r commit_sha < /home/gpadmin/gpdb_src/.git/HEAD
-    local coverage_path="/tmp/coverage/$commit_sha"
-
-    # This file will be copied into GPDB's PYTHONPATH; it sets up the coverage
-    # hook for all Python source files that are executed.
-    cat > /tmp/sitecustomize.py <<SITEEOF
-import coverage
-coverage.process_startup()
-SITEEOF
-
-    # Set up coverage.py to handle analysis from multiple parallel processes.
-    cat > /tmp/coveragerc <<COVEOF
-[run]
-branch = True
-data_file = $coverage_path/coverage
-parallel = True
-COVEOF
-
-    # Now copy everything over to the hosts.
-    while read -r host; do
-        scp /tmp/sitecustomize.py "$host":/usr/local/greenplum-db-devel/lib/python
-        scp /tmp/coveragerc "$host":/usr/local/greenplum-db-devel
-        ssh "$host" "mkdir -p $coverage_path" < /dev/null
-
-        # Enable coverage instrumentation after sourcing greenplum_path.
-        ssh "$host" "echo 'export COVERAGE_PROCESS_START=/usr/local/greenplum-db-devel/coveragerc' >> /usr/local/greenplum-db-devel/greenplum_path.sh" < /dev/null
-    done < /tmp/hostfile_all
-}
+CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "${CWDIR}/common.bash"
 
-# virtualenv 16.0 and greater does not support python2.6, which is
-# used on centos6
-pip install --user virtualenv~=15.0
-export PATH=$PATH:~/.local/bin
-
-# create virtualenv before sourcing greenplum_path since greenplum_path
-# modifies PYTHONHOME and PYTHONPATH
-#
-# XXX Patch up the vendored Python's RPATH so we can successfully run
-# virtualenv. If we instead set LD_LIBRARY_PATH (as greenplum_path.sh does), the
-# system Python and the vendored Python will collide and virtualenv will fail.
-# This step requires patchelf.
-patchelf \
-    --set-rpath /usr/local/greenplum-db-devel/ext/python/lib \
-    /usr/local/greenplum-db-devel/ext/python/bin/python
-
-virtualenv \
-    --python /usr/local/greenplum-db-devel/ext/python/bin/python /tmp/venv
-
-# Install requirements into the vendored Python stack on all hosts.
-mkdir -p /tmp/py-requirements
-source /tmp/venv/bin/activate
-    pip install --prefix /tmp/py-requirements -r /home/gpadmin/gpdb_src/gpMgmt/requirements-dev.txt
-    while read -r host; do
-        rsync -rz /tmp/py-requirements/ "$host":/usr/local/greenplum-db-devel/ext/python/
-    done < /tmp/hostfile_all
-deactivate
-
-# Enable coverage.py on all hosts. (This modifies greenplum_path.sh and must
-# come before the source below.)
-setup_coverage
+BEHAVE_FLAGS=$@
 
 cat > ~/gpdb-env.sh << EOF
   source /usr/local/greenplum-db-devel/greenplum_path.sh

concourse/tasks/run_behave_on_ccp_cluster.yml

 platform: linux
 inputs:
+ - name: gpdb_src
  - name: ccp_src
  - name: cluster_env_files
 outputs:
@@ -10,40 +11,34 @@ run:
   - -c
   - |
     set -ex
+
     ccp_src/scripts/setup_ssh_to_cluster.sh
 
+    # TODO: ask CCP maintainers for a feature to do this for us
+    scp cluster_env_files/hostfile_all mdw:/tmp
+
     # Install patchelf. We need to SSH as root, hence the use of
     # cluster_env_files.
-    ssh -t ccp-$(cat cluster_env_files/terraform/name)-0 "bash -c '
-        # Our Python installation does not run standalone; it requires
-        # LD_LIBRARY_PATH which causes virtualenv to fail (because the system
-        # and vendored libpythons collide). We will try our best to install
-        # patchelf to fix this later, but it is not available on all platforms.
-        set -ex
-        if which yum > /dev/null; then
-            sudo yum install -y patchelf
-        else
-            set +x
-            echo \"ERROR: install_python_hacks() does not support the current platform and should be modified\"
-            exit 1
-        fi
+    ssh -t ccp-$(cat cluster_env_files/terraform/name)-0 "sudo bash -c '
+        source /home/gpadmin/gpdb_src/concourse/scripts/common.bash
+        install_python_hacks
     '"
 
-    # TODO: ask CCP maintainers for a feature to do this for us
-    scp cluster_env_files/hostfile_all mdw:/tmp
-    ssh -t mdw "$CUSTOM_ENV bash /home/gpadmin/gpdb_src/concourse/scripts/run_behave_test.sh \"$BEHAVE_FLAGS\""
+    ssh -t mdw "
+        source /home/gpadmin/gpdb_src/concourse/scripts/common.bash
+        install_python_requirements_on_multi_host /home/gpadmin/gpdb_src/gpMgmt/requirements-dev.txt
+
+        # Enable coverage.py on all hosts. (This modifies greenplum_path.sh and must
+        # come before the source below.)
+        setup_coverage /home/gpadmin/gpdb_src
 
-    # Collect coverage data.
+        $CUSTOM_ENV bash /home/gpadmin/gpdb_src/concourse/scripts/run_behave_test.sh \"$BEHAVE_FLAGS\"
+    "
+
+    # collect coverage
     while read -r host; do
         scp -r "$host:/tmp/coverage/*" ./coverage/
     done < cluster_env_files/hostfile_all
 
-    # Uniquify the coverage files a little bit with the $TEST_NAME.
-    pushd coverage/*
-        for f in *; do
-            mv "$f" "$TEST_NAME.$f"
-        done
-
-        # Compress coverage files and remove the originals
-        tar --remove-files -cf "ccp-$TEST_NAME.tar" *
-    popd
+    source ./gpdb_src/concourse/scripts/common.bash
+    tar_coverage "${TEST_NAME}_ccp"