gppkg: remove special Concourse task for GPHOME copying

concourse/scripts/gppkg_behave.sh, which installs an identical copy of
GPHOME into a new location, existed for exactly one test. Modify that
test to perform its own copy-install.

concourse/pipelines/gpdb_master-generated.yml

@@ -12,14 +12,12 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2019-04-22 11:38:41.346707
+## Generated by gen_pipeline.py at: 2019-04-22 11:39:53.092437
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'sles', 'win', 'ubuntu16']
 ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions', 'Gpperfmon']
 ## ======================================================================
 
-
-
 ## ======================================================================
 ##   __ _ _ __ ___  _   _ _ __  ___
 ##  / _` | '__/ _ \| | | | '_ \/ __|
@@ -1441,7 +1439,6 @@ jobs:
             <<: *ccp_gen_cluster_default_params
         - task: gpinitsystem
           file: ccp_src/ci/tasks/gpinitsystem.yml
-        
         - task: gpmovemirrors_concourse_cluster_tests
           file: gpdb_src/concourse/tasks/run_behave.yml
           image: ccp-image
@@ -1512,11 +1509,6 @@ jobs:
             <<: *ccp_gen_cluster_default_params
         - task: gpinitsystem
           file: ccp_src/ci/tasks/gpinitsystem.yml
-        
-        - task: setup_gppkg_second_install
-          file: gpdb_src/concourse/tasks/gppkg_behave.yml
-          params:
-            SECOND_BINARY_INSTALL_LOCATION: /tmp/gppkg_migrate
         - task: gppkg_concourse_cluster_tests
           file: gpdb_src/concourse/tasks/run_behave.yml
           image: ccp-image
@@ -1692,7 +1684,6 @@ jobs:
             <<: *ccp_gen_cluster_default_params
         - task: gpinitsystem
           file: ccp_src/ci/tasks/gpinitsystem.yml
-        
         - task: gpcheckcat_concourse_cluster_tests
           file: gpdb_src/concourse/tasks/run_behave.yml
           image: ccp-image
@@ -1742,7 +1733,6 @@ jobs:
             <<: *ccp_gen_cluster_default_params
         - task: gpinitsystem
           file: ccp_src/ci/tasks/gpinitsystem.yml
-        
         - task: gprecoverseg_concourse_cluster_tests
           file: gpdb_src/concourse/tasks/run_behave.yml
           image: ccp-image
@@ -1792,7 +1782,6 @@ jobs:
             <<: *ccp_gen_cluster_default_params
         - task: gpinitsystem
           file: ccp_src/ci/tasks/gpinitsystem.yml
-        
         - task: gpaddmirrors_concourse_cluster_tests
           file: gpdb_src/concourse/tasks/run_behave.yml
           image: ccp-image

concourse/pipelines/templates/gpdb-tpl.yml

-
-{% set gppkg_additional_task = "
-        - task: setup_gppkg_second_install
-          file: gpdb_src/concourse/tasks/gppkg_behave.yml
-          params:
-            SECOND_BINARY_INSTALL_LOCATION: /tmp/gppkg_migrate"
-%}
-
 {% set CLI_BEHAVE_TESTS = [
                       {'name': 'gpmovemirrors',
                        'use_concourse_cluster': true,
@@ -13,8 +5,7 @@
                       {'name': 'gpcheck',
                        'use_concourse_cluster': false},
                       {'name': 'gppkg',
-                       'use_concourse_cluster': true,
-                       'additional_tasks': gppkg_additional_task},
+                       'use_concourse_cluster': true},
                       {'name': 'analyzedb',
                        'use_concourse_cluster': false},
                       {'name': 'gpinitsystem',
@@ -1744,7 +1735,6 @@ jobs:
             <<: *ccp_gen_cluster_default_params
         - task: gpinitsystem
           file: ccp_src/ci/tasks/gpinitsystem.yml
-        [[ test.additional_tasks ]]
         - task: [[ test.name ]]_concourse_cluster_tests
           file: gpdb_src/concourse/tasks/run_behave.yml
           image: ccp-image

concourse/scripts/gppkg_behave.sh

-#!/bin/bash
-# set -euo pipefail
-# IFS=$'\n\t'
-# In the future, this script will be provided by the team that owns the test, and they will decide if they should enforce bash strict mode
-
-set -x
-
-SECOND_BINARY_INSTALL_LOCATION=$SECOND_BINARY_INSTALL_LOCATION
-
-extract_gpdb_tarball() {
-    local node_hostname=$1
-    local tarball_dir=$2
-    # Commonly the incoming binary will be called bin_gpdb.tar.gz. Because many other teams/pipelines tend to use
-    # that naming convention we are not, deliberately. Once the file crosses into our domain, we will not use
-    # the conventional name.  This should make clear that we will install any valid binary, not just those that follow
-    # the naming convention.
-    gpssh -f cluster_env_files/hostfile_all "bash -c \"\
-      mkdir -p $SECOND_BINARY_INSTALL_LOCATION; \
-      tar -xf /tmp/gpdb_binary.tar.gz -C $SECOND_BINARY_INSTALL_LOCATION; \
-      chown -R gpadmin:gpadmin $SECOND_BINARY_INSTALL_LOCATION; \
-    \""
-}
-
-update_gphome() {
-    local node_hostname=$1
-    gpssh -f cluster_env_files/hostfile_all " bash -c \"\
-      cd $SECOND_BINARY_INSTALL_LOCATION; \
-      sed -i "s,^GPHOME=.*$,GPHOME=$SECOND_BINARY_INSTALL_LOCATION," greenplum_path.sh; \
-    \""
-}
-
-## Main
-source /home/gpadmin/gpinitsystem_config;
-source /usr/local/greenplum-db-devel/greenplum_path.sh;
-export PGPORT=$MASTER_PORT;
-export MASTER_DATA_DIRECTORY=$MASTER_DIRECTORY/$SEG_PREFIX-1;
-
-extract_gpdb_tarball
-
-update_gphome

concourse/tasks/gppkg_behave.yml

----
-platform: linux
-
-image_resource:
-  type: docker-image
-  source:
-    repository: pivotaldata/ccp
-    tag: "7"
-
-inputs:
-- name: ccp_src
-- name: cluster_env_files
-- name: gpdb_src
-
-params:
-  SECOND_BINARY_INSTALL_LOCATION:
-
-run:
-  path: bash
-  args:
-  - -c
-  - |
-    set -ex
-    ccp_src/scripts/setup_ssh_to_cluster.sh
-    ssh mdw "mkdir -p /home/gpadmin/cluster_env_files"
-    scp cluster_env_files/hostfile_all mdw:/home/gpadmin/cluster_env_files/hostfile_all
-    ssh -t mdw "export SECOND_BINARY_INSTALL_LOCATION=$SECOND_BINARY_INSTALL_LOCATION && bash /home/gpadmin/gpdb_src/concourse/scripts/gppkg_behave.sh"
-

gpMgmt/test/behave/mgmt_utils/gppkg.feature

@@ -120,6 +120,7 @@ Feature: gppkg tests
     Scenario: gppkg --migrate copies all packages from master to all segment hosts
         Given the database is running
         And the user runs "gppkg -r sample"
+        And a gphome copy is created at /tmp/gppkg_migrate on all hosts
         When a user runs "MASTER_DATA_DIRECTORY=$MASTER_DATA_DIRECTORY gppkg -r sample" with gphome "/tmp/gppkg_migrate"
         And "sample" gppkg files do not exist on any hosts
         When a user runs "MASTER_DATA_DIRECTORY=$MASTER_DATA_DIRECTORY gppkg --install $(pwd)/test/behave/mgmt_utils/steps/data/sample.gppkg" with gphome "/tmp/gppkg_migrate"

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -1948,6 +1948,47 @@ def impl(context, gppkg_name):
 def impl(context, gppkg_name):
     _remove_gppkg_from_host(context, gppkg_name, is_master_host=True)
 
+
+@given('a gphome copy is created at {location} on all hosts')
+def impl(context, location):
+    """
+    Copies the contents of GPHOME from the local machine into a different
+    directory location for all hosts in the cluster.
+    """
+    gphome = os.environ["GPHOME"]
+    greenplum_path = path.join(gphome, 'greenplum_path.sh')
+
+    # First replace the GPHOME envvar in greenplum_path.sh.
+    subprocess.check_call([
+        'sed',
+        '-i.bak', # we use this backup later
+        '-e', r's|^GPHOME=.*$|GPHOME={}|'.format(location),
+        greenplum_path,
+    ])
+
+    try:
+        # Now copy all the files over.
+        hosts = set(get_all_hostnames_as_list(context, 'template1'))
+
+        host_opts = []
+        for host in hosts:
+            host_opts.extend(['-h', host])
+
+        subprocess.check_call([
+            'gpscp',
+            '-rv',
+            ] + host_opts + [
+            os.getenv('GPHOME'),
+            '=:{}'.format(location),
+        ])
+
+    finally:
+        # Put greenplum_path.sh back the way it was.
+        subprocess.check_call([
+            'mv', '{}.bak'.format(greenplum_path), greenplum_path
+        ])
+
+
 @then('gpAdminLogs directory has no "{expected_file}" files')
 def impl(context, expected_file):
     log_dir = _get_gpAdminLogs_directory()