From e6b40a719afea99bfcf2c282e10654054c10082f Mon Sep 17 00:00:00 2001 From: Jacob Champion Date: Fri, 5 Apr 2019 16:36:29 -0700 Subject: [PATCH] gppkg: remove special Concourse task for GPHOME copying concourse/scripts/gppkg_behave.sh, which installs an identical copy of GPHOME into a new location, existed for exactly one test. Modify that test to perform its own copy-install. --- concourse/pipelines/gpdb_master-generated.yml | 13 +----- concourse/pipelines/templates/gpdb-tpl.yml | 12 +----- concourse/scripts/gppkg_behave.sh | 40 ------------------ concourse/tasks/gppkg_behave.yml | 28 ------------- gpMgmt/test/behave/mgmt_utils/gppkg.feature | 1 + .../behave/mgmt_utils/steps/mgmt_utils.py | 41 +++++++++++++++++++ 6 files changed, 44 insertions(+), 91 deletions(-) delete mode 100755 concourse/scripts/gppkg_behave.sh delete mode 100644 concourse/tasks/gppkg_behave.yml diff --git a/concourse/pipelines/gpdb_master-generated.yml b/concourse/pipelines/gpdb_master-generated.yml index 2d5804fb00..28a442b6a0 100644 --- a/concourse/pipelines/gpdb_master-generated.yml +++ b/concourse/pipelines/gpdb_master-generated.yml @@ -12,14 +12,12 @@ ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -t prod). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2019-04-22 11:38:41.346707 +## Generated by gen_pipeline.py at: 2019-04-22 11:39:53.092437 ## Template file: gpdb-tpl.yml ## OS Types: ['centos6', 'centos7', 'sles', 'win', 'ubuntu16'] ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions', 'Gpperfmon'] ## ====================================================================== - - ## ====================================================================== ## __ _ _ __ ___ _ _ _ __ ___ ## / _` | '__/ _ \| | | | '_ \/ __| @@ -1441,7 +1439,6 @@ jobs: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - - task: gpmovemirrors_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave.yml image: ccp-image @@ -1512,11 +1509,6 @@ jobs: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - - - task: setup_gppkg_second_install - file: gpdb_src/concourse/tasks/gppkg_behave.yml - params: - SECOND_BINARY_INSTALL_LOCATION: /tmp/gppkg_migrate - task: gppkg_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave.yml image: ccp-image @@ -1692,7 +1684,6 @@ jobs: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - - task: gpcheckcat_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave.yml image: ccp-image @@ -1742,7 +1733,6 @@ jobs: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - - task: gprecoverseg_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave.yml image: ccp-image @@ -1792,7 +1782,6 @@ jobs: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - - task: gpaddmirrors_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave.yml image: ccp-image diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index 6a439f8b78..2c241ff6c6 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -1,11 +1,3 @@ - -{% set gppkg_additional_task = " - - task: setup_gppkg_second_install - file: gpdb_src/concourse/tasks/gppkg_behave.yml - params: - SECOND_BINARY_INSTALL_LOCATION: /tmp/gppkg_migrate" -%} - {% set CLI_BEHAVE_TESTS = [ {'name': 'gpmovemirrors', 'use_concourse_cluster': true, @@ -13,8 +5,7 @@ {'name': 'gpcheck', 'use_concourse_cluster': false}, {'name': 'gppkg', - 'use_concourse_cluster': true, - 'additional_tasks': gppkg_additional_task}, + 'use_concourse_cluster': true}, {'name': 'analyzedb', 'use_concourse_cluster': false}, {'name': 'gpinitsystem', @@ -1744,7 +1735,6 @@ jobs: <<: *ccp_gen_cluster_default_params - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - [[ test.additional_tasks ]] - task: [[ test.name ]]_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave.yml image: ccp-image diff --git a/concourse/scripts/gppkg_behave.sh b/concourse/scripts/gppkg_behave.sh deleted file mode 100755 index 3e0357a2f1..0000000000 --- a/concourse/scripts/gppkg_behave.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# set -euo pipefail -# IFS=$'\n\t' -# In the future, this script will be provided by the team that owns the test, and they will decide if they should enforce bash strict mode - -set -x - -SECOND_BINARY_INSTALL_LOCATION=$SECOND_BINARY_INSTALL_LOCATION - -extract_gpdb_tarball() { - local node_hostname=$1 - local tarball_dir=$2 - # Commonly the incoming binary will be called bin_gpdb.tar.gz. Because many other teams/pipelines tend to use - # that naming convention we are not, deliberately. Once the file crosses into our domain, we will not use - # the conventional name. This should make clear that we will install any valid binary, not just those that follow - # the naming convention. - gpssh -f cluster_env_files/hostfile_all "bash -c \"\ - mkdir -p $SECOND_BINARY_INSTALL_LOCATION; \ - tar -xf /tmp/gpdb_binary.tar.gz -C $SECOND_BINARY_INSTALL_LOCATION; \ - chown -R gpadmin:gpadmin $SECOND_BINARY_INSTALL_LOCATION; \ - \"" -} - -update_gphome() { - local node_hostname=$1 - gpssh -f cluster_env_files/hostfile_all " bash -c \"\ - cd $SECOND_BINARY_INSTALL_LOCATION; \ - sed -i "s,^GPHOME=.*$,GPHOME=$SECOND_BINARY_INSTALL_LOCATION," greenplum_path.sh; \ - \"" -} - -## Main -source /home/gpadmin/gpinitsystem_config; -source /usr/local/greenplum-db-devel/greenplum_path.sh; -export PGPORT=$MASTER_PORT; -export MASTER_DATA_DIRECTORY=$MASTER_DIRECTORY/$SEG_PREFIX-1; - -extract_gpdb_tarball - -update_gphome diff --git a/concourse/tasks/gppkg_behave.yml b/concourse/tasks/gppkg_behave.yml deleted file mode 100644 index 51a6a1b1b1..0000000000 --- a/concourse/tasks/gppkg_behave.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: pivotaldata/ccp - tag: "7" - -inputs: -- name: ccp_src -- name: cluster_env_files -- name: gpdb_src - -params: - SECOND_BINARY_INSTALL_LOCATION: - -run: - path: bash - args: - - -c - - | - set -ex - ccp_src/scripts/setup_ssh_to_cluster.sh - ssh mdw "mkdir -p /home/gpadmin/cluster_env_files" - scp cluster_env_files/hostfile_all mdw:/home/gpadmin/cluster_env_files/hostfile_all - ssh -t mdw "export SECOND_BINARY_INSTALL_LOCATION=$SECOND_BINARY_INSTALL_LOCATION && bash /home/gpadmin/gpdb_src/concourse/scripts/gppkg_behave.sh" - diff --git a/gpMgmt/test/behave/mgmt_utils/gppkg.feature b/gpMgmt/test/behave/mgmt_utils/gppkg.feature index 60e52c96af..c9588da943 100644 --- a/gpMgmt/test/behave/mgmt_utils/gppkg.feature +++ b/gpMgmt/test/behave/mgmt_utils/gppkg.feature @@ -120,6 +120,7 @@ Feature: gppkg tests Scenario: gppkg --migrate copies all packages from master to all segment hosts Given the database is running And the user runs "gppkg -r sample" + And a gphome copy is created at /tmp/gppkg_migrate on all hosts When a user runs "MASTER_DATA_DIRECTORY=$MASTER_DATA_DIRECTORY gppkg -r sample" with gphome "/tmp/gppkg_migrate" And "sample" gppkg files do not exist on any hosts When a user runs "MASTER_DATA_DIRECTORY=$MASTER_DATA_DIRECTORY gppkg --install $(pwd)/test/behave/mgmt_utils/steps/data/sample.gppkg" with gphome "/tmp/gppkg_migrate" diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index d991a2e37d..40c8fe8f2f 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -1948,6 +1948,47 @@ def impl(context, gppkg_name): def impl(context, gppkg_name): _remove_gppkg_from_host(context, gppkg_name, is_master_host=True) + +@given('a gphome copy is created at {location} on all hosts') +def impl(context, location): + """ + Copies the contents of GPHOME from the local machine into a different + directory location for all hosts in the cluster. + """ + gphome = os.environ["GPHOME"] + greenplum_path = path.join(gphome, 'greenplum_path.sh') + + # First replace the GPHOME envvar in greenplum_path.sh. + subprocess.check_call([ + 'sed', + '-i.bak', # we use this backup later + '-e', r's|^GPHOME=.*$|GPHOME={}|'.format(location), + greenplum_path, + ]) + + try: + # Now copy all the files over. + hosts = set(get_all_hostnames_as_list(context, 'template1')) + + host_opts = [] + for host in hosts: + host_opts.extend(['-h', host]) + + subprocess.check_call([ + 'gpscp', + '-rv', + ] + host_opts + [ + os.getenv('GPHOME'), + '=:{}'.format(location), + ]) + + finally: + # Put greenplum_path.sh back the way it was. + subprocess.check_call([ + 'mv', '{}.bak'.format(greenplum_path), greenplum_path + ]) + + @then('gpAdminLogs directory has no "{expected_file}" files') def impl(context, expected_file): log_dir = _get_gpAdminLogs_directory() -- GitLab