From 5bc545104ca94d872d4370070aeb812290098581 Mon Sep 17 00:00:00 2001 From: Marbin Tan Date: Tue, 11 Jul 2017 14:25:08 -0700 Subject: [PATCH] behave: Add multinode tests for gppkg --clean This commit introduces BEHAVE_FLAGS as a new parameter in concourse. This will help us run specific tests within a specific scenario, it used to be just all or nothing. Now we can separate multi-host testing vs single host testing. Add tests for gppkg --clean for multi-host * gppkg --clean should install to the segment host with no gppkg * gppkg --clean should remove on all segment hosts when gppkg does not exist in master --- concourse/pipelines/pipeline.yml | 3 +- concourse/scripts/behave_gpdb.bash | 14 +++-- concourse/tasks/behave_gpdb.yml | 1 + gpMgmt/test/behave/mgmt_utils/gppkg.feature | 24 +++++++- .../behave/mgmt_utils/steps/mgmt_utils.py | 61 +++++++++++++++++-- 5 files changed, 90 insertions(+), 13 deletions(-) diff --git a/concourse/pipelines/pipeline.yml b/concourse/pipelines/pipeline.yml index 1655a9fda0..bcd79eb6d6 100644 --- a/concourse/pipelines/pipeline.yml +++ b/concourse/pipelines/pipeline.yml @@ -662,7 +662,8 @@ jobs: file: gpdb_src/concourse/tasks/behave_gpdb.yml image: centos-gpdb-dev-6 params: - BEHAVE_TAGS: gppkg + # Disable multinode tests for concourse. + BEHAVE_FLAGS: --tags=gppkg --tags=-gppkg_multinode_clean - name: MM_pt-rebuild plan: diff --git a/concourse/scripts/behave_gpdb.bash b/concourse/scripts/behave_gpdb.bash index 17c2d53585..1a74f7f51e 100755 --- a/concourse/scripts/behave_gpdb.bash +++ b/concourse/scripts/behave_gpdb.bash @@ -8,12 +8,18 @@ source "${CWDIR}/common.bash" function gen_env(){ cat > /opt/run_test.sh <<-EOF + BEHAVE_TAGS="${BEHAVE_TAGS}" + BEHAVE_FLAGS="${BEHAVE_FLAGS}" + source /usr/local/greenplum-db-devel/greenplum_path.sh cd "\${1}/gpdb_src/gpAux" source gpdemo/gpdemo-env.sh cd "\${1}/gpdb_src/gpMgmt/" - # Does this report failures? - make -f Makefile.behave behave tags=${BEHAVE_TAGS} + if [ ! -z "\${BEHAVE_TAGS}" ]; then + make -f Makefile.behave behave tags=\${BEHAVE_TAGS} + else + flags="\${BEHAVE_FLAGS}" make -f Makefile.behave behave + fi EOF chmod a+x /opt/run_test.sh @@ -55,8 +61,8 @@ function gpcheck_setup() { function _main() { - if [ -z "${BEHAVE_TAGS}" ]; then - echo "FATAL: BEHAVE_TAGS is not set" + if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then + echo "FATAL: BEHAVE_TAGS or BEHAVE_FLAGS not set" exit 1 fi diff --git a/concourse/tasks/behave_gpdb.yml b/concourse/tasks/behave_gpdb.yml index 0f9eae946f..a4a2b8e18a 100644 --- a/concourse/tasks/behave_gpdb.yml +++ b/concourse/tasks/behave_gpdb.yml @@ -7,6 +7,7 @@ inputs: outputs: params: BEHAVE_TAGS: "" + BEHAVE_FLAGS: "" BLDWRAP_POSTGRES_CONF_ADDONS: "" GPCHECK_SETUP: false run: diff --git a/gpMgmt/test/behave/mgmt_utils/gppkg.feature b/gpMgmt/test/behave/mgmt_utils/gppkg.feature index af2440bd2a..449056205d 100644 --- a/gpMgmt/test/behave/mgmt_utils/gppkg.feature +++ b/gpMgmt/test/behave/mgmt_utils/gppkg.feature @@ -56,7 +56,7 @@ Feature: gppkg tests Then gppkg should return a return code of 0 And gppkg should print "This is a sample message shown after successful installation" to stdout And gppkg should print "Completed local installation of sample" to stdout - And "sample" gppkg files exist on all segment hosts + And "sample" gppkg files exist on all hosts @gppkg_install_remove Scenario: gppkg --install should report failure because the package is already installed @@ -73,7 +73,7 @@ Feature: gppkg tests And gppkg should print "Uninstalling package sample.gppkg" to stdout And gppkg should print "Completed local uninstallation of sample.gppkg" to stdout And gppkg should print "sample.gppkg successfully uninstalled" to stdout - And "sample" gppkg files does not exist on all segment hosts + And "sample" gppkg files does not exist on all hosts @gppkg_install_remove Scenario: gppkg --remove should report failure when the package is not installed @@ -92,3 +92,23 @@ Feature: gppkg tests Then gppkg should return a return code of 0 And gppkg should print "Starting gppkg with args: --query --all" to stdout And gppkg should print "sample" to stdout + + @gppkg_multinode_clean + Scenario: gppkg --clean should install to the segment host with no gppkg + Given the database is running + When the user runs "gppkg --install test/behave/mgmt_utils/steps/data/sample.gppkg" + And gppkg "sample" is removed from a segment host + And the user runs "gppkg --clean" + Then gppkg should return a return code of 0 + And gppkg should print "The following packages will be installed on .*: sample.gppkg" to stdout + And "sample" gppkg files exist on all hosts + + @gppkg_multinode_clean + Scenario: gppkg --clean should remove on all segment hosts when gppkg does not exist in master + Given the database is running + When the user runs "gppkg --install test/behave/mgmt_utils/steps/data/sample.gppkg" + And gppkg "sample" is removed from master host + And the user runs "gppkg --clean" + Then gppkg should return a return code of 0 + And gppkg should print "The following packages will be uninstalled on .*: sample.gppkg" to stdout + And "sample" gppkg files does not exist on all hosts diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 93dbf52885..eaf327fb5c 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -5114,14 +5114,14 @@ def impl(context): if not context.exception: raise Exception("Directory for date %s still exists" % context.full_backup_timestamp[0:8]) -@then('"{gppkg_name}" gppkg files exist on all segment hosts') +@then('"{gppkg_name}" gppkg files exist on all hosts') def impl(context, gppkg_name): remote_gphome = os.environ.get('GPHOME') gparray = GpArray.initFromCatalog(dbconn.DbURL()) hostlist = get_all_hostnames_as_list(context, 'template1') - #We can assume the GPDB is installed at the same location for all hosts + # We can assume the GPDB is installed at the same location for all hosts rpm_command_list_all = 'rpm -qa --dbpath %s/share/packages/database' % remote_gphome for hostname in set(hostlist): @@ -5134,14 +5134,12 @@ def impl(context, gppkg_name): if not gppkg_name in cmd.get_stdout(): raise Exception( '"%s" gppkg is not installed on host: %s. \nInstalled packages: %s' % (gppkg_name, hostname, cmd.get_stdout())) -@then('"{gppkg_name}" gppkg files does not exist on all segment hosts') +@then('"{gppkg_name}" gppkg files does not exist on all hosts') def impl(context, gppkg_name): remote_gphome = os.environ.get('GPHOME') - gparray = GpArray.initFromCatalog(dbconn.DbURL()) - hostlist = get_all_hostnames_as_list(context, 'template1') - #We can assume the GPDB is installed at the same location for all hosts + # We can assume the GPDB is installed at the same location for all hosts rpm_command_list_all = 'rpm -qa --dbpath %s/share/packages/database' % remote_gphome for hostname in set(hostlist): @@ -5153,3 +5151,54 @@ def impl(context, gppkg_name): if gppkg_name in cmd.get_stdout(): raise Exception( '"%s" gppkg is installed on host: %s. \nInstalled packages: %s' % (gppkg_name, hostname, cmd.get_stdout())) + +def _remove_gppkg_from_host(context, gppkg_name, is_master_host): + remote_gphome = os.environ.get('GPHOME') + + if is_master_host: + hostname = get_master_hostname()[0][0] # returns a list of list + else: + hostlist = get_segment_hostlist() + if not hostlist: + raise Exception("Current GPDB setup is not a multi-host cluster.") + + # Let's just pick whatever is the first host in the list, it shouldn't + # matter which one we remove from + hostname = hostlist[0] + + rpm_command_list_all = 'rpm -qa --dbpath %s/share/packages/database' % remote_gphome + cmd = Command(name='get all rpm from the host', + cmdStr=rpm_command_list_all, + ctxt=REMOTE, + remoteHost=hostname) + cmd.run(validateAfter=True) + installed_gppkgs = cmd.get_stdout_lines() + if not installed_gppkgs: + raise Exception("Found no packages installed") + + full_gppkg_name = next((gppkg for gppkg in installed_gppkgs if gppkg_name in gppkg), None) + if not full_gppkg_name: + raise Exception("Found no matches for gppkg '%s'\n" + "gppkgs installed:\n%s" % (gppkg_name, installed_gppkgs)) + + rpm_remove_command = 'rpm -e %s --dbpath %s/share/packages/database' % (full_gppkg_name, remote_gphome) + cmd = Command(name='Cleanly remove from the remove host', + cmdStr=rpm_remove_command, + ctxt=REMOTE, + remoteHost=hostname) + cmd.run(validateAfter=True) + + remove_archive_gppgk = 'rm -f %s/share/packages/archive/%s.gppkg' % (remote_gphome, gppkg_name) + cmd = Command(name='Remove archive gppkg', + cmdStr=remove_archive_gppgk, + ctxt=REMOTE, + remoteHost=hostname) + cmd.run(validateAfter=True) + +@when('gppkg "{gppkg_name}" is removed from a segment host') +def impl(context, gppkg_name): + _remove_gppkg_from_host(context, gppkg_name, is_master_host=False) + +@when('gppkg "{gppkg_name}" is removed from master host') +def impl(context, gppkg_name): + _remove_gppkg_from_host(context, gppkg_name, is_master_host=True) -- GitLab