behave: Add multinode tests for gppkg --clean

This commit introduces BEHAVE_FLAGS as a new parameter in concourse.
This will help us run specific tests within a specific scenario, it used
to be just all or nothing. Now we can separate multi-host testing vs
single host testing.

Add tests for gppkg --clean for multi-host
 * gppkg --clean should install to the segment host with no gppkg
 * gppkg --clean should remove on all segment hosts when gppkg does not exist in master

concourse/pipelines/pipeline.yml

@@ -662,7 +662,8 @@ jobs:
     file: gpdb_src/concourse/tasks/behave_gpdb.yml
     image: centos-gpdb-dev-6
     params:
-      BEHAVE_TAGS: gppkg
+      # Disable multinode tests for concourse.
+      BEHAVE_FLAGS: --tags=gppkg --tags=-gppkg_multinode_clean
 
 - name: MM_pt-rebuild
   plan:

concourse/scripts/behave_gpdb.bash

@@ -8,12 +8,18 @@ source "${CWDIR}/common.bash"
 function gen_env(){
   cat > /opt/run_test.sh <<-EOF
 
+		BEHAVE_TAGS="${BEHAVE_TAGS}"
+		BEHAVE_FLAGS="${BEHAVE_FLAGS}"
+
 		source /usr/local/greenplum-db-devel/greenplum_path.sh
 		cd "\${1}/gpdb_src/gpAux"
 		source gpdemo/gpdemo-env.sh
 		cd "\${1}/gpdb_src/gpMgmt/"
-		# Does this report failures?
-		make -f Makefile.behave behave tags=${BEHAVE_TAGS}
+		if [ ! -z "\${BEHAVE_TAGS}" ]; then
+		    make -f Makefile.behave behave tags=\${BEHAVE_TAGS}
+		else
+		    flags="\${BEHAVE_FLAGS}" make -f Makefile.behave behave
+		fi
 	EOF
 
 	chmod a+x /opt/run_test.sh
@@ -55,8 +61,8 @@ function gpcheck_setup() {
 
 function _main() {
 
-    if [ -z "${BEHAVE_TAGS}" ]; then
-        echo "FATAL: BEHAVE_TAGS is not set"
+    if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then
+        echo "FATAL: BEHAVE_TAGS or BEHAVE_FLAGS not set"
         exit 1
     fi
 

concourse/tasks/behave_gpdb.yml

@@ -7,6 +7,7 @@ inputs:
 outputs:
 params:
   BEHAVE_TAGS: ""
+  BEHAVE_FLAGS: ""
   BLDWRAP_POSTGRES_CONF_ADDONS: ""
   GPCHECK_SETUP: false
 run:

gpMgmt/test/behave/mgmt_utils/gppkg.feature

@@ -56,7 +56,7 @@ Feature: gppkg tests
         Then gppkg should return a return code of 0
         And gppkg should print "This is a sample message shown after successful installation" to stdout
         And gppkg should print "Completed local installation of sample" to stdout
-        And "sample" gppkg files exist on all segment hosts
+        And "sample" gppkg files exist on all hosts
 
     @gppkg_install_remove
     Scenario: gppkg --install should report failure because the package is already installed
@@ -73,7 +73,7 @@ Feature: gppkg tests
         And gppkg should print "Uninstalling package sample.gppkg" to stdout
         And gppkg should print "Completed local uninstallation of sample.gppkg" to stdout
         And gppkg should print "sample.gppkg successfully uninstalled" to stdout
-        And "sample" gppkg files does not exist on all segment hosts
+        And "sample" gppkg files does not exist on all hosts
 
     @gppkg_install_remove
     Scenario: gppkg --remove should report failure when the package is not installed
@@ -92,3 +92,23 @@ Feature: gppkg tests
         Then gppkg should return a return code of 0
         And gppkg should print "Starting gppkg with args: --query --all" to stdout
         And gppkg should print "sample" to stdout
+
+    @gppkg_multinode_clean
+    Scenario: gppkg --clean should install to the segment host with no gppkg
+        Given the database is running
+        When the user runs "gppkg --install test/behave/mgmt_utils/steps/data/sample.gppkg"
+        And gppkg "sample" is removed from a segment host
+        And the user runs "gppkg --clean"
+        Then gppkg should return a return code of 0
+        And gppkg should print "The following packages will be installed on .*: sample.gppkg" to stdout
+        And "sample" gppkg files exist on all hosts
+
+    @gppkg_multinode_clean
+    Scenario: gppkg --clean should remove on all segment hosts when gppkg does not exist in master
+        Given the database is running
+        When the user runs "gppkg --install test/behave/mgmt_utils/steps/data/sample.gppkg"
+        And gppkg "sample" is removed from master host
+        And the user runs "gppkg --clean"
+        Then gppkg should return a return code of 0
+        And gppkg should print "The following packages will be uninstalled on .*: sample.gppkg" to stdout
+        And "sample" gppkg files does not exist on all hosts

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -5114,14 +5114,14 @@ def impl(context):
     if not context.exception:
         raise Exception("Directory for date %s still exists" % context.full_backup_timestamp[0:8])
 
-@then('"{gppkg_name}" gppkg files exist on all segment hosts')
+@then('"{gppkg_name}" gppkg files exist on all hosts')
 def impl(context, gppkg_name):
     remote_gphome = os.environ.get('GPHOME')
     gparray = GpArray.initFromCatalog(dbconn.DbURL())
 
     hostlist = get_all_hostnames_as_list(context, 'template1')
 
-    #We can assume the GPDB is installed at the same location for all hosts
+    # We can assume the GPDB is installed at the same location for all hosts
     rpm_command_list_all = 'rpm -qa --dbpath %s/share/packages/database' % remote_gphome
 
     for hostname in set(hostlist):
@@ -5134,14 +5134,12 @@ def impl(context, gppkg_name):
         if not gppkg_name in cmd.get_stdout():
             raise Exception( '"%s" gppkg is not installed on host: %s. \nInstalled packages: %s' % (gppkg_name, hostname, cmd.get_stdout()))
 
-@then('"{gppkg_name}" gppkg files does not exist on all segment hosts')
+@then('"{gppkg_name}" gppkg files does not exist on all hosts')
 def impl(context, gppkg_name):
     remote_gphome = os.environ.get('GPHOME')
-    gparray = GpArray.initFromCatalog(dbconn.DbURL())
-
     hostlist = get_all_hostnames_as_list(context, 'template1')
 
-    #We can assume the GPDB is installed at the same location for all hosts
+    # We can assume the GPDB is installed at the same location for all hosts
     rpm_command_list_all = 'rpm -qa --dbpath %s/share/packages/database' % remote_gphome
 
     for hostname in set(hostlist):
@@ -5153,3 +5151,54 @@ def impl(context, gppkg_name):
 
         if gppkg_name in cmd.get_stdout():
             raise Exception( '"%s" gppkg is installed on host: %s. \nInstalled packages: %s' % (gppkg_name, hostname, cmd.get_stdout()))
+
+def _remove_gppkg_from_host(context, gppkg_name, is_master_host):
+    remote_gphome = os.environ.get('GPHOME')
+
+    if is_master_host:
+        hostname = get_master_hostname()[0][0] # returns a list of list
+    else:
+        hostlist = get_segment_hostlist()
+        if not hostlist:
+            raise Exception("Current GPDB setup is not a multi-host cluster.")
+
+        # Let's just pick whatever is the first host in the list, it shouldn't
+        # matter which one we remove from
+        hostname = hostlist[0]
+
+    rpm_command_list_all = 'rpm -qa --dbpath %s/share/packages/database' % remote_gphome
+    cmd = Command(name='get all rpm from the host',
+                  cmdStr=rpm_command_list_all,
+                  ctxt=REMOTE,
+                  remoteHost=hostname)
+    cmd.run(validateAfter=True)
+    installed_gppkgs = cmd.get_stdout_lines()
+    if not installed_gppkgs:
+        raise Exception("Found no packages installed")
+
+    full_gppkg_name = next((gppkg for gppkg in installed_gppkgs if gppkg_name in gppkg), None)
+    if not full_gppkg_name:
+        raise Exception("Found no matches for gppkg '%s'\n"
+                        "gppkgs installed:\n%s" % (gppkg_name, installed_gppkgs))
+
+    rpm_remove_command = 'rpm -e %s --dbpath %s/share/packages/database' % (full_gppkg_name, remote_gphome)
+    cmd = Command(name='Cleanly remove from the remove host',
+                  cmdStr=rpm_remove_command,
+                  ctxt=REMOTE,
+                  remoteHost=hostname)
+    cmd.run(validateAfter=True)
+
+    remove_archive_gppgk = 'rm -f %s/share/packages/archive/%s.gppkg' % (remote_gphome, gppkg_name)
+    cmd = Command(name='Remove archive gppkg',
+                  cmdStr=remove_archive_gppgk,
+                  ctxt=REMOTE,
+                  remoteHost=hostname)
+    cmd.run(validateAfter=True)
+
+@when('gppkg "{gppkg_name}" is removed from a segment host')
+def impl(context, gppkg_name):
+    _remove_gppkg_from_host(context, gppkg_name, is_master_host=False)
+
+@when('gppkg "{gppkg_name}" is removed from master host')
+def impl(context, gppkg_name):
+    _remove_gppkg_from_host(context, gppkg_name, is_master_host=True)