gpmovemirrors: add multi-host tablespace test

Create the tablespace directory on all segments to support
running multi-host tablespace tests.
Co-authored-by: NJacob Champion <pchampion@pivotal.io>
(cherry picked from commit 83f48aac)

gpMgmt/test/behave/mgmt_utils/gpmovemirrors.feature

@@ -97,4 +97,20 @@ Feature: Tests for gpmovemirrors
         And the segments are synchronized
         And verify that mirror segments are in "group" configuration
         And verify that mirrors are recognized after a restart
-        And the user runs "gpstop -aqM fast"
+
+    @concourse_cluster
+    Scenario: tablespaces work on a multi-host environment
+        Given verify that mirror segments are in "group" configuration
+          And a tablespace is created with data
+          And a sample gpmovemirrors input file is created in "spread" configuration
+         When the user runs "gpmovemirrors --input=/tmp/gpmovemirrors_input_spread"
+         Then gpmovemirrors should return a return code of 0
+          And verify the database has mirrors
+          And all the segments are running
+          And the segments are synchronized
+          And verify that mirrors are recognized after a restart
+          And the tablespace is valid
+
+         When user stops all primary processes
+          And user can start transactions
+         Then the tablespace is valid

gpMgmt/test/behave/mgmt_utils/steps/tablespace_mgmt_utils.py

+import pipes
 import shutil
 import tempfile
 
@@ -5,7 +6,8 @@ from behave import given, when, then
 from pygresql import pg
 
 from gppylib.db import dbconn
-
+from gppylib.gparray import GpArray
+from test.behave_utils.utils import run_cmd
 
 class Tablespace:
     def __init__(self, name):
@@ -15,6 +17,10 @@ class Tablespace:
         self.table_counter = 0
         self.initial_data = None
 
+        gparray = GpArray.initFromCatalog(dbconn.DbURL())
+        for host in gparray.getHostList():
+            run_cmd('ssh %s mkdir -p %s' % (pipes.quote(host), pipes.quote(self.path)))
+
         with dbconn.connect(dbconn.DbURL()) as conn:
             db = pg.DB(conn)
             db.query("CREATE TABLESPACE %s LOCATION '%s'" % (self.name, self.path))
@@ -40,7 +46,9 @@ class Tablespace:
             # before removing them below.
             _checkpoint_and_wait_for_replication_replay(db)
 
-        shutil.rmtree(self.path)
+        gparray = GpArray.initFromCatalog(dbconn.DbURL())
+        for host in gparray.getHostList():
+            run_cmd('ssh %s rm -rf %s' % (pipes.quote(host), pipes.quote(self.path)))
 
     def verify(self, hostname=None, port=0):
         """