Add gprecoverseg -s to show progress sequentially

When -s is present, show pg_basebackup progress sequentially instead
of inplace. Useful when writing to a file, or if a tty does not support
escape sequences. Defaults to showing the progress inplace.

gpMgmt/bin/gppylib/mainUtils.py

@@ -307,7 +307,8 @@ def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
 def addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption, includeUsageOption=False):
     """
     Add the standard options for help and logging
-    to the specified parser object.
+    to the specified parser object. Returns the logging OptionGroup so that
+    callers may modify as needed.
     """
     parser.set_usage('%prog [--help] [options] ')
     parser.remove_option('-h')
@@ -330,6 +331,7 @@ def addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption, includ
     if includeNonInteractiveOption:
         addTo.add_option('-a', dest="interactive", action='store_false', default=True,
                          help="quiet mode, do not require user input for confirmations")
+    return addTo
 
 
 def addMasterDirectoryOptionForSingleClusterProgram(addTo):

gpMgmt/bin/gppylib/operations/buildMirrorSegments.py

@@ -149,10 +149,11 @@ class GpMirrorToBuild:
 
 
 class GpMirrorListToBuild:
-    def __init__(self, toBuild, pool, quiet, parallelDegree, additionalWarnings=None, logger=logger, forceoverwrite=False):
+    def __init__(self, toBuild, pool, quiet, parallelDegree, additionalWarnings=None, logger=logger, forceoverwrite=False, showProgressInplace=True):
         self.__mirrorsToBuild = toBuild
         self.__pool = pool
         self.__quiet = quiet
+        self.__showProgressInplace = showProgressInplace
         self.__parallelDegree = parallelDegree
         self.__forceoverwrite = forceoverwrite
         self.__additionalWarnings = additionalWarnings or []
@@ -451,7 +452,7 @@ class GpMirrorListToBuild:
             self.__pool.join()
         else:
             if progressCmds:
-                self._join_and_show_segment_progress(progressCmds, inplace=True)
+                self._join_and_show_segment_progress(progressCmds, inplace=self.__showProgressInplace)
             else:
                 base.join_and_indicate_progress(self.__pool)
 
@@ -487,8 +488,8 @@ class GpMirrorListToBuild:
             destSegment.primaryHostname = srcSegment.getSegmentHostName()
             destSegment.primarySegmentPort = srcSegment.getSegmentPort()
             destSegment.progressFile = '%s/pg_basebackup.%s.dbid%s.out' % (gplog.get_logger_dir(),
-                                                                               timeStamp,
-                                                                               destSegment.getSegmentDbId())
+                                                                           timeStamp,
+                                                                           destSegment.getSegmentDbId())
             srcSegments.append(srcSegment)
             destSegments.append(destSegment)
             isTargetReusedLocation.append(directive.isTargetReusedLocation())

gpMgmt/bin/gppylib/programs/clsRecoverSegment.py

@@ -127,6 +127,12 @@ class GpRecoverSegmentProgram:
         self.__pool = None
         self.logger = logger
 
+        # If user did not specify a value for showProgressInplace and
+        # stdout is a tty then send escape sequences to gprecoverseg
+        # output. Otherwise do not show progress inplace.
+        if self.__options.showProgressInplace is None:
+            self.__options.showProgressInplace = sys.stdout.isatty()
+
     def outputToFile(self, mirrorBuilder, gpArray, fileName):
         lines = []
 
@@ -251,7 +257,8 @@ class GpRecoverSegmentProgram:
         self._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
 
         return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet,
-                                   self.__options.parallelDegree, forceoverwrite=True)
+                                   self.__options.parallelDegree, forceoverwrite=True,
+                                   showProgressInplace=self.__options.showProgressInplace)
 
     def findAndValidatePeersForFailedSegments(self, gpArray, failedSegments):
         dbIdToPeerMap = gpArray.getDbIdToPeerMap()
@@ -387,7 +394,8 @@ class GpRecoverSegmentProgram:
         return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet,
                                    self.__options.parallelDegree,
                                    interfaceHostnameWarnings,
-                                   forceoverwrite=True)
+                                   forceoverwrite=True,
+                                   showProgressInplace=self.__options.showProgressInplace)
 
     def _output_segments_with_persistent_mirroring_disabled(self, segs_persistent_mirroring_disabled=None):
         if segs_persistent_mirroring_disabled:
@@ -685,7 +693,10 @@ class GpRecoverSegmentProgram:
                            version='%prog version $Revision$')
         parser.setHelp(help)
 
-        addStandardLoggingAndHelpOptions(parser, True)
+        loggingGroup = addStandardLoggingAndHelpOptions(parser, True)
+        loggingGroup.add_option("-s", None, default=None, action='store_false',
+                                dest='showProgressInplace',
+                                help='Show pg_basebackup progress sequentially instead of inplace')
 
         addTo = OptionGroup(parser, "Connection Options")
         parser.add_option_group(addTo)

gpMgmt/doc/gprecoverseg_help

@@ -10,7 +10,7 @@ Synopsis
 gprecoverseg [-p <new_recover_host>[,...]]
              |-i <recover_config_file> 
              [-d <master_data_directory>] [-B <parallel_processes>] 
-             [-F] [-a] [-q] [-l <logfile_directory>]
+             [-F] [-a] [-q] [-s] [-l <logfile_directory>]
 
 
 gprecoverseg -r
@@ -219,6 +219,12 @@ their preferred roles. All segments must be valid and synchronized before
 running gprecoverseg -r. If there are any in progress queries, they will 
 be cancelled and rolled back.
 
+-s
+
+Show pg_basebackup progress sequentially instead of inplace. Useful when
+writing to a file, or if a tty does not support escape sequences. Defaults to
+showing the progress inplace.
+
 
 -v
 

gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature

@@ -21,6 +21,19 @@ Feature: gprecoverseg tests
         Then gprecoverseg should return a return code of 0
         And gprecoverseg should not print "Unhandled exception in thread started by <bound method Worker.__bootstrap" to stdout
 
+    Scenario: gprecoverseg displays pg_basebackup progress to the user
+        Given the database is running
+        And all the segments are running
+        And the segments are synchronized
+        And user kills all mirror processes
+        When user can start transactions
+        And the user runs "gprecoverseg -F -a -s"
+        Then gprecoverseg should return a return code of 0
+        And gprecoverseg should print "pg_basebackup: base backup completed" to stdout for each mirror
+        And gpAdminLogs directory has no "pg_basebackup*" files
+        And all the segments are running
+        And the segments are synchronized
+
     Scenario: When gprecoverseg full recovery is executed and an existing postmaster.pid on the killed primary segment corresponds to a non postgres process
         Given the database is running
         And all the segments are running

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -1919,14 +1919,12 @@ def impl(context, gppkg_name):
 def impl(context, gppkg_name):
     _remove_gppkg_from_host(context, gppkg_name, is_master_host=True)
 
-
-@given('gpAdminLogs directory has no "{prefix}" files')
-def impl(context, prefix):
+@then('gpAdminLogs directory has no "{expected_file}" files')
+def impl(context, expected_file):
     log_dir = _get_gpAdminLogs_directory()
-    items = glob.glob('%s/%s_*.log' % (log_dir, prefix))
-    for item in items:
-        os.remove(item)
-
+    files_found = glob.glob('%s/%s' % (log_dir, expected_file))
+    if files_found:
+        raise Exception("expected no %s files in %s, but found %s" % (expected_file, log_dir, files_found))
 
 @given('"{filepath}" is copied to the install directory')
 def impl(context, filepath):

gpMgmt/test/behave/mgmt_utils/steps/recoverseg_mgmt_utils.py

@@ -108,3 +108,13 @@ def runCommandOnRemoteSegment(context, cid, sql_cmd):
     host = host.strip()
     psql_cmd = "PGDATABASE=\'template1\' PGOPTIONS=\'-c gp_session_role=utility\' psql -h %s -p %s -c \"%s\"; " % (host, port, sql_cmd)
     Command(name='Running Remote command: %s' % psql_cmd, cmdStr = psql_cmd).run(validateAfter=True)
+
+@then('gprecoverseg should print "{output}" to stdout for each mirror')
+def impl(context, output):
+    gparray = GpArray.initFromCatalog(dbconn.DbURL())
+    segments = gparray.getDbList()
+
+    for segment in segments:
+        if segment.isSegmentMirror():
+            expected = r'\(dbid {}\): {}'.format(segment.dbid, output)
+            check_stdout_msg(context, expected)