gpperfmon: tolerate the time it might take for test cluster to cycle

On our normal workstations, gpsmon comes up within 5 seconds very
consistently, but in other environments, it might need more time
Signed-off-by: NC.J. Jameson <cjameson@pivotal.io>

gpMgmt/test/behave/mgmt_utils/gpperfmon.feature

@@ -3,6 +3,7 @@ Feature: gpperfmon
 
     @gpperfmon_install
     Scenario: install gpperfmon
+        Given the environment variable "PGDATABASE" is set to "template1"
         Given the database "gpperfmon" does not exist
         When the user runs "gpperfmon_install --port 15432 --enable --password foo"
         Then gpperfmon_install should return a return code of 0
@@ -12,21 +13,19 @@ Feature: gpperfmon
     @gpperfmon_run
     Scenario: run gpperfmon
         # important: to succeed on macOS, see steps in gpdb/gpAux/gpperfmon/README
+        Given the environment variable "PGDATABASE" is set to "template1"
         Given the database "gpperfmon" does not exist
         When the user runs "gpperfmon_install --port 15432 --enable --password foo"
         Then gpperfmon_install should return a return code of 0
         When the user runs command "pkill postgres"
-        And waiting "5" seconds
-        And the user runs "gpstart -a"
+        Then wait until the process "postgres" goes down
+        When the user runs "gpstart -a"
         Then gpstart should return a return code of 0
-        Then verify that a role "gpmon" exists in database "gpperfmon"
-        Then verify that the last line of the master postgres configuration file contains the string "gpperfmon_log_alert_level=warning"
+        And verify that a role "gpmon" exists in database "gpperfmon"
+        And verify that the last line of the master postgres configuration file contains the string "gpperfmon_log_alert_level=warning"
         And verify that there is a "heap" table "database_history" in "gpperfmon"
-        When the user runs command "pgrep gpmmon"
-        Then pgrep should return a return code of 0
-        When waiting "5" seconds
-        When the user runs command "pgrep gpsmon"
-        Then pgrep should return a return code of 0
+        Then wait until the process "gpmmon" is up
+        And wait until the process "gpsmon" is up
 
 #    todo this test may have never run. Is it valid? Worthy of fixing?
 #    Scenario: drop old partition

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -4105,6 +4105,42 @@ def impl(context):
         time.sleep(5)
         current_time = datetime.now()
 
+@when('wait until the process "{proc}" goes down')
+@then('wait until the process "{proc}" goes down')
+@given('wait until the process "{proc}" goes down')
+def impl(context, proc):
+    cmd = Command(name='pgrep for %s' % proc, cmdStr="pgrep %s" % proc)
+    start_time = current_time = datetime.now()
+    while (current_time - start_time).seconds < 120:
+        cmd.run()
+        if cmd.get_results().rc != 0: # 1 is a good outcome; 2 or 3 we should fail fast because they're errors
+            break
+        time.sleep(2)
+        current_time = datetime.now()
+    context.ret_code = cmd.get_results().rc
+    context.error_message = ''
+    if context.ret_code > 1:
+        context.error_message = 'pgrep internal error'
+    check_return_code(context, 1) # 1 means no processes matched, but search was successful
+
+@when('wait until the process "{proc}" is up')
+@then('wait until the process "{proc}" is up')
+@given('wait until the process "{proc}" is up')
+def impl(context, proc):
+    cmd = Command(name='pgrep for %s' % proc, cmdStr="pgrep %s" % proc)
+    start_time = current_time = datetime.now()
+    while (current_time - start_time).seconds < 120:
+        cmd.run()
+        if cmd.get_results().rc != 1: # 0 is a good outcome; 2 or 3 we should fail fast because they're errors
+            break
+        time.sleep(2)
+        current_time = datetime.now()
+    context.ret_code = cmd.get_results().rc
+    context.error_message = ''
+    if context.ret_code > 1:
+        context.error_message = 'pgrep internal error'
+    check_return_code(context, 0) # 0 means one or more processes were matched
+
 @when('run gppersistent_rebuild with the saved content id')
 @then('run gppersistent_rebuild with the saved content id')
 def impl(context):