behave: test that gpstart continues if standby is unreachable

Add a failing behave test to ensure that gpstart prompts and continues
successfully if the standby host is unreachable. The subsequent commit
will fix the test case.
Co-authored-by: NKalen Krempely <kkrempely@vmware.com>

gpMgmt/test/behave/mgmt_utils/environment.py

@@ -67,6 +67,9 @@ def before_scenario(context, scenario):
         drop_database_if_exists(context, 'testdb')
 
 def after_scenario(context, scenario):
+    if 'cleanup_standby_host_failure' in context:
+        context.cleanup_standby_host_failure(context)
+
     if 'gpexpand' in context.feature.tags \
         or 'gpaddmirrors' in context.feature.tags \
         or 'gpinitstandby' in context.feature.tags:

gpMgmt/test/behave/mgmt_utils/gpstart.feature

@@ -99,3 +99,17 @@ Feature: Validate command line arguments
           And gpstart should print "Number of segments not attempted to start: 1" to stdout
          # Cleanup
          Then the user runs "gprecoverseg -a"
+
+    Scenario: gpstart starts even if the standby host is unreachable
+        Given the database is running
+          And the temporary filespace is moved
+          And the catalog has a standby master entry
+
+         When the database is not running
+          And the standby host goes down
+          And gpstart is run with prompts accepted
+
+         Then gpstart should print "Continue only if you are certain that the standby is not acting as the master." to stdout
+          And gpstart should print "No standby master configured" to stdout
+          And gpstart should return a return code of 0
+          And all the segments are running

gpMgmt/test/behave/mgmt_utils/steps/gpstart.py

+import os
+import signal
+import subprocess
+
+from behave import given, when, then
+from  test.behave_utils import utils
+
+@given('the temporary filespace is moved')
+def impl(context):
+    context.execute_steps(u'''
+        Given a filespace_config_file for filespace "tempfs" is created using config file "tempfs_config" in directory "/tmp"
+          And a filespace is created using config file "tempfs_config" in directory "/tmp"
+          And the user runs "gpfilespace --movetempfilespace tempfs"
+    ''')
+
+def _run_sql(sql, opts=None):
+    env = None
+
+    if opts is not None:
+        env = os.environ.copy()
+
+        options = ''
+        for key, value in opts.items():
+            options += "-c {}={} ".format(key, value)
+
+        env['PGOPTIONS'] = options
+
+    subprocess.check_call([
+        "psql",
+        "postgres",
+        "-c", sql,
+    ], env=env)
+
+@when('the standby host goes down')
+def impl(context):
+    """
+    Fakes a host failure by updating the standby segment entry to point at an
+    invalid hostname and address.
+    """
+    opts = {
+        'gp_session_role': 'utility',
+        'allow_system_table_mods': 'dml',
+    }
+
+    subprocess.check_call(['gpstart', '-am'])
+    _run_sql("""
+        UPDATE gp_segment_configuration
+           SET hostname = 'standby.invalid',
+                address = 'standby.invalid'
+         WHERE content = -1 AND role = 'm'
+    """, opts=opts)
+    subprocess.check_call(['gpstop', '-am'])
+
+    def cleanup(context):
+        """
+        Reverses the above SQL by starting up in master-only utility mode. Since
+        the standby host is incorrect, a regular gpstart call won't work.
+        """
+
+        utils.stop_database_if_started(context)
+
+        subprocess.check_call(['gpstart', '-am'])
+        _run_sql("""
+            UPDATE gp_segment_configuration
+               SET hostname = master.hostname,
+                    address = master.address
+              FROM (
+                     SELECT hostname, address
+                       FROM gp_segment_configuration
+                      WHERE content = -1 and role = 'p'
+                   ) master
+             WHERE content = -1 AND role = 'm'
+        """, opts=opts)
+        subprocess.check_call(['gpstop', '-am'])
+
+    context.cleanup_standby_host_failure = cleanup
+
+def _handle_sigpipe():
+    """
+    Work around https://bugs.python.org/issue1615376, which is not fixed until
+    Python 3.2. This bug interferes with Bash pipelines that rely on SIGPIPE to
+    exit cleanly.
+    """
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+@when('gpstart is run with prompts accepted')
+def impl(context):
+    """
+    Runs `yes | gpstart`.
+    """
+
+    p = subprocess.Popen(
+        [ "bash", "-c", "yes | gpstart" ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        preexec_fn=_handle_sigpipe,
+    )
+
+    context.stdout_message, context.stderr_message = p.communicate()
+    context.ret_code = p.returncode

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -4235,6 +4235,7 @@ def impl(context):
         context.standby_host = standby
         run_gpcommand(context, 'gpinitstandby -ra')
 
+@given('the catalog has a standby master entry')
 @then('verify the standby master entries in catalog')
 def impl(context):
 	check_segment_config_query = "SELECT * FROM gp_segment_configuration WHERE content = -1 AND role = 'm'"