gpssh-exkeys: Add Behave tests

This commit adds some tests for gpssh-exkeys, as we want to be confident
in our test coverage before refactoring it to remove Paramiko.
Co-authored-by: NDavid Krieger <dkrieger@pivotal.io>
Co-authored-by: NJacob Champion <pchampion@pivotal.io>
Co-authored-by: NShoaib Lari <slari@pivotal.io>

gpMgmt/test/behave/mgmt_utils/environment.py

@@ -8,6 +8,7 @@ from test.behave_utils.utils import drop_database_if_exists, start_database_if_n
                                             run_command, check_user_permissions, run_gpcommand
 from steps.mirrors_mgmt_utils import MirrorMgmtContext
 from steps.gpconfig_mgmt_utils import GpConfigContext
+from steps.gpssh_exkeys_mgmt_utils import GpsshExkeysMgmtContext
 from gppylib.db import dbconn
 
 def before_all(context):
@@ -16,7 +17,7 @@ def before_all(context):
 
 def before_feature(context, feature):
     # we should be able to run gpexpand without having a cluster initialized
-    tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig']
+    tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys']
     if set(context.feature.tags).intersection(tags_to_skip):
         return
 
@@ -83,7 +84,10 @@ def before_scenario(context, scenario):
     if 'gpconfig' in context.feature.tags:
         context.gpconfig_context = GpConfigContext()
 
-    tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig']
+    if 'gpssh-exkeys' in context.feature.tags:
+        context.gpssh_exkeys_context = GpsshExkeysMgmtContext(context)
+
+    tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys']
     if set(context.feature.tags).intersection(tags_to_skip):
         return
 
@@ -93,6 +97,11 @@ def before_scenario(context, scenario):
 
 
 def after_scenario(context, scenario):
+    #TODO: you'd think that the scenario.skip() in before_scenario() would
+    #  cause this to not be needed
+    if "skip" in scenario.effective_tags:
+        return
+
     if 'tablespaces' in context:
         for tablespace in context.tablespaces.values():
             tablespace.cleanup()
@@ -102,11 +111,13 @@ def after_scenario(context, scenario):
     if set(context.feature.tags).intersection(tags_to_skip):
         return
 
-    if 'gpmovemirrors' in context.feature.tags:
+    tags_to_cleanup = ['gpmovemirrors', 'gpssh-exkeys']
+    if set(context.feature.tags).intersection(tags_to_cleanup):
         if 'temp_base_dir' in context:
             shutil.rmtree(context.temp_base_dir)
 
-    if 'analyzedb' not in context.feature.tags:
+    tags_to_not_restart_db = ['analyzedb', 'gpssh-exkeys']
+    if not set(context.feature.tags).intersection(tags_to_not_restart_db):
         start_database_if_not_started(context)
 
         home_dir = os.path.expanduser('~')

gpMgmt/test/behave/mgmt_utils/gpssh_exkeys.feature

+@gpssh-exkeys
+Feature: gpssh-exkeys behave tests
+
+    @concourse_cluster
+    Scenario: fail sensibly if 1-N is not in place
+        Given the gpssh-exkeys master host is set to "mdw"
+          And the gpssh-exkeys segment host is set to "sdw1,sdw2,sdw3"
+          And the local SSH configuration is backed up and removed
+         When gpssh-exkeys is run
+         Then gpssh-exkeys should return a return code of 1
+          And gpssh-exkeys writes "[ERROR]: Failed to ssh to sdw" to stderr
+          And gpssh-exkeys writes "[ERROR]: Expected passwordless ssh to host sdw" to stderr
+
+    @concourse_cluster
+    Scenario: N-to-N exchange works
+        Given the gpssh-exkeys master host is set to "mdw"
+          And the gpssh-exkeys segment host is set to "sdw1,sdw2,sdw3"
+          And all SSH configurations are backed up and stripped
+          And the segments can only be accessed using the master key
+          And there is no duplication in the "authorized_keys" files
+         Then all hosts "cannot" reach each other or themselves automatically
+
+         When gpssh-exkeys is run successfully
+         Then all hosts "can" reach each other or themselves automatically
+
+         # run it again to make sure that gpssh-exkeys is idempotent
+         When gpssh-exkeys is run successfully
+         Then all hosts "can" reach each other or themselves automatically
+          And there is no duplication in the "known_hosts" files
+          And there is no duplication in the "authorized_keys" files
+
+    @concourse_cluster
+    Scenario: additional hosts may be added after initial run
+        Given the gpssh-exkeys master host is set to "mdw"
+          And the gpssh-exkeys segment host is set to "sdw1,sdw2,sdw3"
+          And all SSH configurations are backed up and stripped
+          And the segments can only be accessed using the master key
+          And there is no duplication in the "authorized_keys" files
+         Then all hosts "cannot" reach each other or themselves automatically
+
+         When gpssh-exkeys is run successfully on hosts "sdw1,sdw2"
+        Given the gpssh-exkeys segment host is set to "sdw1,sdw2"
+         Then all hosts "can" reach each other or themselves automatically
+
+         When gpssh-exkeys is run successfully on additional hosts "sdw3"
+        Given the gpssh-exkeys segment host is set to "sdw1,sdw2,sdw3"
+         Then all hosts "can" reach each other or themselves automatically
+          And there is no duplication in the "known_hosts" files
+          And there is no duplication in the "authorized_keys" files
+
+    @concourse_cluster
+    Scenario: hostfiles are accepted as well
+        Given the gpssh-exkeys master host is set to "mdw"
+          And the gpssh-exkeys segment host is set to "sdw1,sdw2,sdw3"
+          And all SSH configurations are backed up and stripped
+          And the segments can only be accessed using the master key
+          And there is no duplication in the "authorized_keys" files
+         Then all hosts "cannot" reach each other or themselves automatically
+
+         When gpssh-exkeys is run successfully with a hostfile
+         Then all hosts "can" reach each other or themselves automatically
+
+    @skip
+    @concourse_cluster
+    Scenario: IPv6 addresses are accepted
+        Given the gpssh-exkeys master host is set to "mdw"
+          And the gpssh-exkeys segment host is set to "sdw1,sdw2,sdw3"
+          And all SSH configurations are backed up and stripped
+          And the segments can only be accessed using the master key
+          And there is no duplication in the "authorized_keys" files
+         Then all hosts "cannot" reach each other or themselves automatically
+
+         When gpssh-exkeys is run successfully with IPv6 addresses
+         Then all hosts "can" reach each other or themselves automatically

gpMgmt/test/behave/mgmt_utils/steps/gpssh_exkeys_mgmt_utils.py

+from os import path
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+
+import pipes
+
+from behave import given, when, then
+from test.behave_utils.utils import *
+
+from mgmt_utils import *
+
+class GpsshExkeysMgmtContext:
+    """
+    This class is intended to store per-Scenario state that is built up over a
+    series of steps.
+    """
+    def __init__(self, context):
+        self.master_host = None
+        self.segment_hosts = None
+        make_temp_dir(context, '/tmp/gpssh-exkeys', '0700')
+        self.working_directory = context.temp_base_dir
+
+    def allHosts(self):
+        allHosts = [self.master_host]
+        allHosts.extend(self.segment_hosts)
+        return allHosts
+
+
+@given('the gpssh-exkeys master host is set to "{host}"')
+def impl(context, host):
+    context.gpssh_exkeys_context.master_host = host
+
+@given('the gpssh-exkeys segment host is set to "{hosts}"')
+def impl(context, hosts):
+    context.gpssh_exkeys_context.segment_hosts = [ h.strip() for h in hosts.split(',') ]
+
+def run_exkeys(hosts, capture=False):
+    """
+    Runs gpssh-exkeys for the given list of hosts. If capture is True, the
+    (returncode, stdout, stderr) from the gpssh-exkeys run is returned;
+    otherwise an exception is thrown on failure and all stdout/err is untouched.
+    """
+    host_opts = []
+    for host in hosts:
+        host_opts.extend(['-h', host])
+
+    args = [ 'gpssh-exkeys', '-v' ] + host_opts
+
+    if not capture:
+        subprocess.check_call(args)
+        return
+
+    # Capture stdout/err for later use, while routing it through tee(1) so that
+    # developers can still see the live stream output.
+    #
+    # XXX This is a very heavy-weight solution, using pipes.Template() for the
+    # creation of shell pipeline processes. It's also platform-specific as it
+    # relies on the functionality of /dev/stdout and /dev/stderr.
+    #
+    # The overview: we open up two shell processes running tee(1), using
+    # pipes.Template(), and connect their standard output to the stdout/err of
+    # the current Python process using Template.open(). We then connect the
+    # stdout/stderr streams of subprocess.call() to the stdin of those tee
+    # pipelines. tee(1) will duplicate all output to temporary files, which we
+    # read after the subprocess call completes. NamedTemporaryFile() then cleans
+    # up those files when we return.
+    with tempfile.NamedTemporaryFile() as temp_out, tempfile.NamedTemporaryFile() as temp_err:
+        pipe_out = pipes.Template()
+        pipe_out.append('tee %s' % pipes.quote(temp_out.name), '--')
+
+        pipe_err = pipes.Template()
+        pipe_err.append('tee %s' % pipes.quote(temp_err.name), '--')
+
+        with pipe_out.open('/dev/stdout', 'w') as out, pipe_err.open('/dev/stderr', 'w') as err:
+            ret = subprocess.call(args, stdout=out, stderr=err)
+
+        stored_out = temp_out.read()
+        stored_err = temp_err.read()
+
+    return ret, stored_out, stored_err
+
+@then('gpssh-exkeys writes "{output}" to stderr')
+def impl(context, output):
+    if 'stderr' not in context:
+        raise Exception('context has no stored stderr (did you run the correct steps?)')
+
+    if output not in context.stderr:
+        msg = 'expected stderr content not found. stderr:\n%s' % context.stderr
+        raise Exception(msg)
+
+@when('gpssh-exkeys is run')
+def impl(context):
+    hosts = context.gpssh_exkeys_context.allHosts()
+    code, stdout, stderr = run_exkeys(hosts, capture=True)
+    context.ret_code = code
+    context.stdout = stdout
+    context.stderr = stderr
+
+@when('gpssh-exkeys is run successfully')
+def impl(context):
+    run_exkeys(context.gpssh_exkeys_context.allHosts())
+
+@given('gpssh-exkeys is run successfully on hosts "{hosts}"')
+@when('gpssh-exkeys is run successfully on hosts "{hosts}"')
+def impl(context, hosts):
+    run_exkeys([ h.strip() for h in hosts.split(',') ])
+
+@when('gpssh-exkeys is run successfully on additional hosts "{new_hosts}"')
+def impl(context, new_hosts):
+    new_hosts = [ h.strip() for h in new_hosts.split(',') ]
+    old_hosts = [
+        h for h in context.gpssh_exkeys_context.allHosts() if h not in new_hosts
+    ]
+
+    old_host_file = tempfile.NamedTemporaryFile()
+    new_host_file = tempfile.NamedTemporaryFile()
+
+    with old_host_file, new_host_file:
+        for h in old_hosts:
+            old_host_file.write(h + '\n')
+        old_host_file.flush()
+
+        for h in new_hosts:
+            new_host_file.write(h + '\n')
+        new_host_file.flush()
+
+        subprocess.check_call([
+            'gpssh-exkeys',
+            '-v',
+            '-e', old_host_file.name,
+            '-x', new_host_file.name,
+        ])
+
+@when('gpssh-exkeys is run successfully with a hostfile')
+def impl(context):
+    with tempfile.NamedTemporaryFile() as host_file:
+        for h in context.gpssh_exkeys_context.allHosts():
+            host_file.write(h + '\n')
+        host_file.flush()
+
+        subprocess.check_call([
+            'gpssh-exkeys',
+            '-v',
+            '-f', host_file.name,
+        ])
+
+@when('gpssh-exkeys is run successfully with IPv6 addresses')
+def impl(context):
+    ipv6_addrs = []
+    for host in context.gpssh_exkeys_context.allHosts():
+        # Try to look up an IPv6 address for each host.
+        try:
+            addrs = socket.getaddrinfo(host, None, socket.AF_INET6)
+        except socket.gaierror as err:
+            raise Exception, \
+                "failed to find IPv6 address for host '{}': {}".format(host, err), \
+                sys.exc_info()[2]
+
+        # getaddrinfo() return value is a bit opaque. For AF_INET6, it's a list
+        # of (family, socktype, proto, canonname, (address, port, flow info, scope id))
+        # nested tuples. We're interested in the address piece of the first
+        # entry in that list.
+        addr = addrs[0][4][0]
+        print host, "maps to", addr
+
+        ipv6_addrs.append(addr)
+
+    run_exkeys(ipv6_addrs)
+
+
+@then('all hosts "{works}" reach each other or themselves automatically')
+def impl(context, works):
+    steps = u'''
+    Then the segment hosts "{0}" reach each other or themselves automatically
+     And the segment hosts "{0}" reach the master
+     And the master host "{0}" reach itself
+    '''.format(works)
+    context.execute_steps(steps)
+
+
+# TODO: we are currently not using gpssh so we can control StrictHostKeyChecking=yes
+@then('the segment hosts "{works}" reach each other or themselves automatically')
+def impl(context, works):
+    ret = 255
+    if (works == 'can'):
+        ret = 0
+    # NOTE: we tried using scp with files instead, but -o BatchMode=yes -o StrictHostKeyChecking=yes
+    # still asked us for a prompt.
+    # we're not using gpssh here because we want to test each connection
+    for fromHost in context.gpssh_exkeys_context.segment_hosts:
+        for toHost in context.gpssh_exkeys_context.segment_hosts:
+            cmd = u'''
+            When the user runs command "ssh -o BatchMode=yes -o StrictHostKeyChecking=yes %s \"ssh -o BatchMode=yes -o StrictHostKeyChecking=yes %s hostname\"" eok
+            And ssh should return a return code of %d
+            ''' % (fromHost, toHost, ret)
+            print "CMD:%s" % cmd
+            context.execute_steps(cmd)
+
+
+@then('the segment hosts "{works}" reach the master')
+def impl(context, works):
+    host_opts = []
+    for host in context.gpssh_exkeys_context.segment_hosts:
+        host_opts.extend(['-h', host])
+
+    subprocess.check_call([
+        'gpssh',
+        '-e',
+        ] + host_opts + [
+        '{}ssh -o BatchMode=yes -o StrictHostKeyChecking=yes mdw true'.format(
+            "" if (works == 'can') else "! "
+        )
+    ])
+
+
+@then('the master host "{works}" reach itself')
+def impl(context, works):
+    result = subprocess.call(['ssh', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', 'mdw', 'true'])
+    should_work = (works == 'can')
+    did_work = (result == 0)
+    if should_work != did_work:
+        expected_code = '0' if should_work else 'not 0'
+        raise Exception('actual result of ssh mdw: %s (expected: %s)', result, expected_code)
+
+
+@given('all SSH configurations are backed up and stripped')
+def impl(context):
+    """
+    Strips out part of the ssh secrets to setup the cluster so only ssh from
+    the local to the remotes works.
+    """
+    host_opts = []
+    for host in context.gpssh_exkeys_context.segment_hosts:
+        host_opts.extend(['-h', host])
+
+    # Everything except authorized_keys is moved elsewhere.
+    subprocess.check_call([
+        'gpssh',
+        '-e',
+        ] + host_opts + [(
+        'mkdir -p /tmp/ssh.bak '
+        '&& mv -f ~/.ssh/* /tmp/ssh.bak '
+        '&& cp -fp /tmp/ssh.bak/authorized_keys ~/.ssh/'
+    )])
+
+    # Also backup .ssh on mdw, leaving the key configuration in .ssh
+    home_ssh = path.expanduser('~/.ssh')
+    backup_path = '/tmp/ssh.bak/'
+    os.makedirs(backup_path)
+    for ssh_file in os.listdir(home_ssh):
+        if not ssh_file.startswith('id_rsa'):
+            shutil.move(path.join(home_ssh, ssh_file), backup_path)
+
+    # Make sure the configuration is restored at the end.
+    def cleanup():
+        subprocess.check_call([
+            'gpssh',
+            '-e',
+            ] + host_opts + [
+            'mv -f /tmp/ssh.bak/* ~/.ssh/',
+        ])
+        for f in os.listdir(backup_path):
+            shutil.move(path.join(backup_path, f), path.join(home_ssh, f))
+        os.rmdir(backup_path)
+
+    context.add_cleanup(cleanup)
+
+@given('the local SSH configuration is backed up and removed')
+def impl(context):
+    """
+    Strips out part of the ssh secrets to setup the cluster so only ssh from
+    the local to the remotes works.
+    """
+    # Also backup .ssh on mdw, leaving the key configuration in .ssh
+    home_ssh = path.expanduser('~/.ssh')
+    backup_path = tempfile.mkdtemp()
+    for ssh_file in os.listdir(home_ssh):
+        shutil.move(path.join(home_ssh, ssh_file), backup_path)
+
+    # Make sure the configuration is restored at the end.
+    def cleanup():
+        for f in os.listdir(backup_path):
+            shutil.move(path.join(backup_path, f), path.join(home_ssh, f))
+        os.rmdir(backup_path)
+
+    context.add_cleanup(cleanup)
+
+@given('the segments can only be accessed using the master key')
+def impl(context):
+    host_opts = []
+    for host in context.gpssh_exkeys_context.segment_hosts:
+        host_opts.extend(['-h', host])
+
+    # This blows away any existing authorized_keys file on the segments.
+    subprocess.check_call([
+        'gpscp',
+        '-v',
+        ] + host_opts + [
+        '~/.ssh/id_rsa.pub',
+        '=:~/.ssh/authorized_keys'
+    ])
+
+@given('there is no duplication in the "{ssh_type}" files')
+@then('there is no duplication in the "{ssh_type}" files')
+def impl(context, ssh_type):
+    host_opts = []
+    for host in context.gpssh_exkeys_context.segment_hosts:
+        host_opts.extend(['-h', host])
+
+    # ssh'ing to localhost need not be set up yet
+    subprocess.check_call([ 'bash', '-c', '! sort %s | uniq -d | grep .' % path.join('~/.ssh',pipes.quote(ssh_type))])
+
+    subprocess.check_call([
+        'gpssh',
+        '-e',
+        ] + host_opts + [
+        '! sort %s | uniq -d | grep .' % path.join('~/.ssh',pipes.quote(ssh_type))
+    ])

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -372,6 +372,11 @@ def impl(context, command):
     if has_exception(context):
         raise context.exception
 
+@given('the user runs command "{command}" eok')
+@when('the user runs command "{command}" eok')
+@then('the user runs command "{command}" eok')
+def impl(context, command):
+    run_command(context, command)
 
 @when('the user runs async command "{command}"')
 def impl(context, command):