Expose gpssh's pexpect.pxssh settings in a configurable file.

The gpssh utility can have issues during the initial ssh connection
step due to different network configurations that may cause delay.
This commit exposes two key pxssh variables, delaybeforesend and
sync_multiplier, into configurables so that users can change them
as needed to ensure their gpssh works as intended in their network.

Authors: Christopher Hajas and Jimmy Yih

gpMgmt/bin/gpinitsystem

@@ -1261,6 +1261,8 @@ CREATE_QD_DB () {
 		fi
 		LOG_MSG "[INFO]:-Adding gp_dumpall access to $PG_HBA for master host"
 		BUILD_MASTER_PG_HBA_FILE $GP_DIR
+		LOG_MSG "[INFO]:-Creating gpssh configuration file"
+		BUILD_GPSSH_CONF $GP_DIR
 		LOG_MSG "[INFO]:-Creating perfmon directories and configuration file"
 		BUILD_PERFMON $GP_DIR
 		ERROR_CHK $? "create perfmon directories and configuration file" 1

gpMgmt/bin/gppylib/test/behave/mgmt_utils/gpssh.feature

+@gpssh
+Feature: gpssh behave tests
+
+    Scenario: gpssh -d and -t options
+        When the user runs "gpssh -v -h localhost hostname"
+        Then gpssh should return a return code of 0
+        And gpssh should print delaybeforesend 0.05 and prompt_validation_timeout 1.0 to stdout
+        When the user runs "gpssh -v -h localhost -d 0.051 -t 1.01 hostname"
+        Then gpssh should return a return code of 0
+        And gpssh should print Skip parsing gpssh.conf to stdout
+        And gpssh should print delaybeforesend 0.051 and prompt_validation_timeout 1.01 to stdout
+        When the user runs "gpssh -v -h localhost -d -1 hostname"
+        Then gpssh should return a return code of 1
+        And gpssh should print delaybeforesend cannot be negative to stdout
+        When the user runs "gpssh -v -h localhost -t 0 hostname"
+        Then gpssh should return a return code of 1
+        And gpssh should print prompt_validation_timeout cannot be negative or zero to stdout

gpMgmt/bin/gppylib/util/ssh_utils.py

@@ -155,7 +155,7 @@ class Session(cmd.Cmd):
         self.peerStringFormatRaw = "[%%%ds]" % cnt
         return self.peerStringFormatRaw
 
-    def login(self, hostList=None, userName=None):
+    def login(self, hostList=None, userName=None, delaybeforesend=0.05, sync_multiplier=1.0):
         '''This is the normal entry point used to add host names to the object and log in to each of them'''
         if self.verbose: print '\n[Reset ...]'
         if not (self.hostList or hostList):
@@ -181,12 +181,13 @@ class Session(cmd.Cmd):
 
         def connect_host(host):
             self.hostList.append(host)
-            p = pxssh.pxssh(options={"StrictHostKeyChecking": "no",
+            p = pxssh.pxssh(delaybeforesend=delaybeforesend,
+                            options={"StrictHostKeyChecking": "no",
                                      "BatchMode": "yes"})
             try:
                 # The sync_multiplier value is passed onto pexpect.pxssh which is used to determine timeout
                 # values for prompt verification after an ssh connection is established.
-                p.login(host, self.userName, sync_multiplier=1.0)
+                p.login(host, self.userName, sync_multiplier=sync_multiplier)
                 p.x_peer = host
                 p.x_pid = p.pid
                 good_list.append(p)

gpMgmt/bin/gppylib/util/test/unit/test_unit_ssh_utils.py

 #!/usr/bin/env python
 
+import mock
 import sys, os, pwd
 import unittest2 as unittest
 
@@ -9,7 +10,7 @@ try:
         raise Exception("GPHOME not set")
     location = "%s/bin/lib" % gphome
     sys.path.append(location)
-    from gppylib.util.ssh_utils import HostList, Session
+    from gppylib.util.ssh_utils import HostList, Session, pxssh
 except Exception as e:
     print "PYTHON PATH: %s" % ":".join(sys.path)
     print str(e)
@@ -41,6 +42,31 @@ class SshUtilsTestCase(unittest.TestCase):
         s.login(['localhost', 'fakehost'], uname)
         pxssh_hosts = [pxssh_session.x_peer for pxssh_session in s.pxssh_list]
         self.assertEqual(pxssh_hosts, ['localhost'])
- 
+
+    def test02_pxssh_delaybeforesend(self):
+        '''
+        test that delaybeforesend is changed properly
+        '''
+        p1 = pxssh.pxssh()
+        self.assertEquals(p1.delaybeforesend, 0.05)
+
+        p2 = pxssh.pxssh(delaybeforesend=3.0,
+                        options={"StrictHostKeyChecking": "no",
+                                 "BatchMode": "yes"})
+        self.assertEquals(p2.delaybeforesend, 3.0)
+
+    def test03_pxssh_sync_multiplier(self):
+        '''
+        test that sync_multiplier is changed properly
+        '''
+        with mock.patch.object(pxssh.pxssh, 'login', return_value=None) as mock_login:
+            session1 = Session()
+            session1.login(['localhost'], 'gpadmin', 0.05, 1.0)
+            mock_login.assert_called_with('localhost', 'gpadmin', sync_multiplier=1.0)
+
+            session2 = Session()
+            session2.login(['localhost'], 'gpadmin', 1.0, 4.0)
+            mock_login.assert_called_with('localhost', 'gpadmin', sync_multiplier=4.0)
+
 if __name__ == "__main__":
     unittest.main()

gpMgmt/bin/gpssh

@@ -33,10 +33,9 @@ import atexit
 import signal
 import pexpect
 import time
+import ConfigParser
 from gppylib.util import ssh_utils
 from gppylib.gpparseopts import OptParser
-from gppylib.gpcoverage import GpCoverage
-
 
 #
 # all the command line options
@@ -53,7 +52,8 @@ class __globals__:
     opt['-s'] = False
     argcmd = None
     session = None
-
+    DELAY_BEFORE_SEND = None
+    PROMPT_VALIDATION_TIMEOUT = None
 
 GV = __globals__()
 
@@ -75,10 +75,9 @@ def print_version():
 #############
 def parseCommandLine():
     try:
-        (options, args) = getopt.getopt(sys.argv[1:], '?evsh:f:D:u:', ['version'])
+        (options, args) = getopt.getopt(sys.argv[1:], '?evsh:f:D:u:d:t:', ['version'])
     except Exception, e:
         usage('Error: ' + str(e))
-
     for (switch, val) in options:
         if (switch == '-?'):              usage(0)
         elif (switch[1] in 'evDs'):    GV.opt[switch] = True
@@ -86,14 +85,49 @@ def parseCommandLine():
         elif (switch == '-h'):        GV.opt[switch].append(val)
         elif (switch == '--version'): print_version()
         elif (switch[1] in 'u'):      GV.USER = val
-
+        elif (switch == '-d'):        GV.DELAY_BEFORE_SEND = float(val)
+        elif (switch == '-t'):	      GV.PROMPT_VALIDATION_TIMEOUT = float(val)
     hf = (len(GV.opt['-h']) and 1 or 0) + (GV.opt['-f'] and 1 or 0)
     if hf != 1:
         usage('Error: please specify at least one of -h or -f args, but not both')
 
     if (len(args) >= 1):
         GV.argcmd = " ".join(args)
-           
+
+def parseConfigFile():
+    if GV.DELAY_BEFORE_SEND is not None and GV.PROMPT_VALIDATION_TIMEOUT is not None:
+        if GV.opt['-v']:
+            print 'Skip parsing gpssh.conf as both -d and -t are being used'
+        return
+
+    try:
+        config = ConfigParser.RawConfigParser()
+        config.read(os.environ.get('MASTER_DATA_DIRECTORY') + '/gpssh.conf')
+        if GV.DELAY_BEFORE_SEND is None:
+            GV.DELAY_BEFORE_SEND = config.getfloat('gpssh', 'delaybeforesend')
+        if GV.PROMPT_VALIDATION_TIMEOUT is None:
+            GV.PROMPT_VALIDATION_TIMEOUT = config.getfloat('gpssh', 'prompt_validation_timeout')
+    except Exception:
+        if GV.opt['-v']:
+            print '[WARN] Reference default values as $MASTER_DATA_DIRECTORY/gpssh.conf could not be found'
+
+        if GV.DELAY_BEFORE_SEND is None:
+            GV.DELAY_BEFORE_SEND = 0.05
+        if GV.PROMPT_VALIDATION_TIMEOUT is None:
+            GV.PROMPT_VALIDATION_TIMEOUT = 1.0
+
+    if GV.DELAY_BEFORE_SEND < 0.0:
+        print '[ERROR] delaybeforesend cannot be negative'
+        sys.exit(1)
+
+    if GV.PROMPT_VALIDATION_TIMEOUT <= 0.0:
+        print '[ERROR] prompt_validation_timeout cannot be negative or zero'
+        sys.exit(1)
+
+    if GV.opt['-v']:
+        print 'Using delaybeforesend %s and prompt_validation_timeout %s' % (GV.DELAY_BEFORE_SEND,
+                                                                             GV.PROMPT_VALIDATION_TIMEOUT)
+
 def sessionCleanup():
     while True:
         try:
@@ -152,7 +186,7 @@ def interactive():
             if not GV.session:
                 GV.session = ssh_utils.Session()
                 GV.session.verbose=GV.opt['-v']
-                GV.session.login(GV.opt['-h'], GV.USER)
+                GV.session.login(GV.opt['-h'], GV.USER, GV.DELAY_BEFORE_SEND, GV.PROMPT_VALIDATION_TIMEOUT)
                 GV.session.echoCommand=GV.opt['-e']
             if GV.opt['-s']:
                 GV.session.executeCommand("source {0}/greenplum_path.sh".format(os.environ["GPHOME"]))
@@ -171,12 +205,12 @@ def interactive():
             GV.session.reset()
             pass
 
-#############
-coverage = GpCoverage()
-coverage.start()
-
 try:
-    parseCommandLine()              #Read options from the command line
+    #Parse options from the command line
+    parseCommandLine()
+
+    #Parse gpssh.conf file
+    parseConfigFile()
 
     #Acquire the list of hosts from command line arguments
     hostlist = ssh_utils.HostList()
@@ -199,7 +233,7 @@ try:
             try:
                 GV.session = ssh_utils.Session()
                 GV.session.verbose=GV.opt['-v']
-                GV.session.login(GV.opt['-h'], GV.USER)
+                GV.session.login(GV.opt['-h'], GV.USER, GV.DELAY_BEFORE_SEND, GV.PROMPT_VALIDATION_TIMEOUT)
                 GV.session.echoCommand=GV.opt['-e']
                 if GV.opt['-s']:
                     GV.session.executeCommand("source {0}/greenplum_path.sh".format(os.environ["GPHOME"]))
@@ -220,6 +254,3 @@ try:
 except KeyboardInterrupt:
     sessionCleanup()
     sys.exit('\nInterrupted...')
-finally:
-    coverage.stop()
-    coverage.generate_report()

gpMgmt/bin/lib/gp_bash_functions.sh

@@ -1194,6 +1194,27 @@ BUILD_MASTER_PG_HBA_FILE () {
         LOG_MSG "[INFO]:-End Function $FUNCNAME"
 }
 
+BUILD_GPSSH_CONF () {
+        LOG_MSG "[INFO]:-Start Function $FUNCNAME"
+        if [ $# -eq 0 ];then ERROR_EXIT "[FATAL]:-Passed zero parameters, expected at least 1" 2;fi
+        GP_DIR=$1
+        $CAT <<_EOF_ >> $GP_DIR/gpssh.conf
+[gpssh]
+# delaybeforesend specifies the time in seconds to wait at the
+# beginning of an ssh interaction before doing anything.
+# Increasing this value can have a big runtime impact at the
+# beginning of gpssh.
+delaybeforesend = 0.05
+
+# prompt_validation_timeout specifies a timeout multiplier that
+# will be used in validating the ssh prompt. Increasing this
+# value will have a small runtime impact at the beginning of
+# gpssh.
+prompt_validation_timeout = 1.0
+_EOF_
+        LOG_MSG "[INFO]:-End Function $FUNCNAME"
+}
+
 BUILD_PERFMON() {
 	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
 	GP_DIR=$1

gpMgmt/bin/lib/pexpect/pxssh.py

@@ -86,12 +86,12 @@ class pxssh (spawn):
     '''
 
     def __init__ (self, timeout=30, maxread=2000, searchwindowsize=None,
-                    logfile=None, cwd=None, env=None, options={}):
+                    logfile=None, cwd=None, env=None, options={}, delaybeforesend=0.05):
 
         spawn.__init__(self, None, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, logfile=logfile, cwd=cwd, env=env)
 
         self.name = '<pxssh>'
-
+        self.delaybeforesend = delaybeforesend
         #SUBTLE HACK ALERT! Note that the command that SETS the prompt uses a
         #slightly different string than the regular expression to match it. This
         #is because when you set the prompt the command will echo back, but we

gpMgmt/doc/gpssh_help

@@ -8,7 +8,7 @@ SYNOPSIS
 *****************************************************
 
 gpssh { -f <hostfile_gpssh> | -h <hostname> [-h <hostname> ...] } 
-[-u userid] [-s] [-v] [-e] [<bash_command>]
+[-u userid] [-s] [-v] [-e] [-d seconds] [-t multiplier] [<bash_command>]
 
 gpssh -? 
 
@@ -50,6 +50,11 @@ to $HOME of the session user on the remote hosts after login. To ensure
 commands are executed correctly on all remote hosts, you should always 
 enter absolute paths.
 
+If you encounter network timeout problems when using gpssh, you can
+use -d and -t options or set parameters in the gpssh.conf file to
+control the timing that gpssh uses when validating the initial ssh
+connection. For information about the configuration file, see gpssh
+Configuration File.
 
 *****************************************************
 OPTIONS
@@ -62,6 +67,16 @@ OPTIONS
  will start an interactive session.
 
 
+-d (delay) seconds
+
+ Optional. Specifies the time, in seconds, to wait at the start of a
+ gpssh interaction with ssh. Default is 0.05. This option overrides the
+ delaybeforesend value that is specified in the gpssh.conf configuration
+ file.
+
+ Increasing this value can cause a long wait time during gpssh startup.
+
+
 -e (echo)
 
  Optional. Echoes the commands passed to each host and their 
@@ -91,7 +106,17 @@ OPTIONS
 
  This option is valid for both interactive mode and single command mode. 
 
- 
+
+-t multiplier
+
+ Optional. A decimal number greater than 0 (zero) that is the multiplier
+ for the timeout that gpssh uses when validating the ssh prompt. Default
+ is 1. This option overrides the prompt_validation_timeout value that is
+ specified in the gpssh.conf configuration file.
+
+ Increasing this value has a small impact during gpssh startup.
+
+
 -u <userid>
 
  Specifies the userid for this SSH session. 
@@ -115,6 +140,49 @@ OPTIONS
 
 
 
+*****************************************************
+gpssh Configuration File
+*****************************************************
+The gpssh.conf file contains parameters that let you adjust the timing
+that gpssh uses when validating the initial ssh connection. These
+parameters affect the network connection before the gpssh session
+executes commands with ssh. The location of the file is specified by the
+environment variable MASTER_DATA_DIRECTORY. If the environment variable
+is not defined or the gpssh.conf file does not exist in the directory
+gpssh uses the default values or the values set with the -d and -t
+options. For information about the environment variable, see the
+Greenplum Database Reference Guide.
+
+The gpssh.conf file is a text file that consists of a [gpssh] section
+and parameters. On a line, the # (pound sign) indicates the start of a
+comment. This is an example gpssh.conf file.
+
+
+    [gpssh]
+    delaybeforesend = 0.05
+    prompt_validation_timeout = 1.0
+
+
+These are the gpssh.conf parameters.
+
+delaybeforesend = seconds
+
+ Specifies the time, in seconds, to wait at the start of a gpssh
+ interaction with ssh. Default is 0.05. Increasing this value can
+ cause a long wait time during gpssh startup. The -d option
+ overrides this parameter.
+
+
+prompt_validation_timeout = multiplier
+
+ A decimal number greater than 0 (zero) that is the multiplier for the
+ timeout that gpssh uses when validating the ssh prompt. Increasing this
+ value has a small impact during gpssh startup. Default is 1. The -t
+ option overrides this parameter.
+
+
+
+
 *****************************************************
 EXAMPLES
 *****************************************************