From 9a181d4b3a6e64499eaf442ed6c56b19fbfda5db Mon Sep 17 00:00:00 2001 From: Jamie McAtamney Date: Tue, 30 Apr 2019 16:12:25 -0700 Subject: [PATCH] Remove gpseginstall Now that the enterprise version of GPDB is only provided via RPM, including gpseginstall in the distribution would cause conflicts if users try to install GPDB with RPMs and with gpseginstall on the same cluster. While it could be preserved for use by the OSS community, there are several standard tools for copying GPDB files to segment hosts in a cluster, and recommendations for using one or more of those tools will be included in the GPDB documentation. In addition to removing gpseginstall itself, this commit removes references to it in other utilities' documentation and removes code in gppylib that was only called by gpseginstall. Co-authored-by: Jamie McAtamney Co-authored-by: Kalen Krempely (cherry picked from commit 6401468510b02eb566e99dc406f6a2f589145a11) --- gpMgmt/bin/.gitignore | 1 - gpMgmt/bin/gppylib/commands/base.py | 205 +---- gpMgmt/bin/gpseginstall | 973 ---------------------- gpMgmt/doc/gpinitsystem_help | 2 +- gpMgmt/doc/gpscp_help | 2 +- gpMgmt/doc/gpseginstall_help | 157 ---- gpMgmt/doc/gpssh-exkeys_help | 2 +- gpMgmt/test/behave_utils/cluster_setup.py | 30 - 8 files changed, 13 insertions(+), 1359 deletions(-) delete mode 100755 gpMgmt/bin/gpseginstall delete mode 100644 gpMgmt/doc/gpseginstall_help diff --git a/gpMgmt/bin/.gitignore b/gpMgmt/bin/.gitignore index 2c93dbf96c..b92b29a65c 100644 --- a/gpMgmt/bin/.gitignore +++ b/gpMgmt/bin/.gitignore @@ -26,7 +26,6 @@ ext /gpreloadc /gpscpc /gpsdc -/gpseginstallc /gpssh-exkeysc /gpsshc /gpstartc diff --git a/gpMgmt/bin/gppylib/commands/base.py b/gpMgmt/bin/gppylib/commands/base.py index bca5f24281..90a5671b2c 100755 --- a/gpMgmt/bin/gppylib/commands/base.py +++ b/gpMgmt/bin/gppylib/commands/base.py @@ -29,12 +29,6 @@ from gppylib import gplog from gppylib import gpsubprocess from pygresql.pg import DB -# paramiko prints deprecation warnings which are ugly to the end-user -import warnings - -warnings.simplefilter('ignore', DeprecationWarning) -import paramiko, getpass - logger = gplog.get_default_logger() GPHOME = os.environ.get('GPHOME') @@ -389,20 +383,19 @@ class ExecutionError(Exception): # specify types of execution contexts. LOCAL = 1 REMOTE = 2 -NAKED = 4 gExecutionContextFactory = None # -# @param factory needs to have a createExecutionContext(self, execution_context_id, remoteHost, stdin, nakedExecutionInfo) function +# @param factory needs to have a createExecutionContext(self, execution_context_id, remoteHost, stdin) function # def setExecutionContextFactory(factory): global gExecutionContextFactory gExecutionContextFactory = factory -def createExecutionContext(execution_context_id, remoteHost, stdin, nakedExecutionInfo=None, gphome=None): +def createExecutionContext(execution_context_id, remoteHost, stdin, gphome=None): if gExecutionContextFactory is not None: return gExecutionContextFactory.createExecutionContext(execution_context_id, remoteHost, stdin) elif execution_context_id == LOCAL: @@ -411,13 +404,6 @@ def createExecutionContext(execution_context_id, remoteHost, stdin, nakedExecuti if remoteHost is None: raise Exception("Programmer Error. Specified REMOTE execution context but didn't provide a remoteHost") return RemoteExecutionContext(remoteHost, stdin, gphome) - elif execution_context_id == NAKED: - if remoteHost is None: - raise Exception("Programmer Error. Specified NAKED execution context but didn't provide a remoteHost") - if nakedExecutionInfo is None: - raise Exception( - "Programmer Error. Specified NAKED execution context but didn't provide a NakedExecutionInfo") - return NakedExecutionContext(remoteHost, stdin, nakedExecutionInfo) class ExecutionContext(): @@ -432,10 +418,10 @@ class ExecutionContext(): def execute(self, cmd): pass - def interrupt(self, cmd): + def interrupt(self): pass - def cancel(self, cmd): + def cancel(self): pass @@ -472,190 +458,19 @@ class LocalExecutionContext(ExecutionContext): cmd.set_results(CommandResult( rc, "".join(stdout_value), "".join(stderr_value), self.completed, self.halt)) - def cancel(self, cmd): + def cancel(self): if self.proc: try: os.kill(self.proc.pid, signal.SIGTERM) except OSError: pass - def interrupt(self, cmd): + def interrupt(self): self.halt = True if self.proc: self.proc.cancel() -########################################################################## -# Naked Execution is used to run commands where ssh keys are not exchanged -class NakedExecutionInfo: - SFTP_NONE = 0 - SFTP_PUT = 1 - SFTP_GET = 2 - - def __init__(self, passwordMap, sftp_operation=SFTP_NONE, sftp_remote=None, sftp_local=None): - self.passwordMap = passwordMap - self.sftp_operation = sftp_operation - self.sftp_remote = sftp_remote - self.sftp_local = sftp_local - - -class NakedExecutionPasswordMap: - def __init__(self, hostlist): - self.hostlist = hostlist - self.mapping = dict() - self.unique_passwords = set() - self.complete = False - - # this method throws exceptions on error to create a valid list - def discover(self): - - for host in self.hostlist: - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # TRY NO PASSWORD - try: - client.connect(host) - self.mapping[host] = None - client.close() - continue # next host - except Exception, e: - pass - - try: - client.close() - except Exception, e: - pass - - # TRY EXISTING PASSWORDS - foundit = False - for passwd in self.unique_passwords: - try: - client.connect(host, password=passwd) - foundit = True - self.mapping[host] = passwd - break - except Exception, e: - pass - - if foundit: - continue - - # ASK USER - foundit = False - for attempt in range(5): - try: - passwd = getpass.getpass(' *** Enter password for %s: ' % (host), sys.stderr) - client.connect(host, password=passwd) - foundit = True - self.mapping[host] = passwd - if passwd not in self.unique_passwords: - self.unique_passwords.add(passwd) - break - except Exception, e: - pass - - try: - client.close() - except Exception, e: - pass - - if not foundit: - raise Exception("Did not get a valid password for host " + host) - - if len(self.mapping.keys()) == len(self.hostlist) and len(self.hostlist) > 0: - self.complete = True - - -class NakedExecutionContext(LocalExecutionContext): - def __init__(self, targetHost, stdin, nakedCommandInfo): - - LocalExecutionContext.__init__(self, stdin) - self.targetHost = targetHost - self.passwordMap = nakedCommandInfo.passwordMap - self.sftp_operation = nakedCommandInfo.sftp_operation - self.sftp_remote = nakedCommandInfo.sftp_remote - self.sftp_local = nakedCommandInfo.sftp_local - self.client = None - - def execute(self, cmd): - - self.client = paramiko.SSHClient() - self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - try: - self.client.connect(self.targetHost, password=self.passwordMap.mapping[self.targetHost]) - except paramiko.AuthenticationException: - self.client.close() - cmd.set_results(CommandResult(1, "", "password validation on %s failed" % self.targetHost, False, False)) - return - except Exception, e: - cmd.set_results( - CommandResult(1, "", "connection to host " + self.targetHost + " failed: " + e.__str__(), False, False)) - return - - if self.sftp_operation == NakedExecutionInfo.SFTP_NONE: - self.execute_ssh(cmd) - elif self.sftp_operation == NakedExecutionInfo.SFTP_PUT: - self.execute_sftp_put(cmd) - elif self.sftp_operation == NakedExecutionInfo.SFTP_GET: - self.execute_sftp_get(cmd) - else: - raise Exception("bad NakedExecutionInfo.sftp_operation") - - def execute_ssh(self, cmd): - - try: - stdin, stdout, stderr = self.client.exec_command(cmd.cmdStr) - rc = stdout.channel.recv_exit_status() - self.completed = True - cmd.set_results(CommandResult(rc, stdout.readlines(), stderr.readlines(), self.completed, self.halt)) - stdin.close() - stdout.close() - stderr.close() - except Exception, e: - cmd.set_results(CommandResult(1, "", e.__str__(), False, False)) - finally: - self.client.close() - - def execute_sftp_put(self, cmd): - - ftp = None - try: - ftp = self.client.open_sftp() - ftp.put(self.sftp_local, self.sftp_remote) - self.completed = True - cmd.set_results(CommandResult(0, "", "", self.completed, self.halt)) - except Exception, e: - cmd.set_results(CommandResult(1, "", e.__str__(), False, False)) - finally: - ftp.close() - self.client.close() - - def execute_sftp_get(self, cmd): - - ftp = None - try: - ftp = self.client.open_sftp() - ftp.get(self.sftp_remote, self.sftp_local) - self.completed = True - cmd.set_results(CommandResult(0, "", "", self.completed, self.halt)) - except Exception, e: - cmd.set_results(CommandResult(1, "", e.__str__(), False, False)) - finally: - ftp.close() - self.client.close() - - def interrupt(self, cmd): - self.halt = True - self.client.close() - cmd.set_results(CommandResult(1, "", "command on host " + self.targetHost + " interrupted ", False, False)) - - def cancel(self, cmd): - self.client.close() - cmd.set_results(CommandResult(1, "", "command on host " + self.targetHost + " canceled ", False, False)) - - class RemoteExecutionContext(LocalExecutionContext): trail = set() """ @@ -708,10 +523,10 @@ class Command(object): exec_context = None propagate_env_map = {} # specific environment variables for this command instance - def __init__(self, name, cmdStr, ctxt=LOCAL, remoteHost=None, stdin=None, nakedExecutionInfo=None, gphome=None): + def __init__(self, name, cmdStr, ctxt=LOCAL, remoteHost=None, stdin=None, gphome=None): self.name = name self.cmdStr = cmdStr - self.exec_context = createExecutionContext(ctxt, remoteHost, stdin=stdin, nakedExecutionInfo=nakedExecutionInfo, + self.exec_context = createExecutionContext(ctxt, remoteHost, stdin=stdin, gphome=gphome) self.remoteHost = remoteHost self.logger = gplog.get_default_logger() @@ -772,11 +587,11 @@ class Command(object): def cancel(self): if self.exec_context and isinstance(self.exec_context, ExecutionContext): - self.exec_context.cancel(self) + self.exec_context.cancel() def interrupt(self): if self.exec_context and isinstance(self.exec_context, ExecutionContext): - self.exec_context.interrupt(self) + self.exec_context.interrupt() def was_successful(self): if self.results is None: diff --git a/gpMgmt/bin/gpseginstall b/gpMgmt/bin/gpseginstall deleted file mode 100755 index df8a22a8b9..0000000000 --- a/gpMgmt/bin/gpseginstall +++ /dev/null @@ -1,973 +0,0 @@ -#!/usr/bin/env python -''' -USAGE: gpseginstall -f|--file HOSTFILE [-u|--user USER] [-g|--group GROUP] [-p|--password PASSWORD] [-c|--commands COMMAND_OPTIONS] [--verbose] - where HOSTFILE lists all the hosts to install the software on - where USER will be the user to install the software as. The default is gpadmin. (root only option) - where GROUP will be the group to install the software as. The default is gpadmin. - where PASSWORD will be set for the USER default is changeme (root only option) - - where COMMAND_OPTIONS list a subset of commands to be run from the list below. The default is all commands. - u -- add user (root only) - c -- chown software on master host (root only) - s -- tar, zip and copy over software to machines in cluster - p -- change passwords (root only) - E -- exchange keys for root (root only) - e -- exchange keys for user - l -- check and fix user limits for new user -- requires software to be installed on cluster with command 's' or equivalent (root only) - v -- verify software on remote machines -''' - -import os, sys, re -import subprocess, warnings, logging, tarfile -warnings.simplefilter('ignore', DeprecationWarning) -sys.path.append(sys.path[0] + '/lib') - -try: - import paramiko, getpass, pexpect - import gppylib.userinput - from optparse import Option, OptionParser - from gppylib.gpparseopts import OptParser, OptChecker - from gppylib.gplog import get_default_logger, setup_tool_logging - from gppylib.commands.unix import getLocalHostname, getUserName, SYSTEM - from gppylib.commands.base import WorkerPool, Command, NakedExecutionPasswordMap, NakedExecutionInfo, NAKED -except ImportError, e: - sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) - -EXECNAME = os.path.split(__file__)[-1] - -logger = get_default_logger() -setup_tool_logging(EXECNAME,getLocalHostname(),getUserName()) -hosts = dict() # address to HostMapping object -uniqueseghosts = dict() # hostname to UniqueHost object -options = None -passwordMap = None -nullFile = logging.FileHandler('/dev/null') -logging.getLogger('paramiko.transport').addHandler(nullFile) -pool = None -gphome = None -installation_info = None -commands = set() # selected commands -rootCommands = set(['u', 'p', 'c', 's', 'e', 'E', 'l', 'v']) # valid command types for root -nonRootCommands = set(['s', 'e', 'v']) # valid command types for non-root user -defaultNonRootCommands = set(['s', 'v']) # commands to run by default for non-root user -currentUser = None -isLinux = False -GPSSH_EXKEYS_TIMEOUT = 600 - -class InstallationInfo: - def __init__(self, link_name, binary_path, binary_dir_location, binary_dir_name): - self.install_link_name = link_name # greenplum-db - self.install_binary_path = binary_path # /data/release-4.0 - self.install_binary_dir_location = binary_dir_location # /data - self.install_binary_dir_name = binary_dir_name # release-4.0 - self.install_md5 = None - self.install_version_string = None - - def debugStr(self): - return "link_name %s\nbinary_path %s\nbinary_dir_location %s\nbinary_dir_name %s" % (self.install_link_name, self.install_binary_path, self.install_binary_dir_location, self.install_binary_dir_name) - -class HostMapping: - def __init__(self, address): - self.address = address - self.hostname = None - - def debugStr(self): - return "address %s hostname %s" % (self.address, self.hostname) - -class UniqueHost: - def __init__(self, address, hostname): - self.address = address - self.hostname = hostname - self.userExists = False - - def debugStr(self): - return "address %s hostname %s userExists %s" % (self.address, self.hostname, self.userExists) - -def cli_help(): - help_path = os.path.join(sys.path[0], '..', 'docs', 'cli_help', EXECNAME + '_help') - f = None - try: - try: - f = open(help_path); - return f.read(-1) - except: - return '' - finally: - if f: f.close() - - -def usage(): - print cli_help() or __doc__ - -# True is an error -def parseargs(): - - global options, isLinux - - parser = OptParser(option_class=OptChecker) - parser.remove_option('-h') - parser.add_option('-h', '-?', '--help', action='store_true') - parser.add_option('-f', '--file', type='string') - parser.add_option('-u', '--user', type='string', default='gpadmin') - parser.add_option('-g', '--group', type='string') - parser.add_option('-p', '--password', type='string') - parser.add_option('-c', '--commands', type='string') - parser.add_option('--verbose', action='store_true') - (options, args) = parser.parse_args() - - global gphome - gphome = os.environ.get('GPHOME') - if not gphome: - logger.error("GPHOME not set") - return True - - if SYSTEM.getName() != "linux": - logger.error("This utility is only supported on the linux operating system") - return True - - if SYSTEM.getName() == "linux": - isLinux = True - - if options.help: - usage() - return True - - if not options.file: - logger.error("--file must be specified") - return True - - if not os.path.exists(options.file): - logger.error("file %s does not exist" % options.file) - return True - - if options.user == "root": - logger.error("can not install the software into the account of user root") - return True - - global currentUser - currentUser = getpass.getuser() - - if currentUser == "root": - validCommands = rootCommands - else: - validCommands = nonRootCommands - - if options.user != currentUser: - logger.error("--user option '%s' does not equal non-root user running this utility '%s'" % (options.user, currentUser)) - return True - - if not options.group: - options.group = options.user - - global commands - if options.commands and len(options.commands): - for i in range(len(options.commands)): - if options.commands[i] in validCommands: - if options.commands[i] not in commands: - commands.add(options.commands[i]) - else: - logger.error("'%s' is not a valid command for user(%s)" % (options.commands[i], currentUser)) - return True - else: - if currentUser == "root": - commands = rootCommands - else: - commands = defaultNonRootCommands - - location = os.path.abspath(gphome) - if os.path.islink(location): - link_name = os.path.split(location)[1] - binary_path = os.path.realpath(location) - else: - link_name = None - binary_path = location - - (binary_dir_location, binary_dir_name) = os.path.split(binary_path) - - global installation_info - installation_info = InstallationInfo(link_name, binary_path, binary_dir_location, binary_dir_name) - logger.info("Installation Info:\n" + installation_info.debugStr()) - - return False - -# True is an error -def readHostList(): - - try: - for line in open(options.file, "r"): - hostname = line.strip() - if not len(hostname): - continue - hosts[hostname] = HostMapping(hostname) - except Exception, e: - logger.error("Error while reading file: %s" % options.file) - logger.error(e.__str__()) - return True - - if not len(hosts.keys()): - logger.error("No hosts read from hostfile: %s" % options.file) - return True - - return False - -# True is an error -def discoverPasswordMap(): - - logger.info("check cluster password access") - - global passwordMap - - try: - passwordMap = NakedExecutionPasswordMap(hosts.keys()) - passwordMap.discover() - except Exception, e: - logger.error("could not successfully access all machines") - msg = e.__str__() - if msg: - logger.error("trace: %s" % msg) - return True - - if passwordMap.complete: - return False - else: - return True - -# True is an error -def dedupeHosts(): - - logger.info("de-duplicate hostnames") - - masterHostName = None - - try: - cmd = Command("master", "hostname") - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if i.results.rc or i.results.halt or not i.results.completed: - logger.error("error trying to obtain hostname on master %s" % i.results.stderr) - return True - masterHostName = i.results.stdout.strip() - break - except Exception, e: - logger.error("exception trying to obtain hostname on master: %s" % e.__str__()) - return True - - if not masterHostName: - logger.error("Could not find master hostname") - return True - - logger.info("master hostname: %s" % masterHostName) - - try: - for host in hosts.keys(): - cmd = Command(host, "hostname", NAKED, host, nakedExecutionInfo=NakedExecutionInfo(passwordMap)) - pool.addCommand(cmd) - - pool.join() - items = pool.getCompletedItems() - for i in items: - address = i.remoteHost - if i.results.rc or i.results.halt or not i.results.completed: - logger.error("error obtaining information from host %s" % address) - return True - hostname = i.results.stdout[0].strip() - hosts[address].hostname = hostname - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - global uniqueseghosts - for k in hosts.keys(): - hostname = hosts[k].hostname - address = hosts[k].address - - if hostname in uniqueseghosts: - # Here we do a heuristic: - # there are several interfaces to each host - # and we don't want to pick a slow interface - # it is very likely that a hostname with a '-' will represent a fast link - # so try to pick an address that contains a '-' - # for example choose sdw1-1 over vert1 - if (not re.search("-", uniqueseghosts[hostname].address)) and (re.search("-", address)): - uniqueseghosts[hostname].address = address - continue - - if masterHostName == hostname: - continue - - uniqueseghosts[hostname] = UniqueHost(address, hostname) - - if options.verbose: - for k in uniqueseghosts.keys(): - logger.info("segment hostname: %s (%s)" % (uniqueseghosts[k].address, k)) - - return False - -# True is an error -def checkUsernames(): - - logger.info("check for user %s on cluster" % options.user) - - try: - for k in uniqueseghosts.keys(): - uh = uniqueseghosts[k] - cmd = Command(uh.address, "id %s" % options.user, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap)) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if not i.results.rc: - address = i.remoteHost - hostname = hosts[address].hostname - uniqueseghosts[hostname].userExists = True - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - return False - -def getAddUserCommand(): - if isLinux: - return "useradd -r -m %s" % options.user - else: - return "groupadd %s; useradd -d /export/home/%s -m -g %s -s /bin/bash %s" % (options.user, options.user, options.user, options.user) - -# True is an error -def addUserIfRequired(): - - ################################################################################### - logger.info("add user %s on master" % options.user) - try: - cmd = Command("useradd", getAddUserCommand()) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() # get the completed items but ignore them - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - try: - cmd = Command("checkId", "id %s" % options.user) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if i.results.rc: - logger.error("failed to add user %s to master host: %s" % (options.user, i.results.stderr)) - return True - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - ################################################################################### - logger.info("add user %s on cluster" % options.user) - - failedToAddUser = set() # set of address's where useradd failed - try: - for k in uniqueseghosts.keys(): - uh = uniqueseghosts[k] - if uh.userExists: - continue - - cmd = Command(uh.address, getAddUserCommand(), NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap)) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - address = i.remoteHost - if not i.results.rc: - hostname = hosts[address].hostname - uniqueseghosts[hostname].userExists = True - else: - logger.error("%s: %s" % (address, i.results.stderr)) - failedToAddUser.add(address) - - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - if len(failedToAddUser): - for h in failedToAddUser: - logger.error("could not create user %s on host %s" % (options.user, h)) - return True - - return False - -# True is an error -def getLocalSoftwareVersion(): - - global installation_info - cmdStr = "%s/bin/gpssh --version" % gphome - try: - cmd = Command("version", cmdStr) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if i.results.rc: - logger.error("Failed to run command: %s" % cmd.cmdStr) - logger.error(i.results.stderr) - return True - - installation_info.install_version_string = i.results.stdout.strip() - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - return False - -# True is an error -def simpleLocalCommand(cmdStr, checkError=True, verbose=True): - if verbose: - logger.info(cmdStr) - try: - cmd = Command("simpleLocalCommand", cmdStr) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - if checkError: - for i in items: - if i.results.rc: - logger.error("Failed to run command: %s" % cmd.cmdStr) - logger.error(i.results.stderr) - return True - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - return False - -# True is an error -def simpleRemoteCommand(cmdStr, checkError=True, verbose=True): - - failures = set() - if verbose: - logger.info("remote command: %s" % cmdStr) - - try: - for k in uniqueseghosts.keys(): - uh = uniqueseghosts[k] - cmd = Command(uh.address, cmdStr, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap)) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - if checkError: - for i in items: - if i.results.rc: - logger.error("%s: %s" % (i.remoteHost, i.results.stderr)) - failures.add(i.remoteHost) - - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - if len(failures): - for h in failures: - logger.error("error running command %s on host %s" % (cmdStr, h)) - return True - - return False - - -# True is an error -def chownMasterSoftware(): - - ################################################################################### - if installation_info.install_link_name: - - cmdStr = "chown -R %s:%s %s/%s" % (options.user, options.group, installation_info.install_binary_dir_location, installation_info.install_link_name) - if (simpleLocalCommand(cmdStr, True)): - return True - - ################################################################################### - cmdStr = "chown -R %s:%s %s" % (options.user, options.group, installation_info.install_binary_path) - if (simpleLocalCommand(cmdStr, True)): - return True - - -# True is an error -def changeUserPassword(): - - if not options.password: - password = gppylib.userinput.ask_create_password() - if not password: - logger.error("error obtaining password from user") - return True - else: - password = options.password - cmd = "%s/sbin/gpchangeuserpassword --user %s --password %s" % (gphome, options.user, password) - - logger.info("Changing system passwords ...") - if (simpleLocalCommand(cmd, False, verbose=False)): - return True - - cmdStr = ". %s/greenplum_path.sh; %s" % (gphome, cmd) - - if (simpleRemoteCommand(cmdStr, True, verbose=False)): - return True - -def md5Command(): - if isLinux: - return "md5sum" - else: - return "digest -a md5" - -def md5OutputWords(): - if isLinux: - return 2 - else: - return 1 - - -# True is an error -def setMd5Locally(): - - try: - cmd = Command("md5", "%s %s.tar" % (md5Command(), installation_info.install_binary_path)) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if i.results.rc: - logger.error("Failed to run command: %s" % cmd.cmdStr) - logger.error(i.results.stderr) - return True - else: - fields = i.results.stdout.split() - if len(fields) != md5OutputWords(): - raise Exception("Unexpected output from md5sum: %s" % i.results.stdout) - installation_info.install_md5 = fields[0].strip() - break - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - return False - -# True is an error -def copyOverSoftware(): - - ################################################################################### - cmdStr = "rm -f %s.tar; rm -f %s.tar.gz" % (installation_info.install_binary_path, installation_info.install_binary_path) - if (simpleLocalCommand(cmdStr, False)): - return True - - cmdStr = "cd %s; tar cf %s.tar %s" % (installation_info.install_binary_dir_location, installation_info.install_binary_dir_name, installation_info.install_binary_dir_name) - if (simpleLocalCommand(cmdStr, False)): - return True - - ################################################################################### - if setMd5Locally(): - return True - - ################################################################################### - - cmdStr = "gzip %s.tar" % installation_info.install_binary_path - if (simpleLocalCommand(cmdStr, False)): - return True - - ################################################################################### - cmdStr = "mkdir -p %s" % installation_info.install_binary_dir_location - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - # a couple paranoid checks - if installation_info.install_binary_path == "/" or \ - installation_info.install_binary_path == "/usr" or \ - installation_info.install_binary_path == "/usr/bin" or \ - re.search('boot', installation_info.install_binary_path): - raise Exception("illegal path for installaiton %s" % installation_info.install_binary_path) - - cmdStr = "rm -rf %s" % (installation_info.install_binary_path) - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - logger.info("scp software to remote location") - - failures = set() - - filename = installation_info.install_binary_path + ".tar.gz" - try: - for k in uniqueseghosts.keys(): - uh = uniqueseghosts[k] - cmdStr = "scp %s %s:%s" % (filename, uh.address, installation_info.install_binary_dir_location) - cmd = Command(uh.address, cmdStr) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if i.results.rc: - logger.error("command failed: '%s': %s" % (i.cmdStr, i.results.stderr.strip())) - failures.add(i.name) - - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - if len(failures): - return True - - - ################################################################################### - cmdStr = "gzip -f -d %s.tar.gz" % installation_info.install_binary_path - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - logger.info("md5 check on remote location") - failures.clear() - - try: - for k in uniqueseghosts.keys(): - uh = uniqueseghosts[k] - cmdStr = "%s %s.tar" % (md5Command(), installation_info.install_binary_path) - cmd = Command(uh.address, cmdStr, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap)) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - address = i.remoteHost - if i.results.rc: - logger.error("%s: %s" % (address, i.results.stderr)) - failures.add(address) - else: - fields = i.results.stdout[0].split() - if len(fields) == md5OutputWords(): - md5 = fields[0].strip() - if md5 != installation_info.install_md5: - logger.error("on host %s md5sum %s != expected %s" % (address, md5, installation_info.install_md5)) - failures.add(address) - else: - logger.error("Unexpected output on host %s from md5sum: %s" % (address, i.results.stdout)) - failures.add(address) - - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - if len(failures): - for h in failures: - logger.error("md5sum check of %s.tar on host %s failed" % (installation_info.install_binary_path, h)) - return True - - ################################################################################### - cmdStr = "cd %s; tar xf %s.tar" % (installation_info.install_binary_dir_location, installation_info.install_binary_dir_name) - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - cmdStr = "rm -f %s.tar" % (installation_info.install_binary_path) - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - if installation_info.install_link_name: - cmdStr = "cd %s; rm -f %s; ln -fs %s %s" % (installation_info.install_binary_dir_location, installation_info.install_link_name, installation_info.install_binary_dir_name, installation_info.install_link_name) - if (simpleRemoteCommand(cmdStr, True)): - return True - - if currentUser == "root": - cmdStr = "chown -R %s:%s %s/%s" % (options.user, options.group, installation_info.install_binary_dir_location, installation_info.install_link_name) - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - if currentUser == "root": - cmdStr = "chown -R %s:%s %s" % (options.user, options.group, installation_info.install_binary_path) - if (simpleRemoteCommand(cmdStr, True)): - return True - - ################################################################################### - cmdStr = "rm -f %s.tar.gz" % (installation_info.install_binary_path) - if (simpleLocalCommand(cmdStr, True)): - return True - - ################################################################################### - return False - - -# True is an error -def verifyVersionAtPath(usepath): - - cmdStr = ". %s/greenplum_path.sh; %s/bin/gpssh --version" % (usepath, usepath) - logger.info("remote command: %s" % cmdStr) - - try: - for k in uniqueseghosts.keys(): - uh = uniqueseghosts[k] - cmd = Command(uh.address, cmdStr, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap)) - pool.addCommand(cmd) - pool.join() - items = pool.getCompletedItems() - for i in items: - if i.results.rc: - logger.error("error on host %s with command: %s" % (i.remoteHost, cmdStr)) - return True - if not i.results.stdout: - logger.error("could not find version string from host %s with command: %s" % (i.remoteHost, cmdStr)) - return True - version_string = i.results.stdout[0].strip() - if version_string != installation_info.install_version_string: - logger.error("version string on host %s: '%s' does not match expected: '%s'" % (i.remoteHost, version_string, installation_info.install_version_string)) - return True - - except Exception, e: - logger.error("%s" % e.__str__()) - return True - - return False - - -# True is an error -def verifySoftware(): - - if (getLocalSoftwareVersion()): - return True - - logger.info("version string on master: %s" % installation_info.install_version_string) - - if verifyVersionAtPath(gphome): - return True - - if gphome != installation_info.install_binary_path: - if verifyVersionAtPath(installation_info.install_binary_path): - return True - - return False - -# True is an error -def checkAndFixUserLimits(): - - if not isLinux: - return False - - cmd = "%s/sbin/gpfixuserlimts -f /etc/security/limits.conf -u %s" % (gphome, options.user) - if (simpleLocalCommand(cmd, True)): - return True - - cmdStr = ". %s/greenplum_path.sh; %s" % (gphome, cmd) - if (simpleRemoteCommand(cmdStr, True)): - return True - - return False - - -# True is an error -def interactiveCommand(cmdStr): - - try: - p = subprocess.Popen(cmdStr, shell=True, executable="/bin/bash") - sts = os.waitpid(p.pid, 0)[1] - if sts: - logger.error("error on cmd: %s" % cmdStr) - except Exception, e: - logger.error("Exception running cmd: %s" % cmdStr) - logger.error(e.__str__()) - return True - - return False - -# True is an error -def exchangeKeysUser(): - - if currentUser == "root": - return exchangeKeysAsRootForUser() - else: - return exchangeKeysAsSelf() - -# True is an error -def exchangeKeysAsSelf(): - - cmdStr = 'gpssh-exkeys -f %s' % options.file - return interactiveCommand(cmdStr) - -# True is an error -def exchangeKeysAsRootForUser(): - - ################################################################################### - logger.info("exchange ssh keys for user %s" % options.user) - - testCmd = 'su %s -c "cat %s &> /dev/null"' % (options.user, options.file) - try: - p = subprocess.Popen(testCmd, shell=True, executable="/bin/bash") - sts = os.waitpid(p.pid, 0)[1] - if sts: - logger.error("failed doing a test read of file: %s" % testCmd) - logger.error("%s is not accessible as user %s" % (options.file, options.user)) - return True - except Exception, e: - logger.error("failed doing a test read of file: %s" % testCmd) - logger.error("%s is not accessible as user %s" % (options.file, options.user)) - logger.error(e.__str__()) - return True - - done = False - badPassword = False - child = None - cmdStr = None - - try: - cmdStr = 'su %s -c "gpssh-exkeys -f %s"' % (options.user, options.file) - child = pexpect.spawn(cmdStr) - - index = 0 - while 1: - index = child.expect(["password", "bad", pexpect.EOF, pexpect.TIMEOUT], timeout=GPSSH_EXKEYS_TIMEOUT) - if index == 0: - child.sendline(options.password) - continue - if index == 1: - badPassword = True - break - if index == 2: - done = True - break - if index == 3: - logger.info("Timeout running command: %s" % cmdStr) - break - - except Exception, e: - logger.info("Exception running cmd: %s" % cmdStr) - logger.info(e.__str__()) - - if done: - child.close() - if child.exitstatus: - logger.info("Cmd '%s' failed with error code %s" % (cmdStr, child.exitstatus)) - else: - return False - - # using the supplied password did not work... lets try again in interactive mode - logger.info("gppsh-exkeys failed running from within pexpect ... now try outside of pexpect") - return interactiveCommand(cmdStr) - -# True is an error -def exchangeKeysRoot(): - ################################################################################### - logger.info("exchange ssh keys for user root") - - rootPasswords = list(passwordMap.unique_passwords) - - done = False - child = None - cmdStr = None - passwordIndex = 0 - - - try: - cmdStr = 'gpssh-exkeys -f %s' % options.file - child = pexpect.spawn(cmdStr) - - index = 0 - while 1: - index = child.expect(["password", "bad", pexpect.EOF, pexpect.TIMEOUT], timeout=GPSSH_EXKEYS_TIMEOUT) - if index == 0: - passwordIndex = 0 - child.sendline(rootPasswords[passwordIndex]) - continue - if index == 1: - passwordIndex += 1 - if passwordIndex >= len(rootPasswords): - raise Exception("could not determine root password on all machines") - child.sendline(rootPasswords[passwordIndex]) - continue - if index == 2: - done = True - break - if index == 3: - logger.error("Timeout running command: %s" % cmdStr) - break - - except Exception, e: - logger.info("Error running cmd: %s" % cmdStr) - logger.info(e.__str__()) - - if done: - child.close() - if child.exitstatus: - logger.info("Cmd '%s' failed with error code %s" % (cmdStr, child.exitstatus)) - else: - return False - - # using the supplied password did not work... lets try again in interactive mode - logger.info("gppsh-exkeys failed running from within pexpect ... now try outside of pexpect") - return interactiveCommand(cmdStr) - - -def earlyExit(error = False): - if error: - logger.fatal("early exit from gpseginstall") - if pool: - try: - pool.join() - pool.haltWork() - pool.joinWorkers() - except KeyboardInterrupt: - logger.info("Exiting, please wait ...") - - if error: - sys.exit(1) - else: - logger.info("SUCCESS -- Requested commands completed") - sys.exit(0) - -if (parseargs()): - sys.exit(0) - -pool = WorkerPool() - -try: - - if (readHostList()): - earlyExit(True) - - if (discoverPasswordMap()): - earlyExit(True) - - if (dedupeHosts()): - earlyExit(True) - - if 'u' in commands: - if (checkUsernames()): - earlyExit(True) - - if (addUserIfRequired()): - earlyExit(True) - - if 'c' in commands: - if (chownMasterSoftware()): - earlyExit(True) - - if 's' in commands: - if (copyOverSoftware()): - earlyExit(True) - - if 'p' in commands: - if (changeUserPassword()): - earlyExit(True) - - if 'E' in commands: - if (exchangeKeysRoot()): - earlyExit(True) - - if 'e' in commands: - if (exchangeKeysUser()): - earlyExit(True) - - if 'l' in commands: - if (checkAndFixUserLimits()): - earlyExit(True) - - if 'v' in commands: - if (verifySoftware()): - earlyExit(True) - -except KeyboardInterrupt: - logger.info("Job stopped by user") - earlyExit(True) - -earlyExit(False) diff --git a/gpMgmt/doc/gpinitsystem_help b/gpMgmt/doc/gpinitsystem_help index be1ad7abe8..78060e5e86 100755 --- a/gpMgmt/doc/gpinitsystem_help +++ b/gpMgmt/doc/gpinitsystem_help @@ -400,4 +400,4 @@ Initialize a Greenplum Database array with an optional standby master host: SEE ALSO ***************************************************** -gpseginstall, gpssh-exkeys +gpssh-exkeys diff --git a/gpMgmt/doc/gpscp_help b/gpMgmt/doc/gpscp_help index 866903f94d..a63113af63 100755 --- a/gpMgmt/doc/gpscp_help +++ b/gpMgmt/doc/gpscp_help @@ -131,4 +131,4 @@ SEE ALSO ***************************************************** -gpssh-exkeys, gpssh, gpseginstall +gpssh-exkeys, gpssh diff --git a/gpMgmt/doc/gpseginstall_help b/gpMgmt/doc/gpseginstall_help deleted file mode 100644 index 3602a80461..0000000000 --- a/gpMgmt/doc/gpseginstall_help +++ /dev/null @@ -1,157 +0,0 @@ -COMMAND NAME: gpseginstall - -Installs Greenplum Database on segment hosts. - - -***************************************************** -SYNOPSIS -***************************************************** - -gpseginstall -f host_file [-u user] [-p password] - [-c [u|p|c|s|E|e|l|v]] - -gpseginstall --help - - -***************************************************** -DESCRIPTION -***************************************************** - -The gpseginstall utility provides a simple way to quickly install Greenplum -Database on segment hosts that you specify in a host list file. The utility -does not install or update Greenplum Database on the master host. You can -run gpseginstall as root or as a non-root user. gpseginstall does not perform -database initialization. See gpinitsystem for more information about -initializing Greenplum Database. - -When run as root, gpseginstall default actions are to add a system user -(default is gpadmin), create a password (default is changeme), and deploy and -install Greenplum Database on segment hosts. To do this, gpseginstall locates -the current Greenplum Database binaries on the master from the installation -path in the current user’s environment variables ($GPHOME). It compresses -Greenplum Database software into a tar.gz file and performs an MD5 checksum -to verify file integrity. - -Then, it copies Greenplum Database to the segment hosts, installs -(decompresses) Greenplum Database, and changes the ownership of the Greenplum -Database installation to the system user you specify with the -u option. -Lastly, it exchanges keys between all Greenplum Database hosts as both root -and as the system user you specify with the -u option. gpseginstall also perform -a user limit check and verifies the version number of Greenplum Database on all -the segments. - -If you run gpseginstall as a non-root user, gpseginstall only compresses, copies, -and installs Greenplum Database on segment hosts. It can also exchanges keys -between Greenplum Database hosts for the current system user, and verifies the -version number of Greenplum Database on all the segments. - - -***************************************************** -OPTIONS -***************************************************** - --c | --commands command_option(s) - -This allows you to customize gpseginstall actions. Note that these command -options are executed by default if you do not specify the -c option in the -gpseginstall syntax. - - * u: Adds a system user. (root only) - - * p: Changes the password for a system user. (root only) - - * s: Compresses, copies, decompresses (installs) Greenplum Database on all - segments. - - * c: Changes the ownership of the Greenplum Database installation directory on - the segment hosts. (root only) - - * E: Exchange keys between Greenplum Database master and segment hosts for the - root user. (root only) - - * e: Exchange keys between Greenplum Database master and segment hosts for the - non-root system user. - - * l: (Linux only) Checks and modifies the user limits configuration file - (/etc/security/limits.conf file) when adding a new user to segment hosts. - (root only) - - * v: Verifies the version of Greenplum Database running on all segments. - gpseginstall checks the version number of the Greenplum Database - installation referenced by the $GPHOME environment variable and symbolic - link to the installation directory. An error occurs if there is a version - number mismatch or the Greenplum Database installation directory cannot be - found. - - --f | --file host_file - -This option is required. This specifies the file that lists the segment hosts -onto which you want to install Greenplum Database. - -The host list file must have one host name per line and includes a host name -for each segment host in your Greenplum system. Make sure there are no blank -lines or extra spaces. If a host has multiple configured host names, use only -one host name per host. For example: - -sdw1-1 -sdw2-1 -sdw3-1 -sdw4-1 - -If available, you can use the same gpssh-exkeys host list file you used to -exchange keys between Greenplum Database hosts. - - --p | --password password - -This sets the password for the user you specify with the -u option. The -default password is changeme. This option is only available when you run -gpseginstall as root. - -Best practises: Always use passwords, do not use default passwords, -change default passwords immediately after installation. - - --u | --user user - -This specifies the system user. This user is also the Greenplum Database -administrative user. This user owns Greenplum Database installation and -administers the database. This is also the user under which Greenplum -Database is started/initialized. This option is only available when you run -gpseginstall as root. The default is gpadmin. - - ---help (help) - -Displays the online help. - - -***************************************************** -EXAMPLES -***************************************************** - -As root, install a Greenplum Database on all segments, leave the system user -as the default (gpadmin) and set the gpadmin password to secret123: - -$ gpseginstall -f my_host_list_file -p secret123 - -As a non-root user, compress and copy Greenplum Database binaries to all -segments (as gpadmin) - -$ gpseginstall -f host_file - -As root, add a user (gpadmin2), set the password for the user (secret1234), -exchange keys between hosts as the new user, check user limits, and verify -version numbers, but do not change ownership of Greenplum binaries, -compress/copy/ install Greenplum Database on segments, or exchange keys as -root. - -$ gpseginstall -f host_file -u gpadmin2 -p secret1234 -c upelv - - -***************************************************** -SEE ALSO -***************************************************** - -gpinitsystem \ No newline at end of file diff --git a/gpMgmt/doc/gpssh-exkeys_help b/gpMgmt/doc/gpssh-exkeys_help index 2ae70eab2f..ce070dcf30 100755 --- a/gpMgmt/doc/gpssh-exkeys_help +++ b/gpMgmt/doc/gpssh-exkeys_help @@ -163,5 +163,5 @@ hosts sdw4 and sdw5 as part of a system expansion operation: SEE ALSO ***************************************************** -gpssh, gpscp, gpseginstall +gpssh, gpscp diff --git a/gpMgmt/test/behave_utils/cluster_setup.py b/gpMgmt/test/behave_utils/cluster_setup.py index 1d4a2acd9a..96489fe15d 100755 --- a/gpMgmt/test/behave_utils/cluster_setup.py +++ b/gpMgmt/test/behave_utils/cluster_setup.py @@ -4,36 +4,6 @@ import socket import inspect from gppylib.commands.base import Command -class GpSegInstall(Command): - """ - This is a wrapper for gpseginstall - """ - def __init__(self, gphome, hosts): - self.hostfile = '/tmp/gpseginstall_hosts' - self.gphome = gphome - self.hosts = hosts - cmd_str = "gpseginstall -f %s -u %s" %(self.hostfile, getpass.getuser()) - Command.__init__(self, 'run gpseginstall', cmd_str) - - def run(self, validate=True): - print "Running gpseginstall: %s" % self - with open(self.hostfile, 'w') as f: - for host in self.hosts[1:]: - f.write(host) - f.write('\n') - - res = run_shell_command('gpssh-exkeys -f %s' %self.hostfile, 'gpssh-exkeys') - if res['rc'] > 0: - raise Exception("Failed to do gpssh-exkeys: %s" %res['stderr']) - - res = run_shell_command("gpssh -f %s -e 'mkdir -p %s'" %(self.hostfile, self.gphome), 'gpssh-exkeys') - if res['rc'] > 0: - raise Exception("Failed to create gphome directories on segments: %s" %res[stderr]) - - Command.run(self, validateAfter=validate) - result = self.get_results() - return result - class GpDeleteSystem(Command): """This is a wrapper for gpdeletesystem.""" def __init__(self, mdd=None): -- GitLab